Exemplo n.º 1
0
    def run(self):
        self.parent.clear_plots()
        if self.data is None:
            length, tMin, tMax, fMin, fMax, lMin, lMax, peakF, peakL, peakT = ("",) * 10
        else:
            length = len(self.data)
            tMin = format_time(self.extent.tMin, True)
            tMax = format_time(self.extent.tMax, True)
            fMin = format_precision(self.settings, freq=self.extent.fMin, fancyUnits=True)
            fMax = format_precision(self.settings, freq=self.extent.fMax, fancyUnits=True)
            lMin = format_precision(self.settings, level=self.extent.lMin, fancyUnits=True)
            lMax = format_precision(self.settings, level=self.extent.lMax, fancyUnits=True)
            peak = self.extent.get_peak_flt()
            peakF = format_precision(self.settings, freq=peak[0], fancyUnits=True)
            peakL = format_precision(self.settings, level=peak[1], fancyUnits=True)
            peakT = format_time(peak[2], True)

        text = [
            ["Sweeps", "", length],
            ["Extents", "", ""],
            ["", "Start", tMin],
            ["", "End", tMax],
            ["", "Min frequency", fMin],
            ["", "Max frequency", fMax],
            ["", "Min level", lMin],
            ["", "Max level", lMax],
            ["Peak", "", ""],
            ["", "Level", peakL],
            ["", "Frequency", peakF],
            ["", "Time", peakT],
        ]

        table = Table(self.axes, loc="center")
        table.set_gid("table")

        rows = len(text)
        cols = len(text[0])
        for row in xrange(rows):
            for col in xrange(cols):
                table.add_cell(row, col, text=text[row][col], width=1.0 / cols, height=1.0 / rows)

        if self.settings.grid:
            colour = "LightGray"
        else:
            colour = "w"
        set_table_colour(table, colour)

        for i in range(3):
            table.auto_set_column_width(i)

        self.axes.add_table(table)
        self.parent.redraw_plot()

        self.parent.threadPlot = None
Exemplo n.º 2
0
    def __update_grid(self, level=None):
        self.gridLog.ClearGrid()

        fontCell = self.gridLog.GetDefaultCellFont()
        fontSize = fontCell.GetPointSize()
        fontStyle = fontCell.GetStyle()
        fontWeight = fontCell.GetWeight()
        font = wx.Font(fontSize, wx.FONTFAMILY_MODERN, fontStyle,
                       fontWeight)

        i = 0
        for event in self.log.get(level):
            self.gridLog.SetCellValue(i, 0, format_time(event[0], True))
            self.gridLog.SetCellValue(i, 1, self.log.TEXT_LEVEL[event[1]])
            eventText = '\n'.join(textwrap.wrap(event[2], width=70))
            self.gridLog.SetCellValue(i, 2, eventText)
            self.gridLog.SetCellFont(i, 0, font)
            self.gridLog.SetCellFont(i, 1, font)
            self.gridLog.SetCellFont(i, 2, font)
            self.gridLog.SetCellAlignment(i, 0, wx.ALIGN_LEFT, wx.ALIGN_CENTRE)
            self.gridLog.SetCellAlignment(i, 1, wx.ALIGN_LEFT, wx.ALIGN_CENTRE)
            i += 1

        self.gridLog.AppendRows()
        self.gridLog.SetCellValue(i, 0, '#' * 18)
        self.gridLog.SetCellValue(i, 1, '#' * 5)
        self.gridLog.SetCellValue(i, 2, '#' * 80)
        self.gridLog.AutoSize()
        self.gridLog.DeleteRows(i)

        size = self.gridLog.GetBestSize()
        size.width += wx.SystemSettings.GetMetric(wx.SYS_VSCROLL_X) + 10
        size.height = 400
        self.SetClientSize(size)
        self.sizer.Layout()
Exemplo n.º 3
0
    def annotate_plot(self):

        self.clear_markers()
        fMax, lMax, tMax = self.extent.get_peak_flt()
        y = epoch_to_mpl(tMax)

        start, stop = self.axes.get_xlim()
        textX = ((stop - start) / 50.0) + fMax
        when = format_time(fMax)

        if(matplotlib.__version__ < '1.3'):
            self.axes.annotate('{0:.6f}MHz\n{1:.2f}dB\n{2}'.format(fMax,
                                                                   lMax,
                                                                   when),
                               xy=(fMax, y), xytext=(textX, y),
                               ha='left', va='bottom', size='small',
                               color='w', gid='peak')
            self.axes.plot(fMax, y, marker='x', markersize=10, color='w',
                           mew=3, gid='peak')
            self.axes.plot(fMax, y, marker='x', markersize=10, color='r',
                           gid='peak')
        else:
            effect = patheffects.withStroke(linewidth=3, foreground="w",
                                            alpha=0.75)
            self.axes.annotate('{0:.6f}MHz\n{1:.2f}dB\n{2}'.format(fMax,
                                                                   lMax,
                                                                   when),
                               xy=(fMax, y), xytext=(textX, y),
                               ha='left', va='bottom', size='small',
                               path_effects=[effect], gid='peak')
            self.axes.plot(fMax, y, marker='x', markersize=10, color='r',
                           path_effects=[effect], gid='peak')
Exemplo n.º 4
0
    def __annotate_plot(self):
        f, l, t = self.extent.get_peak_flt()
        when = format_time(t)
        tPos = utc_to_mpl(t)

        text = '{}\n{}\n{when}'.format(*format_precision(self.settings, f, l,
                                                         fancyUnits=True),
                                       when=when)
        if matplotlib.__version__ < '1.3':
            self.axes.text(f, tPos, l,
                           text,
                           ha='left', va='bottom', size='x-small', gid='peak')
            self.axes.plot([f], [tPos], [l], marker='x', markersize=10,
                           mew=3, color='w', gid='peak')
            self.axes.plot([f], [tPos], [l], marker='x', markersize=10,
                           color='r', gid='peak')
        else:
            effect = patheffects.withStroke(linewidth=2, foreground="w",
                                            alpha=0.75)
            self.axes.text(f, tPos, l,
                           text,
                           ha='left', va='bottom', size='x-small', gid='peak',
                           path_effects=[effect])
            self.axes.plot([f], [tPos], [l], marker='x', markersize=10,
                           color='r', gid='peak', path_effects=[effect])
Exemplo n.º 5
0
    def __plot_peak(self, peakF, peakL, peakT):
        when = format_time(peakT)
        tPos = utc_to_mpl(peakT)

        text = '{}\n{}\n{when}'.format(*format_precision(self.settings,
                                                         peakF, peakL,
                                                         fancyUnits=True),
                                       when=when)
        if matplotlib.__version__ < '1.3':
            self.axes.text(peakF, tPos, peakL,
                           text,
                           ha='left', va='bottom', size='x-small', gid='peakText')
            self.axes.plot([peakF], [tPos], [peakL], marker='x', markersize=10,
                           mew=3, color='w', gid='peak')
            self.axes.plot([peakF], [tPos], [peakL], marker='x', markersize=10,
                           color='r', gid='peakShadow')
        else:
            effect = patheffects.withStroke(linewidth=2, foreground="w",
                                            alpha=0.75)
            self.axes.text(peakF, tPos, peakL,
                           text,
                           ha='left', va='bottom', size='x-small', gid='peakText',
                           path_effects=[effect])
            self.axes.plot([peakF], [tPos], [peakL], marker='x', markersize=10,
                           color='r', gid='peak', path_effects=[effect])
Exemplo n.º 6
0
    def __plot_peak(self, peakF, peakL, peakT):
        when = format_time(peakT)
        tPos = utc_to_mpl(peakT)

        text = '{}\n{}\n{when}'.format(*format_precision(self.settings,
                                                         peakF, peakL,
                                                         fancyUnits=True),
                                       when=when)
        if matplotlib.__version__ < '1.3':
            self.axes.text(peakF, tPos, peakL,
                           text,
                           ha='left', va='bottom', size='x-small', gid='peakText')
            self.axes.plot([peakF], [tPos], [peakL], marker='x', markersize=10,
                           mew=3, color='w', gid='peak')
            self.axes.plot([peakF], [tPos], [peakL], marker='x', markersize=10,
                           color='r', gid='peakShadow')
        else:
            effect = patheffects.withStroke(linewidth=2, foreground="w",
                                            alpha=0.75)
            self.axes.text(peakF, tPos, peakL,
                           text,
                           ha='left', va='bottom', size='x-small', gid='peakText',
                           path_effects=[effect])
            self.axes.plot([peakF], [tPos], [peakL], marker='x', markersize=10,
                           color='r', gid='peak', path_effects=[effect])
Exemplo n.º 7
0
    def __annotate_plot(self):
        self.__clear_markers()
        fMax, lMax, tMax = self.extent.get_peak_flt()
        y = epoch_to_mpl(tMax)

        start, stop = self.axes.get_xlim()
        textX = ((stop - start) / 50.0) + fMax
        when = format_time(tMax)

        text = "{0:.6f} MHz\n{1:.2f} $\mathsf{{dB/\sqrt{{Hz}}}}$\n{2}".format(fMax, lMax, when)
        if matplotlib.__version__ < "1.3":
            self.axes.annotate(
                text, xy=(fMax, y), xytext=(textX, y), ha="left", va="bottom", size="x-small", color="w", gid="peak"
            )
            self.axes.plot(fMax, y, marker="x", markersize=10, color="w", mew=3, gid="peak")
            self.axes.plot(fMax, y, marker="x", markersize=10, color="r", gid="peak")
        else:
            effect = patheffects.withStroke(linewidth=2, foreground="w", alpha=0.75)
            self.axes.annotate(
                text,
                xy=(fMax, y),
                xytext=(textX, y),
                ha="left",
                va="bottom",
                size="x-small",
                path_effects=[effect],
                gid="peak",
            )
            self.axes.plot(fMax, y, marker="x", markersize=10, color="r", path_effects=[effect], gid="peak")
Exemplo n.º 8
0
    def __plot_peak(self, peakF, peakL, peakT):
        self.__clear_markers()
        y = utc_to_mpl(peakT)

        start, stop = self.axes.get_xlim()
        textX = ((stop - start) / 50.0) + peakF
        when = format_time(peakT)

        text = '{}\n{}\n{when}'.format(*format_precision(self.settings,
                                                         peakF, peakL,
                                                         fancyUnits=True),
                                       when=when)
        if matplotlib.__version__ < '1.3':
            self.axes.annotate(text,
                               xy=(peakF, y), xytext=(textX, y),
                               ha='left', va='bottom', size='x-small',
                               color='w', gid='peakText')
            self.axes.plot(peakF, y, marker='x', markersize=10, color='w',
                           mew=3, gid='peakShadow')
            self.axes.plot(peakF, y, marker='x', markersize=10, color='r',
                           gid='peak')
        else:
            effect = patheffects.withStroke(linewidth=2, foreground="w",
                                            alpha=0.75)
            self.axes.annotate(text,
                               xy=(peakF, y), xytext=(textX, y),
                               ha='left', va='bottom', size='x-small',
                               path_effects=[effect], gid='peakText')
            self.axes.plot(peakF, y, marker='x', markersize=10, color='r',
                           path_effects=[effect], gid='peak')
Exemplo n.º 9
0
    def __plot_peak(self, peakF, peakL, peakT):
        self.__clear_markers()
        y = utc_to_mpl(peakT)

        start, stop = self.axes.get_xlim()
        textX = ((stop - start) / 50.0) + peakF
        when = format_time(peakT)

        text = '{}\n{}\n{when}'.format(*format_precision(self.settings,
                                                         peakF, peakL,
                                                         fancyUnits=True),
                                       when=when)
        if matplotlib.__version__ < '1.3':
            self.axes.annotate(text,
                               xy=(peakF, y), xytext=(textX, y),
                               ha='left', va='bottom', size='x-small',
                               color='w', gid='peakText')
            self.axes.plot(peakF, y, marker='x', markersize=10, color='w',
                           mew=3, gid='peakShadow')
            self.axes.plot(peakF, y, marker='x', markersize=10, color='r',
                           gid='peak')
        else:
            effect = patheffects.withStroke(linewidth=2, foreground="w",
                                            alpha=0.75)
            self.axes.annotate(text,
                               xy=(peakF, y), xytext=(textX, y),
                               ha='left', va='bottom', size='x-small',
                               path_effects=[effect], gid='peakText')
            self.axes.plot(peakF, y, marker='x', markersize=10, color='r',
                           path_effects=[effect], gid='peak')
Exemplo n.º 10
0
    def __update_grid(self, level=None):
        self.gridLog.ClearGrid()

        fontCell = self.gridLog.GetDefaultCellFont()
        fontSize = fontCell.GetPointSize()
        fontStyle = fontCell.GetStyle()
        fontWeight = fontCell.GetWeight()
        font = wx.Font(fontSize, wx.FONTFAMILY_MODERN, fontStyle, fontWeight)

        i = 0
        for event in self.log.get(level):
            self.gridLog.SetCellValue(i, 0, format_time(event[0], True))
            self.gridLog.SetCellValue(i, 1, self.log.TEXT_LEVEL[event[1]])
            eventText = '\n'.join(textwrap.wrap(event[2], width=70))
            self.gridLog.SetCellValue(i, 2, eventText)
            self.gridLog.SetCellFont(i, 0, font)
            self.gridLog.SetCellFont(i, 1, font)
            self.gridLog.SetCellFont(i, 2, font)
            self.gridLog.SetCellAlignment(i, 0, wx.ALIGN_LEFT, wx.ALIGN_CENTRE)
            self.gridLog.SetCellAlignment(i, 1, wx.ALIGN_LEFT, wx.ALIGN_CENTRE)
            i += 1

        self.gridLog.AppendRows()
        self.gridLog.SetCellValue(i, 0, '#' * 18)
        self.gridLog.SetCellValue(i, 1, '#' * 5)
        self.gridLog.SetCellValue(i, 2, '#' * 80)
        self.gridLog.AutoSize()
        self.gridLog.DeleteRows(i)

        size = self.gridLog.GetBestSize()
        size.width += wx.SystemSettings.GetMetric(wx.SYS_VSCROLL_X) + 10
        size.height = 400
        self.SetClientSize(size)
        self.sizer.Layout()
def calc_inception_scores(run_id,
                          log='inception.txt',
                          num_images=50000,
                          minibatch_size=16,
                          eval_reals=True,
                          reverse_order=False):
    result_subdir = misc.locate_result_subdir(run_id)
    network_pkls = misc.list_network_pkls(result_subdir)
    misc.set_output_log_file(os.path.join(result_subdir, log))

    print 'Importing inception score module...'
    import inception_score

    def calc_inception_score(images):
        if images.shape[1] == 1:
            images = images.repeat(3, axis=1)
        images = list(images.transpose(0, 2, 3, 1))
        return inception_score.get_inception_score(images)

    # Load dataset.
    training_set, drange_orig = load_dataset_for_previous_run(result_subdir,
                                                              shuffle=False)
    reals, labels = training_set.get_random_minibatch(num_images, labels=True)

    # Evaluate reals.
    if eval_reals:
        print 'Evaluating inception score for reals...'
        time_begin = time.time()
        mean, std = calc_inception_score(reals)
        print 'Done in %s' % misc.format_time(time.time() - time_begin)
        print '%-32s mean %-8.4f std %-8.4f' % ('reals', mean, std)

    # Evaluate each network snapshot.
    network_pkls = list(enumerate(network_pkls))
    if reverse_order:
        network_pkls = network_pkls[::-1]
    for network_idx, network_pkl in network_pkls:
        print '%-32s' % os.path.basename(network_pkl),
        net = imgapi_load_net(run_id=result_subdir,
                              snapshot=network_pkl,
                              num_example_latents=num_images,
                              random_seed=network_idx,
                              load_dataset=False)
        fakes = imgapi_generate_batch(net,
                                      net.example_latents,
                                      np.random.permutation(labels),
                                      minibatch_size=minibatch_size,
                                      convert_to_uint8=True)
        mean, std = calc_inception_score(fakes)
        print 'mean %-8.4f std %-8.4f' % (mean, std)
    print
    print 'Done.'
Exemplo n.º 12
0
 def annotate_plot(self):
     f, l, t = self.extent.get_peak_flt()
     when = format_time(t)
     tPos = epoch_to_mpl(t)
     if(matplotlib.__version__ < '1.3'):
         self.axes.text(f, tPos, l,
                        '{0:.6f}MHz\n{1:.2f}dB\n{2}'.format(f, l, when),
                        ha='left', va='bottom', size='small', gid='peak')
         self.axes.plot([f], [tPos], [l], marker='x', markersize=10,
                        mew=3, color='w', gid='peak')
         self.axes.plot([f], [tPos], [l], marker='x', markersize=10,
                        color='r', gid='peak')
     else:
         effect = patheffects.withStroke(linewidth=3, foreground="w",
                                         alpha=0.75)
         self.axes.text(f, tPos, l,
                        '{0:.6f}MHz\n{1:.2f}dB\n{2}'.format(f, l, when),
                        ha='left', va='bottom', size='small', gid='peak',
                        path_effects=[effect])
         self.axes.plot([f], [tPos], [l], marker='x', markersize=10,
                        color='r', gid='peak', path_effects=[effect])
Exemplo n.º 13
0
    def __create_last(self):
        loc = self.server.currentLoc
        if loc[0] is None:
            return ''

        last = ('\t\t<Placemark>\n'
                '\t\t\t<name>Last Location</name>\n'
                '\t\t\t<description>{}</description>\n'
                '\t\t\t<styleUrl>#last</styleUrl>\n'
                '\t\t\t<altitudeMode>clampToGround</altitudeMode>\n'
                '\t\t\t<Point>\n').format(format_time(loc[3]))

        if loc[2] is None:
            last += '\t\t\t\t<coordinates>{},{}</coordinates>\n'.\
                format(loc[1], loc[0])
        else:
            last += '\t\t\t\t<coordinates>{},{},{}</coordinates>\n'.\
                format(loc[1], loc[0], loc[2])

        last += ('\t\t\t</Point>\n' '\t\t</Placemark>\n')

        return last
Exemplo n.º 14
0
def main(args,
         model,
         train_loader,
         test_loader,
         decreasing_lr=[],
         print=print):
    if args.cuda:
        model = model.cuda()
    if isinstance(args.decreasing_lr, str):
        args.decreasing_lr = list(map(int, args.decreasing_lr.split(',')))
    optimizer = optim.Adam(model.parameters(),
                           lr=args.lr,
                           weight_decay=args.weight_decay)
    info = {'best_acc': 0, 'old_file': None, 't_begin': time.time()}
    try:
        # ready to go
        for epoch in range(args.epochs):
            info['epoch'] = epoch
            train(args, model, optimizer, train_loader, info, print)

            elapse_time = time.time() - info['t_begin']
            speed_epoch = elapse_time / (epoch + 1)
            speed_batch = speed_epoch / len(train_loader)
            eta = speed_epoch * args.epochs - elapse_time
            print("Elapsed {}, {:.2f} s/epoch, {:.2f} s/batch, ets {:.2f}s".
                  format(misc.format_time(elapse_time), speed_epoch,
                         speed_batch, eta))

            if epoch % args.test_interval == 0:
                eval(args, model, test_loader, info, print)
    except Exception as e:
        import traceback
        traceback.print_exc()
    except KeyboardInterrupt:
        exit()
    finally:
        print("Total Elapse: {:.2f}, Best Result: {:.3f}%".format(
            time.time() - info['t_begin'], info['best_acc']))
Exemplo n.º 15
0
    def __create_last(self):
        loc = self.server.currentLoc
        if loc[0] is None:
            return ''

        last = ('\t\t<Placemark>\n'
                '\t\t\t<name>Last Location</name>\n'
                '\t\t\t<description>{}</description>\n'
                '\t\t\t<styleUrl>#last</styleUrl>\n'
                '\t\t\t<altitudeMode>clampToGround</altitudeMode>\n'
                '\t\t\t<Point>\n').format(format_time(loc[3]))

        if loc[2] is None:
            last += '\t\t\t\t<coordinates>{},{}</coordinates>\n'.\
                format(loc[1], loc[0])
        else:
            last += '\t\t\t\t<coordinates>{},{},{}</coordinates>\n'.\
                format(loc[1], loc[0], loc[2])

        last += ('\t\t\t</Point>\n'
                 '\t\t</Placemark>\n')

        return last
Exemplo n.º 16
0
def train_progressive_gan(
    G_smoothing=0.999,  # Exponential running average of generator weights.
    D_repeats=1,  # How many times the discriminator is trained per G iteration.
    minibatch_repeats=4,  # Number of minibatches to run before adjusting training parameters.
    reset_opt_for_new_lod=True,  # Reset optimizer internal state (e.g. Adam moments) when new layers are introduced?
    total_kimg=15000,  # Total length of the training, measured in thousands of real images.
    mirror_augment=False,  # Enable mirror augment?
    drange_net=[
        -1, 1
    ],  # Dynamic range used when feeding image data to the networks.
    image_snapshot_ticks=1,  # How often to export image snapshots?
    network_snapshot_ticks=10,  # How often to export network snapshots?
    save_tf_graph=False,  # Include full TensorFlow computation graph in the tfevents file?
    save_weight_histograms=False,  # Include weight histograms in the tfevents file?
    resume_run_id=None,  # Run ID or network pkl to resume training from, None = start from scratch.
    resume_snapshot=None,  # Snapshot index to resume training from, None = autodetect.
    resume_kimg=0.0,  # Assumed training progress at the beginning. Affects reporting and training schedule.
    resume_time=0.0
):  # Assumed wallclock time at the beginning. Affects reporting.

    maintenance_start_time = time.time()
    training_set = dataset.load_dataset(data_dir=config.data_dir,
                                        verbose=True,
                                        **config.dataset)
    #resume_run_id = '/dresden/users/mk1391/evl/pggan_logs/logs_celeba128cc/fsg16_results_0/000-pgan-celeba-preset-v2-2gpus-fp32/network-snapshot-010211.pkl'
    resume_with_new_nets = False
    # Construct networks.
    with tf.device('/gpu:0'):
        if resume_run_id is None or resume_with_new_nets:
            print('Constructing networks...')
            G = tfutil.Network('G',
                               num_channels=training_set.shape[0],
                               resolution=training_set.shape[1],
                               label_size=training_set.label_size,
                               **config.G)
            D = tfutil.Network('D',
                               num_channels=training_set.shape[0],
                               resolution=training_set.shape[1],
                               label_size=training_set.label_size,
                               **config.D)
            Gs = G.clone('Gs')
        if resume_run_id is not None:
            network_pkl = misc.locate_network_pkl(resume_run_id,
                                                  resume_snapshot)
            print('Loading networks from "%s"...' % network_pkl)
            rG, rD, rGs = misc.load_pkl(network_pkl)
            if resume_with_new_nets:
                G.copy_vars_from(rG)
                D.copy_vars_from(rD)
                Gs.copy_vars_from(rGs)
            else:
                G = rG
                D = rD
                Gs = rGs
        Gs_update_op = Gs.setup_as_moving_average_of(G, beta=G_smoothing)
    G.print_layers()
    D.print_layers()

    ### pyramid draw fsg (comment out for actual training to happen)
    #draw_gen_fsg(Gs, 10, os.path.join(config.result_dir, 'pggan_fsg_draw.png'))
    #print('>>> done printing fsgs.')
    #return

    print('Building TensorFlow graph...')
    with tf.name_scope('Inputs'):
        lod_in = tf.placeholder(tf.float32, name='lod_in', shape=[])
        lrate_in = tf.placeholder(tf.float32, name='lrate_in', shape=[])
        minibatch_in = tf.placeholder(tf.int32, name='minibatch_in', shape=[])
        minibatch_split = minibatch_in // config.num_gpus
        reals, labels = training_set.get_minibatch_tf()
        reals_split = tf.split(reals, config.num_gpus)
        labels_split = tf.split(labels, config.num_gpus)
    G_opt = tfutil.Optimizer(name='TrainG',
                             learning_rate=lrate_in,
                             **config.G_opt)
    D_opt = tfutil.Optimizer(name='TrainD',
                             learning_rate=lrate_in,
                             **config.D_opt)
    for gpu in range(config.num_gpus):
        with tf.name_scope('GPU%d' % gpu), tf.device('/gpu:%d' % gpu):
            G_gpu = G if gpu == 0 else G.clone(G.name + '_shadow')
            D_gpu = D if gpu == 0 else D.clone(D.name + '_shadow')
            lod_assign_ops = [
                tf.assign(G_gpu.find_var('lod'), lod_in),
                tf.assign(D_gpu.find_var('lod'), lod_in)
            ]
            reals_gpu = process_reals(reals_split[gpu], lod_in, mirror_augment,
                                      training_set.dynamic_range, drange_net)
            labels_gpu = labels_split[gpu]
            with tf.name_scope('G_loss'), tf.control_dependencies(
                    lod_assign_ops):
                G_loss = tfutil.call_func_by_name(
                    G=G_gpu,
                    D=D_gpu,
                    opt=G_opt,
                    training_set=training_set,
                    minibatch_size=minibatch_split,
                    **config.G_loss)
            with tf.name_scope('D_loss'), tf.control_dependencies(
                    lod_assign_ops):
                D_loss = tfutil.call_func_by_name(
                    G=G_gpu,
                    D=D_gpu,
                    opt=D_opt,
                    training_set=training_set,
                    minibatch_size=minibatch_split,
                    reals=reals_gpu,
                    labels=labels_gpu,
                    **config.D_loss)
            G_opt.register_gradients(tf.reduce_mean(G_loss), G_gpu.trainables)
            D_opt.register_gradients(tf.reduce_mean(D_loss), D_gpu.trainables)
    G_train_op = G_opt.apply_updates()
    D_train_op = D_opt.apply_updates()

    print('Setting up snapshot image grid...')
    grid_size, grid_reals, grid_labels, grid_latents = setup_snapshot_image_grid(
        G, training_set, **config.grid)
    ### shift reals
    print('>>> reals shape: ', grid_reals.shape)
    fc_x = 0.5
    fc_y = 0.5
    im_size = grid_reals.shape[-1]
    kernel_loc = 2.*np.pi*fc_x * np.arange(im_size).reshape((1, 1, im_size)) + \
        2.*np.pi*fc_y * np.arange(im_size).reshape((1, im_size, 1))
    kernel_cos = np.cos(kernel_loc)
    kernel_sin = np.sin(kernel_loc)
    reals_t = (grid_reals / 255.) * 2. - 1
    reals_t *= kernel_cos
    grid_reals_sh = np.rint(
        (reals_t + 1.) * 255. / 2.).clip(0, 255).astype(np.uint8)
    ### end shift reals
    sched = TrainingSchedule(total_kimg * 1000, training_set, **config.sched)
    grid_fakes = Gs.run(grid_latents,
                        grid_labels,
                        minibatch_size=sched.minibatch // config.num_gpus)

    ### fft drawing
    #sys.path.insert(1, '/home/mahyar/CV_Res/ganist')
    #from fig_draw import apply_fft_win
    #data_size = 1000
    #latents = np.random.randn(data_size, *Gs.input_shapes[0][1:])
    #labels = np.zeros([latents.shape[0]] + Gs.input_shapes[1][1:])
    #g_samples = Gs.run(latents, labels, minibatch_size=sched.minibatch//config.num_gpus)
    #g_samples = g_samples.transpose(0, 2, 3, 1)
    #print('>>> g_samples shape: {}'.format(g_samples.shape))
    #apply_fft_win(g_samples, 'fft_pggan_hann.png')
    ### end fft drawing

    print('Setting up result dir...')
    result_subdir = misc.create_result_subdir(config.result_dir, config.desc)
    misc.save_image_grid(grid_reals,
                         os.path.join(result_subdir, 'reals.png'),
                         drange=training_set.dynamic_range,
                         grid_size=grid_size)
    ### drawing shifted real images
    misc.save_image_grid(grid_reals_sh,
                         os.path.join(result_subdir, 'reals_sh.png'),
                         drange=training_set.dynamic_range,
                         grid_size=grid_size)
    misc.save_image_grid(grid_fakes,
                         os.path.join(result_subdir, 'fakes%06d.png' % 0),
                         drange=drange_net,
                         grid_size=grid_size)
    ### drawing shifted fake images
    misc.save_image_grid(grid_fakes * kernel_cos,
                         os.path.join(result_subdir, 'fakes%06d_sh.png' % 0),
                         drange=drange_net,
                         grid_size=grid_size)
    summary_log = tf.summary.FileWriter(result_subdir)
    if save_tf_graph:
        summary_log.add_graph(tf.get_default_graph())
    if save_weight_histograms:
        G.setup_weight_histograms()
        D.setup_weight_histograms()

    #### True cosine fft eval
    #fft_data_size = 1000
    #im_size = training_set.shape[1]
    #freq_centers = [(64/128., 64/128.)]
    #true_samples = sample_true(training_set, fft_data_size, dtype=training_set.dtype, batch_size=32).transpose(0, 2, 3, 1) / 255. * 2. - 1.
    #true_fft, true_fft_hann, true_hist = cosine_eval(true_samples, 'true', freq_centers, log_dir=result_subdir)
    #fractal_eval(true_samples, f'koch_snowflake_true', result_subdir)

    print('Training...')
    cur_nimg = int(resume_kimg * 1000)
    cur_tick = 0
    tick_start_nimg = cur_nimg
    tick_start_time = time.time()
    train_start_time = tick_start_time - resume_time
    prev_lod = -1.0
    while cur_nimg < total_kimg * 1000:

        # Choose training parameters and configure training ops.
        sched = TrainingSchedule(cur_nimg, training_set, **config.sched)
        training_set.configure(sched.minibatch, sched.lod)
        if reset_opt_for_new_lod:
            if np.floor(sched.lod) != np.floor(prev_lod) or np.ceil(
                    sched.lod) != np.ceil(prev_lod):
                G_opt.reset_optimizer_state()
                D_opt.reset_optimizer_state()
        prev_lod = sched.lod

        # Run training ops.
        for repeat in range(minibatch_repeats):
            for _ in range(D_repeats):
                tfutil.run(
                    [D_train_op, Gs_update_op], {
                        lod_in: sched.lod,
                        lrate_in: sched.D_lrate,
                        minibatch_in: sched.minibatch
                    })
                cur_nimg += sched.minibatch
            tfutil.run(
                [G_train_op], {
                    lod_in: sched.lod,
                    lrate_in: sched.G_lrate,
                    minibatch_in: sched.minibatch
                })

        # Perform maintenance tasks once per tick.
        done = (cur_nimg >= total_kimg * 1000)
        if cur_nimg >= tick_start_nimg + sched.tick_kimg * 1000 or done:
            cur_tick += 1
            cur_time = time.time()
            tick_kimg = (cur_nimg - tick_start_nimg) / 1000.0
            tick_start_nimg = cur_nimg
            tick_time = cur_time - tick_start_time
            total_time = cur_time - train_start_time
            maintenance_time = tick_start_time - maintenance_start_time
            maintenance_start_time = cur_time

            # Report progress.
            print(
                'tick %-5d kimg %-8.1f lod %-5.2f minibatch %-4d time %-12s sec/tick %-7.1f sec/kimg %-7.2f maintenance %.1f'
                % (tfutil.autosummary('Progress/tick', cur_tick),
                   tfutil.autosummary('Progress/kimg', cur_nimg / 1000.0),
                   tfutil.autosummary('Progress/lod', sched.lod),
                   tfutil.autosummary('Progress/minibatch', sched.minibatch),
                   misc.format_time(
                       tfutil.autosummary('Timing/total_sec', total_time)),
                   tfutil.autosummary('Timing/sec_per_tick', tick_time),
                   tfutil.autosummary('Timing/sec_per_kimg',
                                      tick_time / tick_kimg),
                   tfutil.autosummary('Timing/maintenance_sec',
                                      maintenance_time)))
            tfutil.autosummary('Timing/total_hours',
                               total_time / (60.0 * 60.0))
            tfutil.autosummary('Timing/total_days',
                               total_time / (24.0 * 60.0 * 60.0))
            tfutil.save_summaries(summary_log, cur_nimg)

            # Save snapshots.
            if cur_tick % image_snapshot_ticks == 0 or done:
                grid_fakes = Gs.run(grid_latents,
                                    grid_labels,
                                    minibatch_size=sched.minibatch //
                                    config.num_gpus)
                misc.save_image_grid(grid_fakes,
                                     os.path.join(
                                         result_subdir,
                                         'fakes%06d.png' % (cur_nimg // 1000)),
                                     drange=drange_net,
                                     grid_size=grid_size)
                ### drawing shifted fake images
                misc.save_image_grid(
                    grid_fakes * kernel_cos,
                    os.path.join(result_subdir,
                                 'fakes%06d_sh.png' % (cur_nimg // 1000)),
                    drange=drange_net,
                    grid_size=grid_size)
                ### drawing fsg
                #draw_gen_fsg(Gs, 10, os.path.join(config.result_dir, 'fakes%06d_fsg_draw.png' % (cur_nimg // 1000)))
                ### Gen fft eval
                #gen_samples = sample_gen(Gs, fft_data_size).transpose(0, 2, 3, 1)
                #print(f'>>> fake_samples: max={np.amax(grid_fakes)} min={np.amin(grid_fakes)}')
                #print(f'>>> gen_samples: max={np.amax(gen_samples)} min={np.amin(gen_samples)}')
                #misc.save_image_grid(gen_samples[:25], os.path.join(result_subdir, 'fakes%06d_gsample.png' % (cur_nimg // 1000)), drange=drange_net, grid_size=grid_size)
                #cosine_eval(gen_samples, f'gen_{cur_nimg//1000:06d}', freq_centers, log_dir=result_subdir, true_fft=true_fft, true_fft_hann=true_fft_hann, true_hist=true_hist)
                #fractal_eval(gen_samples, f'koch_snowflake_fakes{cur_nimg//1000:06d}', result_subdir)
            if cur_tick % network_snapshot_ticks == 0 or done:
                misc.save_pkl(
                    (G, D, Gs),
                    os.path.join(
                        result_subdir,
                        'network-snapshot-%06d.pkl' % (cur_nimg // 1000)))

            # Record start time of the next tick.
            tick_start_time = time.time()

    # Write final results.
    misc.save_pkl((G, D, Gs), os.path.join(result_subdir, 'network-final.pkl'))
    summary_log.close()
    open(os.path.join(result_subdir, '_training-done.txt'), 'wt').close()
Exemplo n.º 17
0
    def __init__(self, parent, scanInfo):
        wx.Dialog.__init__(self, parent, title="Scan Properties")

        self.scanInfo = scanInfo

        box = wx.BoxSizer(wx.VERTICAL)

        grid = wx.GridBagSizer(0, 0)

        boxScan = wx.StaticBoxSizer(wx.StaticBox(self, wx.ID_ANY, "Scan"),
                                    wx.HORIZONTAL)

        gridScan = wx.GridBagSizer(0, 0)

        textDesc = wx.StaticText(self, label="Description")
        gridScan.Add(textDesc, (0, 0), (1, 1), wx.ALL, 5)
        self.textCtrlDesc = wx.TextCtrl(self, value=scanInfo.desc,
                                        style=wx.TE_MULTILINE)
        gridScan.Add(self.textCtrlDesc, (0, 1), (2, 2), wx.ALL | wx.EXPAND, 5)

        textStart = wx.StaticText(self, label="Start")
        gridScan.Add(textStart, (2, 0), (1, 1), wx.ALL, 5)
        textCtrlStart = wx.TextCtrl(self, value="Unknown",
                                    style=wx.TE_READONLY)
        if scanInfo.start is not None:
            textCtrlStart.SetValue(str(scanInfo.start))
        gridScan.Add(textCtrlStart, (2, 1), (1, 1), wx.ALL, 5)
        textMHz1 = wx.StaticText(self, wx.ID_ANY, label="MHz")
        gridScan.Add(textMHz1, (2, 2), (1, 1), wx.ALL, 5)

        textStop = wx.StaticText(self, label="Stop")
        gridScan.Add(textStop, (3, 0), (1, 1), wx.ALL, 5)
        textCtrlStop = wx.TextCtrl(self, value="Unknown",
                                   style=wx.TE_READONLY)
        if scanInfo.stop is not None:
            textCtrlStop.SetValue(str(scanInfo.stop))
        gridScan.Add(textCtrlStop, (3, 1), (1, 1), wx.ALL, 5)
        textMHz2 = wx.StaticText(self, label="MHz")
        gridScan.Add(textMHz2, (3, 2), (1, 1), wx.ALL, 5)

        textDwell = wx.StaticText(self, label="Dwell")
        gridScan.Add(textDwell, (4, 0), (1, 1), wx.ALL, 5)
        textCtrlDwell = wx.TextCtrl(self, value="Unknown",
                                    style=wx.TE_READONLY)
        if scanInfo.dwell is not None:
            textCtrlDwell.SetValue(str(scanInfo.dwell))
        gridScan.Add(textCtrlDwell, (4, 1), (1, 1), wx.ALL, 5)
        textSeconds = wx.StaticText(self, label="seconds")
        gridScan.Add(textSeconds, (4, 2), (1, 1), wx.ALL, 5)

        textNfft = wx.StaticText(self, label="FFT Size")
        gridScan.Add(textNfft, (5, 0), (1, 1), wx.ALL, 5)
        textCtrlNfft = wx.TextCtrl(self, value="Unknown", style=wx.TE_READONLY)
        if scanInfo.nfft is not None:
            textCtrlNfft.SetValue(str(scanInfo.nfft))
        gridScan.Add(textCtrlNfft, (5, 1), (1, 1), wx.ALL, 5)

        textRbw = wx.StaticText(self, label="RBW")
        gridScan.Add(textRbw, (6, 0), (1, 1), wx.ALL, 5)
        rbw = ((SAMPLE_RATE / scanInfo.nfft) / 1000.0) * 2.0
        textCtrlStop = wx.TextCtrl(self, value="{0:.3f}".format(rbw),
                                   style=wx.TE_READONLY)
        gridScan.Add(textCtrlStop, (6, 1), (1, 1), wx.ALL, 5)
        textKHz = wx.StaticText(self, label="kHz")
        gridScan.Add(textKHz, (6, 2), (1, 1), wx.ALL, 5)

        textTime = wx.StaticText(self, label="First scan")
        gridScan.Add(textTime, (7, 0), (1, 1), wx.ALL, 5)
        textCtrlTime = wx.TextCtrl(self, value="Unknown", style=wx.TE_READONLY)
        if scanInfo.timeFirst is not None:
            textCtrlTime.SetValue(format_time(scanInfo.timeFirst, True))
        gridScan.Add(textCtrlTime, (7, 1), (1, 1), wx.ALL, 5)

        textTime = wx.StaticText(self, label="Last scan")
        gridScan.Add(textTime, (8, 0), (1, 1), wx.ALL, 5)
        textCtrlTime = wx.TextCtrl(self, value="Unknown", style=wx.TE_READONLY)
        if scanInfo.timeLast is not None:
            textCtrlTime.SetValue(format_time(scanInfo.timeLast, True))
        gridScan.Add(textCtrlTime, (8, 1), (1, 1), wx.ALL, 5)

        textLat = wx.StaticText(self, label="Latitude")
        gridScan.Add(textLat, (9, 0), (1, 1), wx.ALL, 5)
        self.textCtrlLat = wx.TextCtrl(self, value="Unknown")
        self.textCtrlLat.SetValidator(ValidatorCoord(True))
        if scanInfo.lat is not None:
            self.textCtrlLat.SetValue(str(scanInfo.lat))
        gridScan.Add(self.textCtrlLat, (9, 1), (1, 1), wx.ALL, 5)

        textLon = wx.StaticText(self, label="Longitude")
        gridScan.Add(textLon, (10, 0), (1, 1), wx.ALL, 5)
        self.textCtrlLon = wx.TextCtrl(self, value="Unknown")
        self.textCtrlLon.SetValidator(ValidatorCoord(False))
        if scanInfo.lon is not None:
            self.textCtrlLon.SetValue(str(scanInfo.lon))
        gridScan.Add(self.textCtrlLon, (10, 1), (1, 1), wx.ALL, 5)

        boxScan.Add(gridScan, 0, 0, 5)

        grid.Add(boxScan, (0, 0), (1, 1), wx.ALL | wx.EXPAND, 5)

        boxDevice = wx.StaticBoxSizer(wx.StaticBox(self, label="Device"),
                                      wx.VERTICAL)

        gridDevice = wx.GridBagSizer(0, 0)

        textName = wx.StaticText(self, label="Name")
        gridDevice.Add(textName, (0, 0), (1, 1), wx.ALL, 5)
        textCtrlName = wx.TextCtrl(self, value="Unknown", style=wx.TE_READONLY)
        if scanInfo.name is not None:
            textCtrlName.SetValue(scanInfo.name)
        gridDevice.Add(textCtrlName, (0, 1), (1, 2), wx.ALL | wx.EXPAND, 5)

        textTuner = wx.StaticText(self, label="Tuner")
        gridDevice.Add(textTuner, (1, 0), (1, 1), wx.ALL, 5)
        textCtrlTuner = wx.TextCtrl(self, value="Unknown",
                                    style=wx.TE_READONLY)
        if scanInfo.tuner != -1:
            textCtrlTuner.SetValue(TUNER[scanInfo.tuner])
        gridDevice.Add(textCtrlTuner, (1, 1), (1, 2), wx.ALL | wx.EXPAND, 5)

        testGain = wx.StaticText(self, label="Gain")
        gridDevice.Add(testGain, (2, 0), (1, 1), wx.ALL, 5)
        textCtrlGain = wx.TextCtrl(self, value="Unknown", style=wx.TE_READONLY)
        if scanInfo.gain is not None:
            textCtrlGain.SetValue(str(scanInfo.gain))
        gridDevice.Add(textCtrlGain, (2, 1), (1, 1), wx.ALL, 5)
        textDb = wx.StaticText(self, label="dB")
        gridDevice.Add(textDb, (2, 2), (1, 1), wx.ALL, 5)

        textLo = wx.StaticText(self, label="LO")
        gridDevice.Add(textLo, (3, 0), (1, 1), wx.ALL, 5)
        textCtrlLo = wx.TextCtrl(self, value="Unknown", style=wx.TE_READONLY)
        if scanInfo.lo is not None:
            textCtrlLo.SetValue(str(scanInfo.lo))
        gridDevice.Add(textCtrlLo, (3, 1), (1, 1), wx.ALL, 5)
        textMHz3 = wx.StaticText(self, label="MHz")
        gridDevice.Add(textMHz3, (3, 2), (1, 1), wx.ALL, 5)

        textCal = wx.StaticText(self, label="Calibration")
        gridDevice.Add(textCal, (4, 0), (1, 1), wx.ALL, 5)
        textCtrlCal = wx.TextCtrl(self, value="Unknown", style=wx.TE_READONLY)
        if scanInfo.calibration is not None:
            textCtrlCal.SetValue(str(scanInfo.calibration))
        gridDevice.Add(textCtrlCal, (4, 1), (1, 1), wx.ALL, 5)
        testPpm = wx.StaticText(self, label="ppm")
        gridDevice.Add(testPpm, (4, 2), (1, 1), wx.ALL, 5)

        boxDevice.Add(gridDevice, 1, wx.EXPAND, 5)

        grid.Add(boxDevice, (1, 0), (1, 1), wx.ALL | wx.EXPAND, 5)

        box.Add(grid, 1, wx.ALL | wx.EXPAND, 5)

        sizerButtons = wx.StdDialogButtonSizer()
        buttonOk = wx.Button(self, wx.ID_OK)
        buttonCancel = wx.Button(self, wx.ID_CANCEL)
        sizerButtons.AddButton(buttonOk)
        sizerButtons.AddButton(buttonCancel)
        sizerButtons.Realize()
        self.Bind(wx.EVT_BUTTON, self.__on_ok, buttonOk)
        box.Add(sizerButtons, 0, wx.ALIGN_RIGHT | wx.ALL, 5)

        self.SetSizerAndFit(box)
Exemplo n.º 18
0
    def __init__(self, parent, spectrum, settings):
        self.spectrum = spectrum
        self.settings = settings
        self.sweeps = None
        self.isExporting = False

        wx.Dialog.__init__(self, parent=parent, title='Export Plot Sequence')

        self.queue = Queue.Queue()
        self.timer = wx.Timer(self)
        self.Bind(wx.EVT_TIMER, self.__on_timer, self.timer)
        self.timer.Start(self.POLL)

        self.figure = matplotlib.figure.Figure(facecolor='white')
        self.canvas = FigureCanvas(self, -1, self.figure)
        self.plot = Plotter(self.queue, self.figure, settings)

        textPlot = wx.StaticText(self, label='Plot')

        self.checkAxes = wx.CheckBox(self, label='Axes')
        self.checkAxes.SetValue(True)
        self.Bind(wx.EVT_CHECKBOX, self.__on_axes, self.checkAxes)
        self.checkGrid = wx.CheckBox(self, label='Grid')
        self.checkGrid.SetValue(True)
        self.Bind(wx.EVT_CHECKBOX, self.__on_grid, self.checkGrid)
        self.checkBar = wx.CheckBox(self, label='Bar')
        self.checkBar.SetValue(True)
        self.Bind(wx.EVT_CHECKBOX, self.__on_bar, self.checkBar)

        sizerCheck = wx.BoxSizer(wx.HORIZONTAL)
        sizerCheck.Add(self.checkAxes, flag=wx.ALL, border=5)
        sizerCheck.Add(self.checkGrid, flag=wx.ALL, border=5)
        sizerCheck.Add(self.checkBar, flag=wx.ALL, border=5)

        textRange = wx.StaticText(self, label='Range')

        self.sweepTimeStamps = sorted([timeStamp for timeStamp in spectrum.keys()])
        sweepChoices = [format_time(timeStamp, True) for timeStamp in self.sweepTimeStamps]

        textStart = wx.StaticText(self, label="Start")
        self.choiceStart = wx.Choice(self, choices=sweepChoices)
        self.choiceStart.SetSelection(0)
        self.Bind(wx.EVT_CHOICE, self.__on_choice, self.choiceStart)

        textEnd = wx.StaticText(self, label="End")
        self.choiceEnd = wx.Choice(self, choices=sweepChoices)
        self.choiceEnd.SetSelection(len(self.sweepTimeStamps) - 1)
        self.Bind(wx.EVT_CHOICE, self.__on_choice, self.choiceEnd)

        textSweeps = wx.StaticText(self, label='Sweeps')
        self.textSweeps = wx.StaticText(self, label="")

        textOutput = wx.StaticText(self, label='Output')

        self.textSize = wx.StaticText(self)
        buttonSize = wx.Button(self, label='Change...')
        buttonSize.SetToolTipString('Change exported image size')
        self.Bind(wx.EVT_BUTTON, self.__on_imagesize, buttonSize)
        self.__show_image_size()

        buttonBrowse = wx.Button(self, label='Browse...')
        self.Bind(wx.EVT_BUTTON, self.__on_browse, buttonBrowse)

        self.editDir = wx.TextCtrl(self)
        self.editDir.SetValue(settings.dirExport)

        font = textPlot.GetFont()
        fontSize = font.GetPointSize()
        font.SetPointSize(fontSize + 4)
        textPlot.SetFont(font)
        textRange.SetFont(font)
        textOutput.SetFont(font)

        sizerButtons = wx.StdDialogButtonSizer()
        buttonOk = wx.Button(self, wx.ID_OK)
        buttonCancel = wx.Button(self, wx.ID_CANCEL)
        sizerButtons.AddButton(buttonOk)
        sizerButtons.AddButton(buttonCancel)
        sizerButtons.Realize()
        self.Bind(wx.EVT_BUTTON, self.__on_ok, buttonOk)

        sizerGrid = wx.GridBagSizer(5, 5)
        sizerGrid.Add(self.canvas, pos=(0, 0), span=(10, 6),
                      flag=wx.EXPAND | wx.ALL, border=5)
        sizerGrid.Add(textPlot, pos=(0, 7),
                      flag=wx.TOP | wx.BOTTOM, border=5)
        sizerGrid.Add(sizerCheck, pos=(1, 7), span=(1, 2),
                      flag=wx.ALL, border=5)
        sizerGrid.Add(textRange, pos=(2, 7),
                      flag=wx.TOP | wx.BOTTOM, border=5)
        sizerGrid.Add(textStart, pos=(3, 7),
                      flag=wx.ALIGN_CENTRE_VERTICAL | wx.ALL, border=5)
        sizerGrid.Add(self.choiceStart, pos=(3, 8),
                      flag=wx.ALL, border=5)
        sizerGrid.Add(textEnd, pos=(4, 7),
                      flag=wx.ALIGN_CENTRE_VERTICAL | wx.ALL, border=5)
        sizerGrid.Add(self.choiceEnd, pos=(4, 8),
                      flag=wx.ALL, border=5)
        sizerGrid.Add(textSweeps, pos=(5, 7),
                      flag=wx.ALIGN_CENTRE_VERTICAL | wx.ALL, border=5)
        sizerGrid.Add(self.textSweeps, pos=(5, 8),
                      flag=wx.ALIGN_CENTRE_VERTICAL | wx.ALL, border=5)
        sizerGrid.Add(textOutput, pos=(6, 7),
                      flag=wx.TOP | wx.BOTTOM, border=5)
        sizerGrid.Add(self.textSize, pos=(7, 7),
                      flag=wx.ALIGN_CENTRE_VERTICAL | wx.ALL, border=5)
        sizerGrid.Add(buttonSize, pos=(7, 8),
                      flag=wx.ALL, border=5)
        sizerGrid.Add(self.editDir, pos=(8, 7), span=(1, 2),
                      flag=wx.ALL | wx.EXPAND, border=5)
        sizerGrid.Add(buttonBrowse, pos=(9, 7),
                      flag=wx.ALL, border=5)
        sizerGrid.Add(sizerButtons, pos=(10, 7), span=(1, 2),
                      flag=wx.ALIGN_RIGHT | wx.ALL, border=5)

        self.SetSizerAndFit(sizerGrid)

        self.__draw_plot()
def train_progressive_gan(
    G_smoothing             = 0.999,        # Exponential running average of generator weights.
    D_repeats               = 1,            # How many times the discriminator is trained per G iteration.
    minibatch_repeats       = 4,            # Number of minibatches to run before adjusting training parameters.
    reset_opt_for_new_lod   = True,         # Reset optimizer internal state (e.g. Adam moments) when new layers are introduced?
    total_kimg              = 15000,        # Total length of the training, measured in thousands of real images.
    mirror_augment          = False,        # Enable mirror augment?
    drange_net              = [-1,1],       # Dynamic range used when feeding image data to the networks.
    image_snapshot_ticks    = 1,            # How often to export image snapshots?
    network_snapshot_ticks  = 10,           # How often to export network snapshots?
    save_tf_graph           = False,        # Include full TensorFlow computation graph in the tfevents file?
    save_weight_histograms  = False,        # Include weight histograms in the tfevents file?
    resume_run_id           = None,         # Run ID or network pkl to resume training from, None = start from scratch.
    resume_snapshot         = None,         # Snapshot index to resume training from, None = autodetect.
    resume_kimg             = 0.0,          # Assumed training progress at the beginning. Affects reporting and training schedule.
    resume_time             = 0.0):         # Assumed wallclock time at the beginning. Affects reporting.

    maintenance_start_time = time.time()
    training_set = dataset.load_dataset(data_dir=config.data_dir, verbose=True, **config.dataset)

    # Construct networks.
    with tf.device('/gpu:0'):
        if resume_run_id is not None:
            network_pkl = misc.locate_network_pkl(resume_run_id, resume_snapshot)
            print('Loading networks from "%s"...' % network_pkl)
            G, D, Gs = misc.load_pkl(network_pkl)
        else:
            print('Constructing networks...')
            G = tfutil.Network('G', num_channels=training_set.shape[0], resolution=training_set.shape[1], label_size=training_set.label_size, **config.G)
            D = tfutil.Network('D', num_channels=training_set.shape[0], resolution=training_set.shape[1], label_size=training_set.label_size, **config.D)
            Gs = G.clone('Gs')
        Gs_update_op = Gs.setup_as_moving_average_of(G, beta=G_smoothing)
    G.print_layers(); D.print_layers()

    print('Building TensorFlow graph...')
    with tf.name_scope('Inputs'):
        lod_in          = tf.placeholder(tf.float32, name='lod_in', shape=[])
        lrate_in        = tf.placeholder(tf.float32, name='lrate_in', shape=[])
        minibatch_in    = tf.placeholder(tf.int32, name='minibatch_in', shape=[])
        minibatch_split = minibatch_in // config.num_gpus
        reals, labels   = training_set.get_minibatch_tf()
        reals_split     = tf.split(reals, config.num_gpus)
        labels_split    = tf.split(labels, config.num_gpus)
    G_opt = tfutil.Optimizer(name='TrainG', learning_rate=lrate_in, **config.G_opt)
    D_opt = tfutil.Optimizer(name='TrainD', learning_rate=lrate_in, **config.D_opt)
    for gpu in range(config.num_gpus):
        with tf.name_scope('GPU%d' % gpu), tf.device('/gpu:%d' % gpu):
            G_gpu = G if gpu == 0 else G.clone(G.name + '_shadow')
            D_gpu = D if gpu == 0 else D.clone(D.name + '_shadow')
            lod_assign_ops = [tf.assign(G_gpu.find_var('lod'), lod_in), tf.assign(D_gpu.find_var('lod'), lod_in)]
            reals_gpu = process_reals(reals_split[gpu], lod_in, mirror_augment, training_set.dynamic_range, drange_net)
            labels_gpu = labels_split[gpu]
            with tf.name_scope('G_loss'), tf.control_dependencies(lod_assign_ops):
                G_loss = tfutil.call_func_by_name(G=G_gpu, D=D_gpu, opt=G_opt, training_set=training_set, minibatch_size=minibatch_split, **config.G_loss)
            with tf.name_scope('D_loss'), tf.control_dependencies(lod_assign_ops):
                D_loss = tfutil.call_func_by_name(G=G_gpu, D=D_gpu, opt=D_opt, training_set=training_set, minibatch_size=minibatch_split, reals=reals_gpu, labels=labels_gpu, **config.D_loss)
            G_opt.register_gradients(tf.reduce_mean(G_loss), G_gpu.trainables)
            D_opt.register_gradients(tf.reduce_mean(D_loss), D_gpu.trainables)
    G_train_op = G_opt.apply_updates()
    D_train_op = D_opt.apply_updates()

    print('Setting up snapshot image grid...')
    grid_size, grid_reals, grid_labels, grid_latents = setup_snapshot_image_grid(G, training_set, **config.grid)
    sched = TrainingSchedule(total_kimg * 1000, training_set, **config.sched)
    grid_fakes = Gs.run(grid_latents, grid_labels, minibatch_size=sched.minibatch//config.num_gpus)

    print('Setting up result dir...')
    result_subdir = misc.create_result_subdir(config.result_dir, config.desc)
    misc.save_image_grid(grid_reals, os.path.join(result_subdir, 'reals.png'), drange=training_set.dynamic_range, grid_size=grid_size)
    misc.save_image_grid(grid_fakes, os.path.join(result_subdir, 'fakes%06d.png' % 0), drange=drange_net, grid_size=grid_size)
    summary_log = tf.summary.FileWriter(result_subdir)
    if save_tf_graph:
        summary_log.add_graph(tf.get_default_graph())
    if save_weight_histograms:
        G.setup_weight_histograms(); D.setup_weight_histograms()

    print('Training...')
    cur_nimg = int(resume_kimg * 1000)
    cur_tick = 0
    tick_start_nimg = cur_nimg
    tick_start_time = time.time()
    train_start_time = tick_start_time - resume_time
    prev_lod = -1.0
    while cur_nimg < total_kimg * 1000:

        # Choose training parameters and configure training ops.
        sched = TrainingSchedule(cur_nimg, training_set, **config.sched)
        training_set.configure(sched.minibatch, sched.lod)
        if reset_opt_for_new_lod:
            if np.floor(sched.lod) != np.floor(prev_lod) or np.ceil(sched.lod) != np.ceil(prev_lod):
                G_opt.reset_optimizer_state(); D_opt.reset_optimizer_state()
        prev_lod = sched.lod

        # Run training ops.
        for repeat in range(minibatch_repeats):
            for _ in range(D_repeats):
                tfutil.run([D_train_op, Gs_update_op], {lod_in: sched.lod, lrate_in: sched.D_lrate, minibatch_in: sched.minibatch})
                cur_nimg += sched.minibatch
            tfutil.run([G_train_op], {lod_in: sched.lod, lrate_in: sched.G_lrate, minibatch_in: sched.minibatch})

        # Perform maintenance tasks once per tick.
        done = (cur_nimg >= total_kimg * 1000)
        if cur_nimg >= tick_start_nimg + sched.tick_kimg * 1000 or done:
            cur_tick += 1
            cur_time = time.time()
            tick_kimg = (cur_nimg - tick_start_nimg) / 1000.0
            tick_start_nimg = cur_nimg
            tick_time = cur_time - tick_start_time
            total_time = cur_time - train_start_time
            maintenance_time = tick_start_time - maintenance_start_time
            maintenance_start_time = cur_time

            # Report progress.
            print('tick %-5d kimg %-8.1f lod %-5.2f minibatch %-4d time %-12s sec/tick %-7.1f sec/kimg %-7.2f maintenance %.1f' % (
                tfutil.autosummary('Progress/tick', cur_tick),
                tfutil.autosummary('Progress/kimg', cur_nimg / 1000.0),
                tfutil.autosummary('Progress/lod', sched.lod),
                tfutil.autosummary('Progress/minibatch', sched.minibatch),
                misc.format_time(tfutil.autosummary('Timing/total_sec', total_time)),
                tfutil.autosummary('Timing/sec_per_tick', tick_time),
                tfutil.autosummary('Timing/sec_per_kimg', tick_time / tick_kimg),
                tfutil.autosummary('Timing/maintenance_sec', maintenance_time)))
            tfutil.autosummary('Timing/total_hours', total_time / (60.0 * 60.0))
            tfutil.autosummary('Timing/total_days', total_time / (24.0 * 60.0 * 60.0))
            tfutil.save_summaries(summary_log, cur_nimg)

            # Save snapshots.
            if cur_tick % image_snapshot_ticks == 0 or done:
                grid_fakes = Gs.run(grid_latents, grid_labels, minibatch_size=sched.minibatch//config.num_gpus)
                misc.save_image_grid(grid_fakes, os.path.join(result_subdir, 'fakes%06d.png' % (cur_nimg // 1000)), drange=drange_net, grid_size=grid_size)
            if cur_tick % network_snapshot_ticks == 0 or done:
                misc.save_pkl((G, D, Gs), os.path.join(result_subdir, 'network-snapshot-%06d.pkl' % (cur_nimg // 1000)))

            # Record start time of the next tick.
            tick_start_time = time.time()

    # Write final results.
    misc.save_pkl((G, D, Gs), os.path.join(result_subdir, 'network-final.pkl'))
    summary_log.close()
    open(os.path.join(result_subdir, '_training-done.txt'), 'wt').close()
Exemplo n.º 20
0
    def run(self):
        self.parent.clear_plots()
        if self.data is None:
            self.parent.threadPlot = None
            return

        tMin = format_time(self.extent.tMin, True)
        tMax = format_time(self.extent.tMax, True)
        fMin = format_precision(self.settings, freq=self.extent.fMin,
                                fancyUnits=True)
        fMax = format_precision(self.settings, freq=self.extent.fMax,
                                fancyUnits=True)
        lMin = format_precision(self.settings, level=self.extent.lMin,
                                fancyUnits=True)
        lMax = format_precision(self.settings, level=self.extent.lMax,
                                fancyUnits=True)
        peak = self.extent.get_peak_flt()
        peakF = format_precision(self.settings, freq=peak[0],
                                 fancyUnits=True)
        peakL = format_precision(self.settings, level=peak[1],
                                 fancyUnits=True)
        peakT = format_time(peak[2], True)

        text = [['Sweeps', '', len(self.data)],
                ['Extents', '', ''],
                ['', 'Start', tMin],
                ['', 'End', tMax],
                ['', 'Min frequency', fMin],
                ['', 'Max frequency', fMax],
                ['', 'Min level', lMin],
                ['', 'Max level', lMax],
                ['Peak', '', ''],
                ['', 'Level', peakL],
                ['', 'Frequency', peakF],
                ['', 'Time', peakT],
                ]

        table = Table(self.axes, loc='center', gid='table')

        rows = len(text)
        cols = len(text[0])
        for row in xrange(rows):
            for col in xrange(cols):
                table.add_cell(row, col,
                               text=text[row][col],
                               width=1.0 / cols, height=1.0 / rows)

        if self.settings.grid:
            colour = 'LightGray'
        else:
            colour = 'w'
        set_table_colour(table, colour)

        for i in range(3):
            table.auto_set_column_width(i)

        self.axes.add_table(table)
        noData = find_artists(self.axes, 'noData')
        noData[0].set_alpha(0)
        self.parent.redraw_plot()

        self.parent.threadPlot = None
Exemplo n.º 21
0
def train_progressive_gan(
    G_smoothing=0.999,  # Exponential running average of generator weights.
    D_repeats=1,  # How many times the discriminator is trained per G iteration.
    minibatch_repeats=4,  # Number of minibatches to run before adjusting training parameters.
    reset_opt_for_new_lod=True,  # Reset optimizer internal state (e.g. Adam moments) when new layers are introduced?
    total_kimg=15000,  # Total length of the training, measured in thousands of real images.
    mirror_augment=False,  # Enable mirror augment?
    drange_net=[
        -1, 1
    ],  # Dynamic range used when feeding image data to the networks.
    image_snapshot_ticks=1,  # How often to export image snapshots?
    network_snapshot_ticks=10,  # How often to export network snapshots?
    compute_fid_score=False,  # Compute FID during training once sched.lod=0.0 
    minimum_fid_kimg=0,  # Compute FID after 
    fid_snapshot_ticks=1,  # How often to compute FID
    fid_patience=2,  # When to end training based on FID
    save_tf_graph=False,  # Include full TensorFlow computation graph in the tfevents file?
    save_weight_histograms=False,  # Include weight histograms in the tfevents file?
    resume_run_id=None,  # Run ID or network pkl to resume training from, None = start from scratch.
    resume_snapshot=None,  # Snapshot index to resume training from, None = autodetect.
    resume_kimg=0.0,  # Assumed training progress at the beginning. Affects reporting and training schedule.
    resume_time=0.0,  # Assumed wallclock time at the beginning. Affects reporting.
    result_subdir="./"):
    maintenance_start_time = time.time()
    training_set = dataset.load_dataset(data_dir=config.data_dir,
                                        verbose=True,
                                        **config.dataset)

    # Construct networks.
    with tf.device('/gpu:0'):
        if resume_run_id != "None":
            network_pkl = misc.locate_network_pkl(resume_run_id,
                                                  resume_snapshot)
            resume_pkl_name = os.path.splitext(
                os.path.basename(network_pkl))[0]
            try:
                resume_kimg = int(resume_pkl_name.split('-')[-1])
                print('** Setting resume kimg to', resume_kimg, flush=True)
            except:
                print('** Keeping resume kimg as:', resume_kimg, flush=True)
            print('Loading networks from "%s"...' % network_pkl, flush=True)
            G, D, Gs = misc.load_pkl(network_pkl)
        else:
            print('Constructing networks...', flush=True)
            G = tfutil.Network('G',
                               num_channels=training_set.shape[0],
                               resolution=training_set.shape[1],
                               label_size=training_set.label_size,
                               **config.G)
            D = tfutil.Network('D',
                               num_channels=training_set.shape[0],
                               resolution=training_set.shape[1],
                               label_size=training_set.label_size,
                               **config.D)
            Gs = G.clone('Gs')
        Gs_update_op = Gs.setup_as_moving_average_of(G, beta=G_smoothing)
    G.print_layers()
    D.print_layers()

    print('Building TensorFlow graph...', flush=True)
    with tf.name_scope('Inputs'):
        lod_in = tf.placeholder(tf.float32, name='lod_in', shape=[])
        lrate_in = tf.placeholder(tf.float32, name='lrate_in', shape=[])
        minibatch_in = tf.placeholder(tf.int32, name='minibatch_in', shape=[])
        minibatch_split = minibatch_in // config.num_gpus
        reals, labels = training_set.get_minibatch_tf()
        reals_split = tf.split(reals, config.num_gpus)
        labels_split = tf.split(labels, config.num_gpus)
    G_opt = tfutil.Optimizer(name='TrainG',
                             learning_rate=lrate_in,
                             **config.G_opt)
    D_opt = tfutil.Optimizer(name='TrainD',
                             learning_rate=lrate_in,
                             **config.D_opt)

    for gpu in range(config.num_gpus):
        with tf.name_scope('GPU%d' % gpu), tf.device('/gpu:%d' % gpu):
            G_gpu = G if gpu == 0 else G.clone(G.name + '_shadow')
            D_gpu = D if gpu == 0 else D.clone(D.name + '_shadow')
            reals_gpu = process_reals(reals_split[gpu], lod_in, mirror_augment,
                                      training_set.dynamic_range, drange_net)
            labels_gpu = labels_split[gpu]
            with tf.name_scope('G_loss'):
                G_loss = tfutil.call_func_by_name(
                    G=G_gpu,
                    D=D_gpu,
                    opt=G_opt,
                    training_set=training_set,
                    minibatch_size=minibatch_split,
                    **config.G_loss)
            with tf.name_scope('D_loss'):
                D_loss = tfutil.call_func_by_name(
                    G=G_gpu,
                    D=D_gpu,
                    opt=D_opt,
                    training_set=training_set,
                    minibatch_size=minibatch_split,
                    reals=reals_gpu,
                    labels=labels_gpu,
                    **config.D_loss)

            G_opt.register_gradients(tf.reduce_mean(G_loss), G_gpu.trainables)
            D_opt.register_gradients(tf.reduce_mean(D_loss), D_gpu.trainables)

    G_train_op = G_opt.apply_updates()
    D_train_op = D_opt.apply_updates()

    print('Setting up snapshot image grid...', flush=True)
    grid_size, grid_reals, grid_labels, grid_latents = setup_snapshot_image_grid(
        G, training_set, **config.grid)
    sched = TrainingSchedule(total_kimg * 1000, training_set, **config.sched)
    grid_fakes = Gs.run(grid_latents,
                        grid_labels,
                        minibatch_size=sched.minibatch // config.num_gpus)

    print('Setting up result dir...', flush=True)
    result_subdir = misc.create_result_subdir(config.result_dir, config.desc)
    misc.save_image_grid(grid_reals,
                         os.path.join(result_subdir, 'reals.png'),
                         drange=training_set.dynamic_range,
                         grid_size=grid_size)
    misc.save_image_grid(grid_fakes,
                         os.path.join(result_subdir, 'fakes%06d.png' % 0),
                         drange=drange_net,
                         grid_size=grid_size)
    summary_log = tf.summary.FileWriter(result_subdir)
    if save_tf_graph:
        summary_log.add_graph(tf.get_default_graph())
    if save_weight_histograms:
        G.setup_weight_histograms()
        D.setup_weight_histograms()

    print('Training...', flush=True)
    # FID patience parameters:
    fid_list = []
    fid_steps = 0
    fid_stop = False
    fid_patience_step = 0

    cur_nimg = int(resume_kimg * 1000)
    cur_tick = 0
    tick_start_nimg = cur_nimg
    tick_start_time = time.time()
    train_start_time = tick_start_time - resume_time
    prev_lod = -1.0
    while cur_nimg < total_kimg * 1000:

        # Choose training parameters and configure training ops.
        sched = TrainingSchedule(cur_nimg, training_set, **config.sched)
        training_set.configure(sched.minibatch, sched.lod)
        if reset_opt_for_new_lod:
            if np.floor(sched.lod) != np.floor(prev_lod) or np.ceil(
                    sched.lod) != np.ceil(prev_lod):
                G_opt.reset_optimizer_state()
                D_opt.reset_optimizer_state()
        prev_lod = sched.lod

        # Run training ops.
        for repeat in range(minibatch_repeats):
            for _ in range(D_repeats):
                tfutil.run(
                    [D_train_op, Gs_update_op], {
                        lod_in: sched.lod,
                        lrate_in: sched.D_lrate,
                        minibatch_in: sched.minibatch
                    })
                cur_nimg += sched.minibatch
            tfutil.run(
                [G_train_op], {
                    lod_in: sched.lod,
                    lrate_in: sched.G_lrate,
                    minibatch_in: sched.minibatch
                })

        # Perform maintenance tasks once per tick.
        done = (cur_nimg >= total_kimg * 1000)
        if cur_nimg >= tick_start_nimg + sched.tick_kimg * 1000 or done:
            cur_tick += 1
            cur_time = time.time()
            tick_kimg = (cur_nimg - tick_start_nimg) / 1000.0
            tick_start_nimg = cur_nimg
            tick_time = cur_time - tick_start_time
            total_time = cur_time - train_start_time
            maintenance_time = tick_start_time - maintenance_start_time
            maintenance_start_time = cur_time

            if (compute_fid_score
                    == True) and (cur_tick % fid_snapshot_ticks
                                  == 0) and (sched.lod == 0.0) and (
                                      cur_nimg >= minimum_fid_kimg * 1000):
                fid = compute_fid(Gs=Gs,
                                  minibatch_size=sched.minibatch,
                                  dataset_obj=training_set,
                                  iter_number=cur_nimg / 1000,
                                  lod=0.0,
                                  num_images=10000,
                                  printing=False)
                fid_list.append(fid)

            # Report progress without FID.
            print(
                'tick %-5d kimg %-8.1f lod %-5.2f minibatch %-4d time %-12s sec/tick %-7.1f sec/kimg %-7.2f maintenance %.1f'
                % (tfutil.autosummary('Progress/tick', cur_tick),
                   tfutil.autosummary('Progress/kimg', cur_nimg / 1000.0),
                   tfutil.autosummary('Progress/lod', sched.lod),
                   tfutil.autosummary('Progress/minibatch', sched.minibatch),
                   misc.format_time(
                       tfutil.autosummary('Timing/total_sec', total_time)),
                   tfutil.autosummary('Timing/sec_per_tick', tick_time),
                   tfutil.autosummary('Timing/sec_per_kimg',
                                      tick_time / tick_kimg),
                   tfutil.autosummary('Timing/maintenance_sec',
                                      maintenance_time)),
                flush=True)
            tfutil.autosummary('Timing/total_hours',
                               total_time / (60.0 * 60.0))
            tfutil.autosummary('Timing/total_days',
                               total_time / (24.0 * 60.0 * 60.0))
            tfutil.save_summaries(summary_log, cur_nimg)

            # Save image snapshots.
            if cur_tick % image_snapshot_ticks == 0 or done:
                grid_fakes = Gs.run(grid_latents,
                                    grid_labels,
                                    minibatch_size=sched.minibatch //
                                    config.num_gpus)
                misc.save_image_grid(grid_fakes,
                                     os.path.join(
                                         result_subdir,
                                         'fakes%06d.png' % (cur_nimg // 1000)),
                                     drange=drange_net,
                                     grid_size=grid_size)

            # Save network snapshots
            if cur_tick % network_snapshot_ticks == 0 or done or (
                    compute_fid_score
                    == True) and (cur_tick % fid_snapshot_ticks == 0) and (
                        cur_nimg >= minimum_fid_kimg * 1000):
                misc.save_pkl(
                    (G, D, Gs),
                    os.path.join(
                        result_subdir,
                        'network-snapshot-%06d.pkl' % (cur_nimg // 1000)))

            # End training based on FID patience
            if (compute_fid_score
                    == True) and (cur_tick % fid_snapshot_ticks
                                  == 0) and (sched.lod == 0.0) and (
                                      cur_nimg >= minimum_fid_kimg * 1000):
                fid_patience_step += 1
                if len(fid_list) == 1:
                    fid_patience_step = 0
                    misc.save_pkl((G, D, Gs),
                                  os.path.join(result_subdir,
                                               'network-final-full-conv.pkl'))
                    print(
                        "Save network-final-full-conv for FID: %.3f at kimg %-8.1f."
                        % (fid_list[-1], cur_nimg // 1000),
                        flush=True)
                else:
                    if fid_list[-1] < np.min(fid_list[:-1]):
                        fid_patience_step = 0
                        misc.save_pkl(
                            (G, D, Gs),
                            os.path.join(result_subdir,
                                         'network-final-full-conv.pkl'))
                        print(
                            "Save network-final-full-conv for FID: %.3f at kimg %-8.1f."
                            % (fid_list[-1], cur_nimg // 1000),
                            flush=True)
                    else:
                        print("No improvement for FID: %.3f at kimg %-8.1f." %
                              (fid_list[-1], cur_nimg // 1000),
                              flush=True)
                if fid_patience_step == fid_patience:
                    fid_stop = True
                    print("Training stopped due to FID early-stopping.",
                          flush=True)
                    cur_nimg = total_kimg * 1000

            # Record start time of the next tick.
            tick_start_time = time.time()

    # Write final results.
    # Save final only if FID-Stopping has not happend:
    if fid_stop == False:
        fid = compute_fid(Gs=Gs,
                          minibatch_size=sched.minibatch,
                          dataset_obj=training_set,
                          iter_number=cur_nimg / 1000,
                          lod=0.0,
                          num_images=10000,
                          printing=False)
        print("Final FID: %.3f at kimg %-8.1f." % (fid, cur_nimg // 1000),
              flush=True)
        ### save final FID to .csv file in result_parent_dir
        csv_file = os.path.join(
            os.path.dirname(os.path.dirname(result_subdir)),
            "results_full_conv.csv")
        list_to_append = [
            result_subdir.split("/")[-2] + "/" + result_subdir.split("/")[-1],
            fid
        ]
        with open(csv_file, 'a') as f_object:
            writer_object = writer(f_object)
            writer_object.writerow(list_to_append)
            f_object.close()
        misc.save_pkl((G, D, Gs),
                      os.path.join(result_subdir,
                                   'network-final-full-conv.pkl'))
        print("Save network-final-full-conv.", flush=True)
    summary_log.close()
    open(os.path.join(result_subdir, '_training-done.txt'), 'wt').close()
Exemplo n.º 22
0
def train_classifier(
    smoothing=0.999,  # Exponential running average of encoder weights.
    minibatch_repeats=4,  # Number of minibatches to run before adjusting training parameters.
    reset_opt_for_new_lod=True,  # Reset optimizer internal state (e.g. Adam moments) when new layers are introduced?
    total_kimg=25000,  # Total length of the training, measured in thousands of real images.
    lr_mirror_augment=True,  # Enable mirror augment?
    ud_mirror_augment=False,  # Enable up-down mirror augment?
    drange_net=[
        -1, 1
    ],  # Dynamic range used when feeding image data to the networks.
    image_snapshot_ticks=10,  # How often to export image snapshots?
    save_tf_graph=False,  # Include full TensorFlow computation graph in the tfevents file?
    save_weight_histograms=False
):  # Include weight histograms in the tfevents file?

    maintenance_start_time = time.time()
    training_set = dataset.load_dataset(data_dir=config.data_dir,
                                        verbose=True,
                                        **config.training_set)
    validation_set = dataset.load_dataset(data_dir=config.data_dir,
                                          verbose=True,
                                          **config.validation_set)
    network_snapshot_ticks = total_kimg // 100  # How often to export network snapshots?

    # Construct networks.
    with tf.device('/gpu:0'):
        try:
            network_pkl = misc.locate_network_pkl()
            resume_kimg, resume_time = misc.resume_kimg_time(network_pkl)
            print('Loading networks from "%s"...' % network_pkl)
            EG, D_rec, EGs = misc.load_pkl(network_pkl)
        except:
            print('Constructing networks...')
            resume_kimg = 0.0
            resume_time = 0.0
            EG = tfutil.Network('EG',
                                num_channels=training_set.shape[0],
                                resolution=training_set.shape[1],
                                label_size=training_set.label_size,
                                **config.EG)
            D_rec = tfutil.Network('D_rec',
                                   num_channels=training_set.shape[0],
                                   resolution=training_set.shape[1],
                                   **config.D_rec)
            EGs = EG.clone('EGs')
        EGs_update_op = EGs.setup_as_moving_average_of(EG, beta=smoothing)
    EG.print_layers()
    D_rec.print_layers()

    print('Building TensorFlow graph...')
    with tf.name_scope('Inputs'):
        lod_in = tf.placeholder(tf.float32, name='lod_in', shape=[])
        lrate_in = tf.placeholder(tf.float32, name='lrate_in', shape=[])
        minibatch_in = tf.placeholder(tf.int32, name='minibatch_in', shape=[])
        minibatch_split = minibatch_in // config.num_gpus
        reals, labels = training_set.get_minibatch_tf()
        reals_split = tf.split(reals, config.num_gpus)
        labels_split = tf.split(labels, config.num_gpus)
    EG_opt = tfutil.Optimizer(name='TrainEG',
                              learning_rate=lrate_in,
                              **config.EG_opt)
    D_rec_opt = tfutil.Optimizer(name='TrainD_rec',
                                 learning_rate=lrate_in,
                                 **config.D_rec_opt)
    for gpu in range(config.num_gpus):
        with tf.name_scope('GPU%d' % gpu), tf.device('/gpu:%d' % gpu):
            EG_gpu = EG if gpu == 0 else EG.clone(EG.name + '_shadow_%d' % gpu)
            D_rec_gpu = D_rec if gpu == 0 else D_rec.clone(D_rec.name +
                                                           '_shadow_%d' % gpu)
            reals_fade_gpu, reals_orig_gpu = process_reals(
                reals_split[gpu], lod_in, lr_mirror_augment, ud_mirror_augment,
                training_set.dynamic_range, drange_net)
            labels_gpu = labels_split[gpu]
            with tf.name_scope('EG_loss'):
                EG_loss = tfutil.call_func_by_name(EG=EG_gpu,
                                                   D_rec=D_rec_gpu,
                                                   reals_orig=reals_orig_gpu,
                                                   labels=labels_gpu,
                                                   **config.EG_loss)
            with tf.name_scope('D_rec_loss'):
                D_rec_loss = tfutil.call_func_by_name(
                    EG=EG_gpu,
                    D_rec=D_rec_gpu,
                    D_rec_opt=D_rec_opt,
                    minibatch_size=minibatch_split,
                    reals_orig=reals_orig_gpu,
                    **config.D_rec_loss)
            EG_opt.register_gradients(tf.reduce_mean(EG_loss),
                                      EG_gpu.trainables)
            D_rec_opt.register_gradients(tf.reduce_mean(D_rec_loss),
                                         D_rec_gpu.trainables)
    EG_train_op = EG_opt.apply_updates()
    D_rec_train_op = D_rec_opt.apply_updates()

    print('Setting up snapshot image grid...')
    grid_size, train_reals, train_labels = setup_snapshot_image_grid(
        training_set, drange_net, [450, 10], **config.grid)
    grid_size, val_reals, val_labels = setup_snapshot_image_grid(
        validation_set, drange_net, [450, 10], **config.grid)
    sched = TrainingSchedule(total_kimg * 1000, training_set, **config.sched)

    train_recs, train_fingerprints, train_logits = EGs.run(
        train_reals, minibatch_size=sched.minibatch // config.num_gpus)
    train_preds = np.argmax(train_logits, axis=1)
    train_gt = np.argmax(train_labels, axis=1)
    train_acc = np.float32(np.sum(train_gt == train_preds)) / np.float32(
        len(train_gt))
    print('Training Accuracy = %f' % train_acc)

    val_recs, val_fingerprints, val_logits = EGs.run(
        val_reals, minibatch_size=sched.minibatch // config.num_gpus)
    val_preds = np.argmax(val_logits, axis=1)
    val_gt = np.argmax(val_labels, axis=1)
    val_acc = np.float32(np.sum(val_gt == val_preds)) / np.float32(len(val_gt))
    print('Validation Accuracy = %f' % val_acc)

    print('Setting up result dir...')
    result_subdir = misc.create_result_subdir(config.result_dir, config.desc)
    misc.save_image_grid(train_reals[::30, :, :, :],
                         os.path.join(result_subdir, 'train_reals.png'),
                         drange=drange_net,
                         grid_size=[15, 10])
    misc.save_image_grid(train_recs[::30, :, :, :],
                         os.path.join(result_subdir, 'train_recs-init.png'),
                         drange=drange_net,
                         grid_size=[15, 10])
    misc.save_image_grid(train_fingerprints[::30, :, :, :],
                         os.path.join(result_subdir,
                                      'train_fingerrints-init.png'),
                         drange=drange_net,
                         grid_size=[15, 10])
    misc.save_image_grid(val_reals[::30, :, :, :],
                         os.path.join(result_subdir, 'val_reals.png'),
                         drange=drange_net,
                         grid_size=[15, 10])
    misc.save_image_grid(val_recs[::30, :, :, :],
                         os.path.join(result_subdir, 'val_recs-init.png'),
                         drange=drange_net,
                         grid_size=[15, 10])
    misc.save_image_grid(val_fingerprints[::30, :, :, :],
                         os.path.join(result_subdir,
                                      'val_fingerrints-init.png'),
                         drange=drange_net,
                         grid_size=[15, 10])

    est_fingerprints = np.transpose(
        EGs.vars['Conv_fingerprints/weight'].eval(), axes=[3, 2, 0, 1])
    misc.save_image_grid(
        est_fingerprints,
        os.path.join(result_subdir, 'est_fingerrints-init.png'),
        drange=[np.amin(est_fingerprints),
                np.amax(est_fingerprints)],
        grid_size=[est_fingerprints.shape[0], 1])

    summary_log = tf.summary.FileWriter(result_subdir)
    if save_tf_graph:
        summary_log.add_graph(tf.get_default_graph())
    if save_weight_histograms:
        EG.setup_weight_histograms()
        D_rec.setup_weight_histograms()

    print('Training...')
    cur_nimg = int(resume_kimg * 1000)
    cur_tick = 0
    tick_start_nimg = cur_nimg
    tick_start_time = time.time()
    train_start_time = tick_start_time - resume_time
    prev_lod = -1.0
    while cur_nimg < total_kimg * 1000:

        # Choose training parameters and configure training ops.
        sched = TrainingSchedule(cur_nimg, training_set, **config.sched)
        training_set.configure(sched.minibatch, sched.lod)
        if reset_opt_for_new_lod:
            if np.floor(sched.lod) != np.floor(prev_lod) or np.ceil(
                    sched.lod) != np.ceil(prev_lod):
                EG_opt.reset_optimizer_state()
                D_rec_opt.reset_optimizer_state()
        prev_lod = sched.lod

        # Run training ops.
        for repeat in range(minibatch_repeats):
            tfutil.run(
                [D_rec_train_op], {
                    lod_in: sched.lod,
                    lrate_in: sched.lrate,
                    minibatch_in: sched.minibatch
                })
            tfutil.run(
                [EG_train_op], {
                    lod_in: sched.lod,
                    lrate_in: sched.lrate,
                    minibatch_in: sched.minibatch
                })
            tfutil.run([EGs_update_op], {})
            cur_nimg += sched.minibatch

        # Perform maintenance tasks once per tick.
        done = (cur_nimg >= total_kimg * 1000)
        if cur_nimg >= tick_start_nimg + sched.tick_kimg * 1000 or done:
            cur_tick += 1
            cur_time = time.time()
            tick_kimg = (cur_nimg - tick_start_nimg) / 1000.0
            tick_start_nimg = cur_nimg
            tick_time = cur_time - tick_start_time
            total_time = cur_time - train_start_time
            maintenance_time = tick_start_time - maintenance_start_time
            maintenance_start_time = cur_time

            # Report progress.
            print(
                'tick %-5d kimg %-8.1f lod %-5.2f resolution %-4d minibatch %-4d time %-12s sec/tick %-7.1f sec/kimg %-7.2f maintenance %.1f'
                % (tfutil.autosummary('Progress/tick', cur_tick),
                   tfutil.autosummary('Progress/kimg', cur_nimg / 1000.0),
                   tfutil.autosummary('Progress/lod', sched.lod),
                   tfutil.autosummary('Progress/resolution', sched.resolution),
                   tfutil.autosummary('Progress/minibatch', sched.minibatch),
                   misc.format_time(
                       tfutil.autosummary('Timing/total_sec', total_time)),
                   tfutil.autosummary('Timing/sec_per_tick', tick_time),
                   tfutil.autosummary('Timing/sec_per_kimg',
                                      tick_time / tick_kimg),
                   tfutil.autosummary('Timing/maintenance_sec',
                                      maintenance_time)))
            tfutil.autosummary('Timing/total_hours',
                               total_time / (60.0 * 60.0))
            tfutil.autosummary('Timing/total_days',
                               total_time / (24.0 * 60.0 * 60.0))
            tfutil.save_summaries(summary_log, cur_nimg)

            # Print accuracy.
            if cur_tick % image_snapshot_ticks == 0 or done:

                train_recs, train_fingerprints, train_logits = EGs.run(
                    train_reals,
                    minibatch_size=sched.minibatch // config.num_gpus)
                train_preds = np.argmax(train_logits, axis=1)
                train_gt = np.argmax(train_labels, axis=1)
                train_acc = np.float32(np.sum(
                    train_gt == train_preds)) / np.float32(len(train_gt))
                print('Training Accuracy = %f' % train_acc)

                val_recs, val_fingerprints, val_logits = EGs.run(
                    val_reals,
                    minibatch_size=sched.minibatch // config.num_gpus)
                val_preds = np.argmax(val_logits, axis=1)
                val_gt = np.argmax(val_labels, axis=1)
                val_acc = np.float32(np.sum(val_gt == val_preds)) / np.float32(
                    len(val_gt))
                print('Validation Accuracy = %f' % val_acc)

                misc.save_image_grid(train_recs[::30, :, :, :],
                                     os.path.join(result_subdir,
                                                  'train_recs-final.png'),
                                     drange=drange_net,
                                     grid_size=[15, 10])
                misc.save_image_grid(train_fingerprints[::30, :, :, :],
                                     os.path.join(
                                         result_subdir,
                                         'train_fingerrints-final.png'),
                                     drange=drange_net,
                                     grid_size=[15, 10])
                misc.save_image_grid(val_recs[::30, :, :, :],
                                     os.path.join(result_subdir,
                                                  'val_recs-final.png'),
                                     drange=drange_net,
                                     grid_size=[15, 10])
                misc.save_image_grid(val_fingerprints[::30, :, :, :],
                                     os.path.join(result_subdir,
                                                  'val_fingerrints-final.png'),
                                     drange=drange_net,
                                     grid_size=[15, 10])

                est_fingerprints = np.transpose(
                    EGs.vars['Conv_fingerprints/weight'].eval(),
                    axes=[3, 2, 0, 1])
                misc.save_image_grid(est_fingerprints,
                                     os.path.join(result_subdir,
                                                  'est_fingerrints-final.png'),
                                     drange=[
                                         np.amin(est_fingerprints),
                                         np.amax(est_fingerprints)
                                     ],
                                     grid_size=[est_fingerprints.shape[0], 1])

            if cur_tick % network_snapshot_ticks == 0 or done:
                misc.save_pkl(
                    (EG, D_rec, EGs),
                    os.path.join(
                        result_subdir,
                        'network-snapshot-%06d.pkl' % (cur_nimg // 1000)))

            # Record start time of the next tick.
            tick_start_time = time.time()

    # Write final results.
    misc.save_pkl((EG, D_rec, EGs),
                  os.path.join(result_subdir, 'network-final.pkl'))
    summary_log.close()
    open(os.path.join(result_subdir, '_training-done.txt'), 'wt').close()
Exemplo n.º 23
0
def train_progressive_gan(
    G_smoothing=0.999,  # Exponential running average of generator weights.
    D_repeats=1,  # How many times the discriminator is trained per G iteration.
    minibatch_repeats=4,  # Number of minibatches to run before adjusting training parameters.
    reset_opt_for_new_lod=True,  # Reset optimizer internal state (e.g. Adam moments) when new layers are introduced?
    total_kimg=15000,  # Total length of the training, measured in thousands of real images.
    mirror_augment=False,  # Enable mirror augment?
    drange_net=[
        -1, 1
    ],  # Dynamic range used when feeding image data to the networks.
    image_snapshot_ticks=1,  # How often to export image snapshots?
    network_snapshot_ticks=10,  # How often to export network snapshots?
    save_tf_graph=False,  # Include full TensorFlow computation graph in the tfevents file?
    save_weight_histograms=False,  # Include weight histograms in the tfevents file?
    resume_run_id=None,  # Run ID or network pkl to resume training from, None = start from scratch.
    resume_snapshot=None,  # Snapshot index to resume training from, None = autodetect.
    resume_kimg=0.0,  # Assumed training progress at the beginning. Affects reporting and training schedule.
    resume_time=0.0
):  # Assumed wallclock time at the beginning. Affects reporting.

    maintenance_start_time = time.time()
    training_set = dataset.load_dataset(data_dir=config.data_dir,
                                        verbose=True,
                                        **config.dataset)
    unlabeled_training_set = dataset.load_dataset(
        data_dir=config.unlabeled_data_dir,
        verbose=True,
        **config.unlabeled_dataset)

    # Construct networks.
    with tf.device('/gpu:0'):
        if resume_run_id is not None:
            network_pkl = misc.locate_network_pkl(resume_run_id,
                                                  resume_snapshot)
            print('Loading networks from "%s"...' % network_pkl)
            G, D, Gs = misc.load_pkl(network_pkl)
        else:
            print('Constructing networks...')
            print("Training-Set Label Size: ", training_set.label_size)
            print("Unlabeled-Training-Set Label Size: ",
                  unlabeled_training_set.label_size)
            G = tfutil.Network('G',
                               num_channels=training_set.shape[0],
                               resolution=training_set.shape[1],
                               label_size=training_set.label_size,
                               **config.G)
            D = tfutil.Network('D',
                               num_channels=training_set.shape[0],
                               resolution=training_set.shape[1],
                               label_size=training_set.label_size,
                               **config.D)
            Gs = G.clone('Gs')
        Gs_update_op = Gs.setup_as_moving_average_of(G, beta=G_smoothing)
    G.print_layers()
    D.print_layers()

    print('Building TensorFlow graph...')
    with tf.name_scope('Inputs'):
        lod_in = tf.placeholder(tf.float32, name='lod_in', shape=[])
        lrate_in = tf.placeholder(tf.float32, name='lrate_in', shape=[])
        minibatch_in = tf.placeholder(tf.int32, name='minibatch_in', shape=[])

        minibatch_split = minibatch_in // config.num_gpus
        reals, labels = training_set.get_minibatch_tf()
        unlabeled_reals, _ = unlabeled_training_set.get_minibatch_tf()

        reals_split = tf.split(reals, config.num_gpus)
        unlabeled_reals_split = tf.split(unlabeled_reals, config.num_gpus)

        labels_split = tf.split(labels, config.num_gpus)

    G_opt = tfutil.Optimizer(name='TrainG',
                             learning_rate=lrate_in,
                             **config.G_opt)
    G_opt_pggan = tfutil.Optimizer(name='TrainG_pggan',
                                   learning_rate=lrate_in,
                                   **config.G_opt)
    D_opt_pggan = tfutil.Optimizer(name='TrainD_pggan',
                                   learning_rate=lrate_in,
                                   **config.D_opt)
    D_opt = tfutil.Optimizer(name='TrainD',
                             learning_rate=lrate_in,
                             **config.D_opt)

    print("CUDA_VISIBLE_DEVICES: ", os.environ['CUDA_VISIBLE_DEVICES'])

    for gpu in range(config.num_gpus):
        with tf.name_scope('GPU%d' % gpu), tf.device('/gpu:%d' % gpu):
            G_gpu = G if gpu == 0 else G.clone(G.name + '_shadow')
            D_gpu = D if gpu == 0 else D.clone(D.name + '_shadow')

            lod_assign_ops = [
                tf.assign(G_gpu.find_var('lod'), lod_in),
                tf.assign(D_gpu.find_var('lod'), lod_in)
            ]

            reals_gpu = process_reals(reals_split[gpu], lod_in, mirror_augment,
                                      training_set.dynamic_range, drange_net)
            unlabeled_reals_gpu = process_reals(
                unlabeled_reals_split[gpu], lod_in, mirror_augment,
                unlabeled_training_set.dynamic_range, drange_net)

            labels_gpu = labels_split[gpu]
            with tf.name_scope('G_loss'), tf.control_dependencies(
                    lod_assign_ops):
                G_loss = tfutil.call_func_by_name(
                    G=G_gpu,
                    D=D_gpu,
                    opt=G_opt,
                    training_set=training_set,
                    minibatch_size=minibatch_split,
                    unlabeled_reals=unlabeled_reals_gpu,
                    **config.G_loss)
            with tf.name_scope('G_loss_pggan'), tf.control_dependencies(
                    lod_assign_ops):
                G_loss_pggan = tfutil.call_func_by_name(
                    G=G_gpu,
                    D=D_gpu,
                    opt=G_opt,
                    training_set=training_set,
                    minibatch_size=minibatch_split,
                    **config.G_loss_pggan)

            with tf.name_scope('D_loss'), tf.control_dependencies(
                    lod_assign_ops):
                D_loss = tfutil.call_func_by_name(
                    G=G_gpu,
                    D=D_gpu,
                    opt=D_opt,
                    training_set=training_set,
                    minibatch_size=minibatch_split,
                    reals=reals_gpu,
                    labels=labels_gpu,
                    unlabeled_reals=unlabeled_reals_gpu,
                    **config.D_loss)
            with tf.name_scope('D_loss_pggan'), tf.control_dependencies(
                    lod_assign_ops):
                D_loss_pggan = tfutil.call_func_by_name(
                    G=G_gpu,
                    D=D_gpu,
                    opt=D_opt,
                    training_set=training_set,
                    minibatch_size=minibatch_split,
                    unlabeled_reals=unlabeled_reals_gpu,
                    **config.D_loss_pggan)
            G_opt.register_gradients(tf.reduce_mean(G_loss), G_gpu.trainables)
            G_opt_pggan.register_gradients(tf.reduce_mean(G_loss_pggan),
                                           G_gpu.trainables)
            D_opt_pggan.register_gradients(tf.reduce_mean(D_loss_pggan),
                                           D_gpu.trainables)
            D_opt.register_gradients(tf.reduce_mean(D_loss), D_gpu.trainables)
            print('GPU %d loaded!' % gpu)

    G_train_op = G_opt.apply_updates()
    G_train_op_pggan = G_opt_pggan.apply_updates()
    D_train_op_pggan = D_opt_pggan.apply_updates()
    D_train_op = D_opt.apply_updates()

    print('Setting up snapshot image grid...')
    grid_size, grid_reals, grid_labels, grid_latents = setup_snapshot_image_grid(
        G, training_set, **config.grid)
    sched = TrainingSchedule(total_kimg * TrainingSpeedInt, training_set,
                             **config.sched)
    grid_fakes = Gs.run(grid_latents,
                        grid_labels,
                        minibatch_size=sched.minibatch // config.num_gpus)

    print('Setting up result dir...')
    result_subdir = misc.create_result_subdir(config.result_dir, config.desc)
    misc.save_image_grid(grid_reals,
                         os.path.join(result_subdir, 'reals.png'),
                         drange=training_set.dynamic_range,
                         grid_size=grid_size)
    misc.save_image_grid(grid_fakes,
                         os.path.join(result_subdir, 'fakes%06d.png' % 0),
                         drange=drange_net,
                         grid_size=grid_size)
    summary_log = tf.compat.v1.summary.FileWriter(result_subdir)
    if save_tf_graph:
        summary_log.add_graph(tf.get_default_graph())
    if save_weight_histograms:
        G.setup_weight_histograms()
        D.setup_weight_histograms()

    print("Start Time: ",
          datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
    print('Training...')
    cur_nimg = int(resume_kimg * TrainingSpeedInt)
    cur_tick = 0
    tick_start_nimg = cur_nimg
    tick_start_time = time.time()
    train_start_time = tick_start_time - resume_time
    prev_lod = -1.0

    while cur_nimg < total_kimg * TrainingSpeedInt:

        # Choose training parameters and configure training ops.
        sched = TrainingSchedule(cur_nimg, training_set, **config.sched)
        sched2 = TrainingSchedule(cur_nimg, unlabeled_training_set,
                                  **config.sched)
        training_set.configure(sched.minibatch, sched.lod)
        unlabeled_training_set.configure(sched2.minibatch, sched2.lod)
        if reset_opt_for_new_lod:
            if np.floor(sched.lod) != np.floor(prev_lod) or np.ceil(
                    sched.lod) != np.ceil(prev_lod):
                G_opt.reset_optimizer_state()
                D_opt.reset_optimizer_state()
                G_opt_pggan.reset_optimizer_state()
                D_opt_pggan.reset_optimizer_state()
        prev_lod = sched.lod

        # Run training ops.
        for repeat in range(minibatch_repeats):
            for _ in range(D_repeats):
                # Run the Pggan loss if lod != 0 else run SSL loss with feature matching
                if sched.lod == 0:
                    tfutil.run(
                        [D_train_op, Gs_update_op], {
                            lod_in: sched.lod,
                            lrate_in: sched.D_lrate,
                            minibatch_in: sched.minibatch
                        })
                else:
                    tfutil.run(
                        [D_train_op_pggan, Gs_update_op], {
                            lod_in: sched.lod,
                            lrate_in: sched.D_lrate,
                            minibatch_in: sched.minibatch
                        })
                cur_nimg += sched.minibatch
                #tmp = min(tick_start_nimg + sched.tick_kimg * TrainingSpeedInt, total_kimg * TrainingSpeedInt)
                #print("Tick progress:  {}/{}".format(cur_nimg, tmp), end="\r", flush=True)
            # Run the Pggan loss if lod != 0 else run SSL loss with feature matching
            if sched.lod == 0:
                tfutil.run(
                    [G_train_op], {
                        lod_in: sched.lod,
                        lrate_in: sched.G_lrate,
                        minibatch_in: sched.minibatch
                    })
            else:
                tfutil.run(
                    [G_train_op_pggan], {
                        lod_in: sched.lod,
                        lrate_in: sched.G_lrate,
                        minibatch_in: sched.minibatch
                    })

        # Perform maintenance tasks once per tick.
        done = (cur_nimg >= total_kimg * TrainingSpeedInt)
        if cur_nimg >= tick_start_nimg + sched.tick_kimg * TrainingSpeedInt or done:
            cur_tick += 1
            cur_time = time.time()
            tick_kimg = (cur_nimg - tick_start_nimg) / TrainingSpeedFloat
            tick_start_nimg = cur_nimg
            tick_time = cur_time - tick_start_time
            total_time = cur_time - train_start_time
            maintenance_time = tick_start_time - maintenance_start_time
            maintenance_start_time = cur_time

            # Report progress.
            print(
                'tick %-5d kimg %-8.1f lod %-5.2f minibatch %-4d time %-12s sec/tick %-7.1f sec/kimg %-7.2f maintenance %.1f date %s'
                % (tfutil.autosummary('Progress/tick', cur_tick),
                   tfutil.autosummary('Progress/kimg',
                                      cur_nimg / TrainingSpeedFloat),
                   tfutil.autosummary('Progress/lod', sched.lod),
                   tfutil.autosummary('Progress/minibatch', sched.minibatch),
                   misc.format_time(
                       tfutil.autosummary('Timing/total_sec', total_time)),
                   tfutil.autosummary('Timing/sec_per_tick', tick_time),
                   tfutil.autosummary('Timing/sec_per_kimg',
                                      tick_time / tick_kimg),
                   tfutil.autosummary(
                       'Timing/maintenance_sec', maintenance_time),
                   datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))

            tfutil.autosummary('Timing/total_hours',
                               total_time / (60.0 * 60.0))
            tfutil.autosummary('Timing/total_days',
                               total_time / (24.0 * 60.0 * 60.0))

            #######################
            # VALIDATION ACCURACY #
            #######################

            # example ndim = 512 for an image that is 512x512 pixels
            # All images for SSL-PGGAN must be square
            ndim = 256
            correct = 0
            guesses = 0

            dir_tuple = (config.validation_dog, config.validation_cat)
            # If guessed the wrong class seeing if there is a bias
            FP_RATE = [[0], [0]]
            # For each class
            for indx, directory in enumerate(dir_tuple):
                # Go through every image that needs to be tested
                for filename in os.listdir(directory):
                    guesses += 1
                    #tensor = np.zeros((1, 3, 512, 512))
                    print(filename)
                    img = np.asarray(PIL.Image.open(directory +
                                                    filename)).reshape(
                                                        3, ndim, ndim)
                    img = np.expand_dims(
                        img, axis=0)  # makes the image (1,3,512,512)
                    K_logits_out, fake_logit_out, features_out = test_discriminator(
                        D, img)

                    #print("K Logits Out:",K_logits_out.eval())
                    sample_probs = tf.nn.softmax(K_logits_out)
                    #print("Softmax Output:", sample_probs.eval())
                    label = np.argmax(sample_probs.eval()[0], axis=0)
                    if label == indx:
                        correct += 1
                    else:
                        FP_RATE[indx][0] += 1
                    print("-----------------------------------")
                    print("GUESSED LABEL: ", label)
                    print("CORRECT LABEL: ", indx)
                    validation = (correct / guesses)
                    print("Total Correct: ", correct, "\n", "Total Guesses: ",
                          guesses, "\n", "Percent correct: ", validation)
                    print("False Positives: Dog, Cat", FP_RATE)
                    print()

            tfutil.autosummary('Accuracy/Validation', (correct / guesses))
            tfutil.save_summaries(summary_log, cur_nimg)

            # Save snapshots.
            if cur_tick % image_snapshot_ticks == 0 or done:
                grid_fakes = Gs.run(grid_latents,
                                    grid_labels,
                                    minibatch_size=sched.minibatch //
                                    config.num_gpus)
                misc.save_image_grid(grid_fakes,
                                     os.path.join(
                                         result_subdir, 'fakes%06d.png' %
                                         (cur_nimg // TrainingSpeedInt)),
                                     drange=drange_net,
                                     grid_size=grid_size)
            if cur_tick % network_snapshot_ticks == 0 or done:
                misc.save_pkl((G, D, Gs),
                              os.path.join(
                                  result_subdir, 'network-snapshot-%06d.pkl' %
                                  (cur_nimg // TrainingSpeedInt)))

            # Record start time of the next tick.
            tick_start_time = time.time()

    # Write final results.
    misc.save_pkl((G, D, Gs), os.path.join(result_subdir, 'network-final.pkl'))
    summary_log.close()
    open(os.path.join(result_subdir, '_training-done.txt'), 'wt').close()
Exemplo n.º 24
0
def calc_mnistrgb_histogram(run_id,
                            num_images=25600,
                            log='histogram.txt',
                            minibatch_size=256,
                            num_evals=10,
                            eval_reals=True,
                            final_only=False):

    # Load the classification network.
    # NOTE: The PKL can be downloaded from https://drive.google.com/open?id=0B4qLcYyJmiz0NHFULTdYc05lX0U
    net = network.load_mnist_classifier(
        os.path.join(config.data_dir,
                     '../networks/mnist_classifier_weights.pkl'))
    input_var = T.tensor4()
    output_expr = lasagne.layers.get_output(net,
                                            inputs=input_var,
                                            deterministic=True)
    classify_fn = theano.function([input_var], [output_expr])

    # Process folders
    print 'Processing directory %s' % (run_id)
    result_subdir = misc.locate_result_subdir(run_id)

    network_pkls = misc.list_network_pkls(result_subdir)
    misc.set_output_log_file(os.path.join(result_subdir, log))

    if final_only:
        network_pkls = [network_pkls[-1]]

    # Histogram calculation.
    def calc_histogram(images_all):
        scores = []
        divergences = []
        for i in range(num_evals):
            images = images_all[i * num_images:(i + 1) * num_images]
            model = [0.] * 1000
            for s in range(0, images.shape[0], minibatch_size):
                img = images[s:s + minibatch_size].reshape((-1, 1, 32, 32))
                res = np.asarray(classify_fn(img)[0])
                res = np.argmax(res, axis=1)
                res = res.reshape((-1, 3)) * np.asarray([[1, 10, 100]])
                res = np.sum(res, axis=1)
                for x in res:
                    model[int(x)] += 1.
            model = np.array([b / 25600. for b in model
                              if b > 0])  # remove empty buckets, normalize
            data = np.array([1. / 1000] *
                            len(model))  # corresponding ideal counts
            scores.append(len(model))
            divergences.append(np.sum(
                model *
                np.log(model /
                       data)))  # reverse KL? Metz et al. say KL(model || data)
        scores = np.asarray(scores, dtype=np.float32)
        return np.mean(scores), np.mean(divergences)

    # Load dataset.
    training_set, drange_orig = load_dataset_for_previous_run(result_subdir,
                                                              shuffle=False)
    reals, labels = training_set.get_random_minibatch(num_images * num_evals,
                                                      labels=True)

    # Evaluate reals.
    if eval_reals:
        print 'Evaluating histogram for reals...'
        time_begin = time.time()
        mean, kld = calc_histogram(reals)
        print 'Done in %s' % misc.format_time(time.time() - time_begin)
        print 'mean %-8.4f kld %-8.4f' % (mean, kld)

    # Evaluate each network snapshot.
    latents = None
    for network_idx, network_pkl in enumerate(network_pkls):
        print '%-32s' % os.path.basename(network_pkl),
        net = imgapi_load_net(run_id=result_subdir,
                              snapshot=network_pkl,
                              num_example_latents=num_images * num_evals)
        fakes = imgapi_generate_batch(net,
                                      net.example_latents,
                                      labels,
                                      minibatch_size=minibatch_size,
                                      convert_to_uint8=True)
        mean, kld = calc_histogram(fakes)
        print 'mean %-8.4f kld %-8.4f' % (mean, kld)
Exemplo n.º 25
0
def train_gan(separate_funcs=False,
              D_training_repeats=1,
              G_learning_rate_max=0.0010,
              D_learning_rate_max=0.0010,
              G_smoothing=0.999,
              adam_beta1=0.0,
              adam_beta2=0.99,
              adam_epsilon=1e-8,
              minibatch_default=16,
              minibatch_overrides={},
              rampup_kimg=40,
              rampdown_kimg=0,
              lod_initial_resolution=4,
              lod_training_kimg=400,
              lod_transition_kimg=400,
              total_kimg=10000,
              dequantize_reals=False,
              gdrop_beta=0.9,
              gdrop_lim=0.5,
              gdrop_coef=0.2,
              gdrop_exp=2.0,
              drange_net=[-1, 1],
              drange_viz=[-1, 1],
              image_grid_size=None,
              tick_kimg_default=50,
              tick_kimg_overrides={
                  32: 20,
                  64: 10,
                  128: 10,
                  256: 5,
                  512: 2,
                  1024: 1
              },
              image_snapshot_ticks=4,
              network_snapshot_ticks=40,
              image_grid_type='default',
              resume_network_pkl=None,
              resume_kimg=0.0,
              resume_time=0.0):

    # Load dataset and build networks.
    training_set, drange_orig = load_dataset()
    if resume_network_pkl:
        print 'Resuming', resume_network_pkl
        G, D, _ = misc.load_pkl(
            os.path.join(config.result_dir, resume_network_pkl))
    else:
        G = network.Network(num_channels=training_set.shape[1],
                            resolution=training_set.shape[2],
                            label_size=training_set.labels.shape[1],
                            **config.G)
        D = network.Network(num_channels=training_set.shape[1],
                            resolution=training_set.shape[2],
                            label_size=training_set.labels.shape[1],
                            **config.D)
    Gs = G.create_temporally_smoothed_version(beta=G_smoothing,
                                              explicit_updates=True)
    misc.print_network_topology_info(G.output_layers)
    misc.print_network_topology_info(D.output_layers)

    # Setup snapshot image grid.
    if image_grid_type == 'default':
        if image_grid_size is None:
            w, h = G.output_shape[3], G.output_shape[2]
            image_grid_size = np.clip(1920 / w, 3,
                                      16), np.clip(1080 / h, 2, 16)
        example_real_images, snapshot_fake_labels = training_set.get_random_minibatch(
            np.prod(image_grid_size), labels=True)
        snapshot_fake_latents = random_latents(np.prod(image_grid_size),
                                               G.input_shape)
    elif image_grid_type == 'category':
        W = training_set.labels.shape[1]
        H = W if image_grid_size is None else image_grid_size[1]
        image_grid_size = W, H
        snapshot_fake_latents = random_latents(W * H, G.input_shape)
        snapshot_fake_labels = np.zeros((W * H, W),
                                        dtype=training_set.labels.dtype)
        example_real_images = np.zeros((W * H, ) + training_set.shape[1:],
                                       dtype=training_set.dtype)
        for x in xrange(W):
            snapshot_fake_labels[x::W, x] = 1.0
            indices = np.arange(
                training_set.shape[0])[training_set.labels[:, x] != 0]
            for y in xrange(H):
                example_real_images[x + y * W] = training_set.h5_lods[0][
                    np.random.choice(indices)]
    else:
        raise ValueError('Invalid image_grid_type', image_grid_type)

    # Theano input variables and compile generation func.
    print 'Setting up Theano...'
    real_images_var = T.TensorType('float32', [False] *
                                   len(D.input_shape))('real_images_var')
    real_labels_var = T.TensorType(
        'float32', [False] * len(training_set.labels.shape))('real_labels_var')
    fake_latents_var = T.TensorType('float32', [False] *
                                    len(G.input_shape))('fake_latents_var')
    fake_labels_var = T.TensorType(
        'float32', [False] * len(training_set.labels.shape))('fake_labels_var')
    G_lrate = theano.shared(np.float32(0.0))
    D_lrate = theano.shared(np.float32(0.0))
    gen_fn = theano.function([fake_latents_var, fake_labels_var],
                             Gs.eval_nd(fake_latents_var,
                                        fake_labels_var,
                                        ignore_unused_inputs=True),
                             on_unused_input='ignore')

    # Misc init.
    resolution_log2 = int(np.round(np.log2(G.output_shape[2])))
    initial_lod = max(
        resolution_log2 - int(np.round(np.log2(lod_initial_resolution))), 0)
    cur_lod = 0.0
    min_lod, max_lod = -1.0, -2.0
    fake_score_avg = 0.0

    if config.D.get('mbdisc_kernels', None):
        print 'Initializing minibatch discrimination...'
        if hasattr(D, 'cur_lod'): D.cur_lod.set_value(np.float32(initial_lod))
        D.eval(real_images_var, deterministic=False, init=True)
        init_layers = lasagne.layers.get_all_layers(D.output_layers)
        init_updates = [
            update for layer in init_layers
            for update in getattr(layer, 'init_updates', [])
        ]
        init_fn = theano.function(inputs=[real_images_var],
                                  outputs=None,
                                  updates=init_updates)
        init_reals = training_set.get_random_minibatch(500, lod=initial_lod)
        init_reals = misc.adjust_dynamic_range(init_reals, drange_orig,
                                               drange_net)
        init_fn(init_reals)
        del init_reals

    # Save example images.
    snapshot_fake_images = gen_fn(snapshot_fake_latents, snapshot_fake_labels)
    result_subdir = misc.create_result_subdir(config.result_dir,
                                              config.run_desc)
    misc.save_image_grid(example_real_images,
                         os.path.join(result_subdir, 'reals.png'),
                         drange=drange_orig,
                         grid_size=image_grid_size)
    misc.save_image_grid(snapshot_fake_images,
                         os.path.join(result_subdir, 'fakes%06d.png' % 0),
                         drange=drange_viz,
                         grid_size=image_grid_size)

    # Training loop.
    cur_nimg = int(resume_kimg * 1000)
    cur_tick = 0
    tick_start_nimg = cur_nimg
    tick_start_time = time.time()
    tick_train_out = []
    train_start_time = tick_start_time - resume_time
    while cur_nimg < total_kimg * 1000:

        # Calculate current LOD.
        cur_lod = initial_lod
        if lod_training_kimg or lod_transition_kimg:
            tlod = (cur_nimg / 1000.0) / (lod_training_kimg +
                                          lod_transition_kimg)
            cur_lod -= np.floor(tlod)
            if lod_transition_kimg:
                cur_lod -= max(
                    1.0 + (np.fmod(tlod, 1.0) - 1.0) *
                    (lod_training_kimg + lod_transition_kimg) /
                    lod_transition_kimg, 0.0)
            cur_lod = max(cur_lod, 0.0)

        # Look up resolution-dependent parameters.
        cur_res = 2**(resolution_log2 - int(np.floor(cur_lod)))
        minibatch_size = minibatch_overrides.get(cur_res, minibatch_default)
        tick_duration_kimg = tick_kimg_overrides.get(cur_res,
                                                     tick_kimg_default)

        # Update network config.
        lrate_coef = misc.rampup(cur_nimg / 1000.0, rampup_kimg)
        lrate_coef *= misc.rampdown_linear(cur_nimg / 1000.0, total_kimg,
                                           rampdown_kimg)
        G_lrate.set_value(np.float32(lrate_coef * G_learning_rate_max))
        D_lrate.set_value(np.float32(lrate_coef * D_learning_rate_max))
        if hasattr(G, 'cur_lod'): G.cur_lod.set_value(np.float32(cur_lod))
        if hasattr(D, 'cur_lod'): D.cur_lod.set_value(np.float32(cur_lod))

        # Setup training func for current LOD.
        new_min_lod, new_max_lod = int(np.floor(cur_lod)), int(
            np.ceil(cur_lod))
        if min_lod != new_min_lod or max_lod != new_max_lod:
            print 'Compiling training funcs...'
            min_lod, max_lod = new_min_lod, new_max_lod

            # Pre-process reals.
            real_images_expr = real_images_var
            if dequantize_reals:
                rnd = theano.sandbox.rng_mrg.MRG_RandomStreams(
                    lasagne.random.get_rng().randint(1, 2147462579))
                epsilon_noise = rnd.uniform(size=real_images_expr.shape,
                                            low=-0.5,
                                            high=0.5,
                                            dtype='float32')
                real_images_expr = T.cast(
                    real_images_expr, 'float32'
                ) + epsilon_noise  # match original implementation of Improved Wasserstein
            real_images_expr = misc.adjust_dynamic_range(
                real_images_expr, drange_orig, drange_net)
            if min_lod > 0:  # compensate for shrink_based_on_lod
                real_images_expr = T.extra_ops.repeat(real_images_expr,
                                                      2**min_lod,
                                                      axis=2)
                real_images_expr = T.extra_ops.repeat(real_images_expr,
                                                      2**min_lod,
                                                      axis=3)

            # Optimize loss.
            G_loss, D_loss, real_scores_out, fake_scores_out = evaluate_loss(
                G, D, min_lod, max_lod, real_images_expr, real_labels_var,
                fake_latents_var, fake_labels_var, **config.loss)
            G_updates = adam(G_loss,
                             G.trainable_params(),
                             learning_rate=G_lrate,
                             beta1=adam_beta1,
                             beta2=adam_beta2,
                             epsilon=adam_epsilon).items()
            D_updates = adam(D_loss,
                             D.trainable_params(),
                             learning_rate=D_lrate,
                             beta1=adam_beta1,
                             beta2=adam_beta2,
                             epsilon=adam_epsilon).items()

            # Compile training funcs.
            if not separate_funcs:
                GD_train_fn = theano.function([
                    real_images_var, real_labels_var, fake_latents_var,
                    fake_labels_var
                ], [G_loss, D_loss, real_scores_out, fake_scores_out],
                                              updates=G_updates + D_updates +
                                              Gs.updates,
                                              on_unused_input='ignore')
            else:
                D_train_fn = theano.function([
                    real_images_var, real_labels_var, fake_latents_var,
                    fake_labels_var
                ], [G_loss, D_loss, real_scores_out, fake_scores_out],
                                             updates=D_updates,
                                             on_unused_input='ignore')
                G_train_fn = theano.function(
                    [fake_latents_var, fake_labels_var], [],
                    updates=G_updates + Gs.updates,
                    on_unused_input='ignore')

        # Invoke training funcs.
        if not separate_funcs:
            assert D_training_repeats == 1
            mb_reals, mb_labels = training_set.get_random_minibatch(
                minibatch_size,
                lod=cur_lod,
                shrink_based_on_lod=True,
                labels=True)
            mb_train_out = GD_train_fn(
                mb_reals, mb_labels,
                random_latents(minibatch_size, G.input_shape),
                random_labels(minibatch_size, training_set))
            cur_nimg += minibatch_size
            tick_train_out.append(mb_train_out)
        else:
            for idx in xrange(D_training_repeats):
                mb_reals, mb_labels = training_set.get_random_minibatch(
                    minibatch_size,
                    lod=cur_lod,
                    shrink_based_on_lod=True,
                    labels=True)
                mb_train_out = D_train_fn(
                    mb_reals, mb_labels,
                    random_latents(minibatch_size, G.input_shape),
                    random_labels(minibatch_size, training_set))
                cur_nimg += minibatch_size
                tick_train_out.append(mb_train_out)
            G_train_fn(random_latents(minibatch_size, G.input_shape),
                       random_labels(minibatch_size, training_set))

        # Fade in D noise if we're close to becoming unstable
        fake_score_cur = np.clip(np.mean(mb_train_out[1]), 0.0, 1.0)
        fake_score_avg = fake_score_avg * gdrop_beta + fake_score_cur * (
            1.0 - gdrop_beta)
        gdrop_strength = gdrop_coef * (max(fake_score_avg - gdrop_lim, 0.0)**
                                       gdrop_exp)
        if hasattr(D, 'gdrop_strength'):
            D.gdrop_strength.set_value(np.float32(gdrop_strength))

        # Perform maintenance operations once per tick.
        if cur_nimg >= tick_start_nimg + tick_duration_kimg * 1000 or cur_nimg >= total_kimg * 1000:
            cur_tick += 1
            cur_time = time.time()
            tick_kimg = (cur_nimg - tick_start_nimg) / 1000.0
            tick_start_nimg = cur_nimg
            tick_time = cur_time - tick_start_time
            tick_start_time = cur_time
            tick_train_avg = tuple(
                np.mean(np.concatenate([np.asarray(v).flatten()
                                        for v in vals]))
                for vals in zip(*tick_train_out))
            tick_train_out = []

            # Print progress.
            print 'tick %-5d kimg %-8.1f lod %-5.2f minibatch %-4d time %-12s sec/tick %-9.1f sec/kimg %-6.1f Dgdrop %-8.4f Gloss %-8.4f Dloss %-8.4f Dreal %-8.4f Dfake %-8.4f' % (
                (cur_tick, cur_nimg / 1000.0, cur_lod, minibatch_size,
                 misc.format_time(cur_time - train_start_time), tick_time,
                 tick_time / tick_kimg, gdrop_strength) + tick_train_avg)

            # Visualize generated images.
            if cur_tick % image_snapshot_ticks == 0 or cur_nimg >= total_kimg * 1000:
                snapshot_fake_images = gen_fn(snapshot_fake_latents,
                                              snapshot_fake_labels)
                misc.save_image_grid(snapshot_fake_images,
                                     os.path.join(
                                         result_subdir,
                                         'fakes%06d.png' % (cur_nimg / 1000)),
                                     drange=drange_viz,
                                     grid_size=image_grid_size)

            # Save network snapshot every N ticks.
            if cur_tick % network_snapshot_ticks == 0 or cur_nimg >= total_kimg * 1000:
                misc.save_pkl(
                    (G, D, Gs),
                    os.path.join(
                        result_subdir,
                        'network-snapshot-%06d.pkl' % (cur_nimg / 1000)))

    # Write final results.
    misc.save_pkl((G, D, Gs), os.path.join(result_subdir, 'network-final.pkl'))
    training_set.close()
    print 'Done.'
    with open(os.path.join(result_subdir, '_training-done.txt'), 'wt'):
        pass
Exemplo n.º 26
0
def train_gan(
        separate_funcs=False,
        D_training_repeats=1,
        G_learning_rate_max=0.0010,
        D_learning_rate_max=0.0010,
        G_smoothing=0.999,
        adam_beta1=0.0,
        adam_beta2=0.99,
        adam_epsilon=1e-8,
        minibatch_default=16,
        minibatch_overrides={},
        rampup_kimg=40 / speed_factor,
        rampdown_kimg=0,
        lod_initial_resolution=4,
        lod_training_kimg=400 / speed_factor,
        lod_transition_kimg=400 / speed_factor,
        #lod_training_kimg       = 40,
        #lod_transition_kimg     = 40,
        total_kimg=10000 / speed_factor,
        dequantize_reals=False,
        gdrop_beta=0.9,
        gdrop_lim=0.5,
        gdrop_coef=0.2,
        gdrop_exp=2.0,
        drange_net=[-1, 1],
        drange_viz=[-1, 1],
        image_grid_size=None,
        #tick_kimg_default       = 1,
        tick_kimg_default=50 / speed_factor,
        tick_kimg_overrides={
            32: 20,
            64: 10,
            128: 10,
            256: 5,
            512: 2,
            1024: 1
        },
        image_snapshot_ticks=4,
        network_snapshot_ticks=40,
        image_grid_type='default',
        #resume_network_pkl      = '006-celeb128-progressive-growing/network-snapshot-002009.pkl',
        resume_network_pkl=None,
        resume_kimg=0,
        resume_time=0.0):

    # Load dataset and build networks.
    training_set, drange_orig = load_dataset()

    print "*************** test the format of dataset ***************"
    print training_set
    print drange_orig
    # training_set是dataset模块解析h5之后的对象,
    # drange_orig 为training_set.get_dynamic_range()

    if resume_network_pkl:
        print 'Resuming', resume_network_pkl
        G, D, _ = misc.load_pkl(
            os.path.join(config.result_dir, resume_network_pkl))
    else:
        G = network.Network(num_channels=training_set.shape[1],
                            resolution=training_set.shape[2],
                            label_size=training_set.labels.shape[1],
                            **config.G)
        D = network.Network(num_channels=training_set.shape[1],
                            resolution=training_set.shape[2],
                            label_size=training_set.labels.shape[1],
                            **config.D)
    Gs = G.create_temporally_smoothed_version(beta=G_smoothing,
                                              explicit_updates=True)

    # G,D对象可以由misc解析pkl之后生成,也可以由network模块构造

    print G
    print D

    #misc.print_network_topology_info(G.output_layers)
    #misc.print_network_topology_info(D.output_layers)

    # Setup snapshot image grid.
    # 设置中途输出图片的格式
    if image_grid_type == 'default':
        if image_grid_size is None:
            w, h = G.output_shape[3], G.output_shape[2]
            image_grid_size = np.clip(1920 / w, 3,
                                      16), np.clip(1080 / h, 2, 16)
        example_real_images, snapshot_fake_labels = training_set.get_random_minibatch(
            np.prod(image_grid_size), labels=True)
        snapshot_fake_latents = random_latents(np.prod(image_grid_size),
                                               G.input_shape)
    else:
        raise ValueError('Invalid image_grid_type', image_grid_type)

    # Theano input variables and compile generation func.
    print 'Setting up Theano...'
    real_images_var = T.TensorType('float32', [False] *
                                   len(D.input_shape))('real_images_var')
    # <class 'theano.tensor.var.TensorVariable'>
    # print type(real_images_var),real_images_var
    real_labels_var = T.TensorType(
        'float32', [False] * len(training_set.labels.shape))('real_labels_var')
    fake_latents_var = T.TensorType('float32', [False] *
                                    len(G.input_shape))('fake_latents_var')
    fake_labels_var = T.TensorType(
        'float32', [False] * len(training_set.labels.shape))('fake_labels_var')
    # 带有_var的均为输入张量
    G_lrate = theano.shared(np.float32(0.0))
    D_lrate = theano.shared(np.float32(0.0))
    # share语法就是用来设定默认值的,返回复制的对象
    gen_fn = theano.function([fake_latents_var, fake_labels_var],
                             Gs.eval_nd(fake_latents_var,
                                        fake_labels_var,
                                        ignore_unused_inputs=True),
                             on_unused_input='ignore')

    # gen_fn 是一个函数,输入为:[fake_latents_var, fake_labels_var],
    #                  输出位:Gs.eval_nd(fake_latents_var, fake_labels_var, ignore_unused_inputs=True),

    #生成函数

    # Misc init.
    #读入当前分辨率
    resolution_log2 = int(np.round(np.log2(G.output_shape[2])))
    #lod 精细度
    initial_lod = max(
        resolution_log2 - int(np.round(np.log2(lod_initial_resolution))), 0)
    cur_lod = 0.0
    min_lod, max_lod = -1.0, -2.0
    fake_score_avg = 0.0

    # Save example images.
    snapshot_fake_images = gen_fn(snapshot_fake_latents, snapshot_fake_labels)
    result_subdir = misc.create_result_subdir(config.result_dir,
                                              config.run_desc)
    misc.save_image_grid(example_real_images,
                         os.path.join(result_subdir, 'reals.png'),
                         drange=drange_orig,
                         grid_size=image_grid_size)
    misc.save_image_grid(snapshot_fake_images,
                         os.path.join(result_subdir, 'fakes%06d.png' % 0),
                         drange=drange_viz,
                         grid_size=image_grid_size)

    # Training loop.
    # 这里才是主训练入口
    # 注意在训练过程中不会跳出最外层while循环,因此更换分辨率等操作必然在while循环里

    #现有图片数
    cur_nimg = int(resume_kimg * 1000)
    cur_tick = 0

    tick_start_nimg = cur_nimg
    tick_start_time = time.time()
    tick_train_out = []
    train_start_time = tick_start_time - resume_time
    while cur_nimg < total_kimg * 1000:

        # Calculate current LOD.
        #计算当前精细度
        cur_lod = initial_lod
        if lod_training_kimg or lod_transition_kimg:
            tlod = (cur_nimg / (1000.0 / speed_factor)) / (lod_training_kimg +
                                                           lod_transition_kimg)
            cur_lod -= np.floor(tlod)
            if lod_transition_kimg:
                cur_lod -= max(
                    1.0 + (np.fmod(tlod, 1.0) - 1.0) *
                    (lod_training_kimg + lod_transition_kimg) /
                    lod_transition_kimg, 0.0)
            cur_lod = max(cur_lod, 0.0)

        # Look up resolution-dependent parameters.
        cur_res = 2**(resolution_log2 - int(np.floor(cur_lod)))
        # 当前分辨率
        minibatch_size = minibatch_overrides.get(cur_res, minibatch_default)
        tick_duration_kimg = tick_kimg_overrides.get(cur_res,
                                                     tick_kimg_default)

        # Update network config.
        # 更新网络结构
        lrate_coef = misc.rampup(cur_nimg / 1000.0, rampup_kimg)
        lrate_coef *= misc.rampdown_linear(cur_nimg / 1000.0, total_kimg,
                                           rampdown_kimg)
        G_lrate.set_value(np.float32(lrate_coef * G_learning_rate_max))
        D_lrate.set_value(np.float32(lrate_coef * D_learning_rate_max))

        if hasattr(G, 'cur_lod'): G.cur_lod.set_value(np.float32(cur_lod))
        if hasattr(D, 'cur_lod'): D.cur_lod.set_value(np.float32(cur_lod))

        # Setup training func for current LOD.
        new_min_lod, new_max_lod = int(np.floor(cur_lod)), int(
            np.ceil(cur_lod))

        #print " cur_lod%f\n  min_lod %f\n new_min_lod %f\n max_lod %f\n new_max_lod %f\n"%(cur_lod,min_lod,new_min_lod,max_lod,new_max_lod)

        if min_lod != new_min_lod or max_lod != new_max_lod:
            print 'Compiling training funcs...'
            min_lod, max_lod = new_min_lod, new_max_lod

            # Pre-process reals.
            real_images_expr = real_images_var
            if dequantize_reals:
                rnd = theano.sandbox.rng_mrg.MRG_RandomStreams(
                    lasagne.random.get_rng().randint(1, 2147462579))
                epsilon_noise = rnd.uniform(size=real_images_expr.shape,
                                            low=-0.5,
                                            high=0.5,
                                            dtype='float32')
                real_images_expr = T.cast(
                    real_images_expr, 'float32'
                ) + epsilon_noise  # match original implementation of Improved Wasserstein
            real_images_expr = misc.adjust_dynamic_range(
                real_images_expr, drange_orig, drange_net)
            if min_lod > 0:  # compensate for shrink_based_on_lod
                real_images_expr = T.extra_ops.repeat(real_images_expr,
                                                      2**min_lod,
                                                      axis=2)
                real_images_expr = T.extra_ops.repeat(real_images_expr,
                                                      2**min_lod,
                                                      axis=3)

            # Optimize loss.
            G_loss, D_loss, real_scores_out, fake_scores_out = evaluate_loss(
                G, D, min_lod, max_lod, real_images_expr, real_labels_var,
                fake_latents_var, fake_labels_var, **config.loss)
            G_updates = adam(G_loss,
                             G.trainable_params(),
                             learning_rate=G_lrate,
                             beta1=adam_beta1,
                             beta2=adam_beta2,
                             epsilon=adam_epsilon).items()

            D_updates = adam(D_loss,
                             D.trainable_params(),
                             learning_rate=D_lrate,
                             beta1=adam_beta1,
                             beta2=adam_beta2,
                             epsilon=adam_epsilon).items()

            D_train_fn = theano.function([
                real_images_var, real_labels_var, fake_latents_var,
                fake_labels_var
            ], [G_loss, D_loss, real_scores_out, fake_scores_out],
                                         updates=D_updates,
                                         on_unused_input='ignore')
            G_train_fn = theano.function([fake_latents_var, fake_labels_var],
                                         [],
                                         updates=G_updates + Gs.updates,
                                         on_unused_input='ignore')

        for idx in xrange(D_training_repeats):
            mb_reals, mb_labels = training_set.get_random_minibatch(
                minibatch_size,
                lod=cur_lod,
                shrink_based_on_lod=True,
                labels=True)

            print "******* test minibatch"
            print "mb_reals"
            print idx, D_training_repeats
            print mb_reals.shape, mb_labels.shape
            #print mb_reals
            print "mb_labels"
            #print mb_labels

            mb_train_out = D_train_fn(
                mb_reals, mb_labels,
                random_latents(minibatch_size, G.input_shape),
                random_labels(minibatch_size, training_set))
            cur_nimg += minibatch_size
            tick_train_out.append(mb_train_out)
        G_train_fn(random_latents(minibatch_size, G.input_shape),
                   random_labels(minibatch_size, training_set))

        # Fade in D noise if we're close to becoming unstable
        fake_score_cur = np.clip(np.mean(mb_train_out[1]), 0.0, 1.0)
        fake_score_avg = fake_score_avg * gdrop_beta + fake_score_cur * (
            1.0 - gdrop_beta)
        gdrop_strength = gdrop_coef * (max(fake_score_avg - gdrop_lim, 0.0)**
                                       gdrop_exp)
        if hasattr(D, 'gdrop_strength'):
            D.gdrop_strength.set_value(np.float32(gdrop_strength))

        # Perform maintenance operations once per tick.
        if cur_nimg >= tick_start_nimg + tick_duration_kimg * 1000 or cur_nimg >= total_kimg * 1000:
            cur_tick += 1
            cur_time = time.time()
            tick_kimg = (cur_nimg - tick_start_nimg) / 1000.0
            tick_start_nimg = cur_nimg
            tick_time = cur_time - tick_start_time
            tick_start_time = cur_time
            tick_train_avg = tuple(
                np.mean(np.concatenate([np.asarray(v).flatten()
                                        for v in vals]))
                for vals in zip(*tick_train_out))
            tick_train_out = []

            # Print progress.
            print 'tick %-5d kimg %-8.1f lod %-5.2f minibatch %-4d time %-12s sec/tick %-9.1f sec/kimg %-6.1f Dgdrop %-8.4f Gloss %-8.4f Dloss %-8.4f Dreal %-8.4f Dfake %-8.4f' % (
                (cur_tick, cur_nimg / 1000.0, cur_lod, minibatch_size,
                 misc.format_time(cur_time - train_start_time), tick_time,
                 tick_time / tick_kimg, gdrop_strength) + tick_train_avg)

            # Visualize generated images.
            if cur_tick % image_snapshot_ticks == 0 or cur_nimg >= total_kimg * 1000:
                snapshot_fake_images = gen_fn(snapshot_fake_latents,
                                              snapshot_fake_labels)
                misc.save_image_grid(snapshot_fake_images,
                                     os.path.join(
                                         result_subdir,
                                         'fakes%06d.png' % (cur_nimg / 1000)),
                                     drange=drange_viz,
                                     grid_size=image_grid_size)

            # Save network snapshot every N ticks.
            if cur_tick % network_snapshot_ticks == 0 or cur_nimg >= total_kimg * 1000:
                misc.save_pkl(
                    (G, D, Gs),
                    os.path.join(
                        result_subdir,
                        'network-snapshot-%06d.pkl' % (cur_nimg / 1000)))

    # Write final results.
    misc.save_pkl((G, D, Gs), os.path.join(result_subdir, 'network-final.pkl'))
    training_set.close()
    print 'Done.'
    with open(os.path.join(result_subdir, '_training-done.txt'), 'wt'):
        pass
Exemplo n.º 27
0
def calc_sliced_wasserstein_scores(run_id,
                                   log='sliced-wasserstein.txt',
                                   resolution_min=16,
                                   resolution_max=1024,
                                   num_images=8192,
                                   nhoods_per_image=64,
                                   nhood_size=7,
                                   dir_repeats=1,
                                   dirs_per_repeat=147,
                                   minibatch_size=16):

    import sliced_wasserstein
    result_subdir = misc.locate_result_subdir(run_id)
    network_pkls = misc.list_network_pkls(result_subdir)
    misc.set_output_log_file(os.path.join(result_subdir, log))

    # Load dataset.
    print 'Loading dataset...'
    training_set, drange_orig = load_dataset_for_previous_run(result_subdir)
    assert training_set.shape[1] == 3  # RGB
    assert num_images % minibatch_size == 0

    # Select resolutions.
    resolution_full = training_set.shape[3]
    resolution_min = min(resolution_min, resolution_full)
    resolution_max = min(resolution_max, resolution_full)
    base_lod = int(np.log2(resolution_full)) - int(np.log2(resolution_max))
    resolutions = [
        2**i for i in xrange(int(np.log2(resolution_max)),
                             int(np.log2(resolution_min)) - 1, -1)
    ]

    # Collect descriptors for reals.
    print 'Extracting descriptors for reals...',
    time_begin = time.time()
    desc_real = [[] for res in resolutions]
    desc_test = [[] for res in resolutions]
    for minibatch_begin in xrange(0, num_images, minibatch_size):
        minibatch = training_set.get_random_minibatch(minibatch_size,
                                                      lod=base_lod)
        for lod, level in enumerate(
                sliced_wasserstein.generate_laplacian_pyramid(
                    minibatch, len(resolutions))):
            desc_real[lod].append(
                sliced_wasserstein.get_descriptors_for_minibatch(
                    level, nhood_size, nhoods_per_image))
            desc_test[lod].append(
                sliced_wasserstein.get_descriptors_for_minibatch(
                    level, nhood_size, nhoods_per_image))
    print 'done in %s' % misc.format_time(time.time() - time_begin)

    # Evaluate scores for reals.
    print 'Evaluating scores for reals...',
    time_begin = time.time()
    scores = []
    for lod, res in enumerate(resolutions):
        desc_real[lod] = sliced_wasserstein.finalize_descriptors(
            desc_real[lod])
        desc_test[lod] = sliced_wasserstein.finalize_descriptors(
            desc_test[lod])
        scores.append(
            sliced_wasserstein.sliced_wasserstein(desc_real[lod],
                                                  desc_test[lod], dir_repeats,
                                                  dirs_per_repeat))
    del desc_test
    print 'done in %s' % misc.format_time(time.time() - time_begin)

    # Print table header.
    print
    print '%-32s' % 'Case',
    for lod, res in enumerate(resolutions):
        print '%-12s' % ('%dx%d' % (res, res)),
    print 'Average'
    print '%-32s' % '---',
    for lod, res in enumerate(resolutions):
        print '%-12s' % '---',
    print '---'
    print '%-32s' % 'reals',
    for lod, res in enumerate(resolutions):
        print '%-12.6f' % scores[lod],
    print '%.6f' % np.mean(scores)

    # Process each network snapshot.
    for network_idx, network_pkl in enumerate(network_pkls):
        print '%-32s' % os.path.basename(network_pkl),
        net = imgapi_load_net(run_id=result_subdir,
                              snapshot=network_pkl,
                              num_example_latents=num_images,
                              random_seed=network_idx)

        # Extract descriptors for generated images.
        desc_fake = [[] for res in resolutions]
        for minibatch_begin in xrange(0, num_images, minibatch_size):
            latents = net.example_latents[minibatch_begin:minibatch_begin +
                                          minibatch_size]
            labels = net.example_labels[minibatch_begin:minibatch_begin +
                                        minibatch_size]
            minibatch = imgapi_generate_batch(net,
                                              latents,
                                              labels,
                                              minibatch_size=minibatch_size,
                                              convert_to_uint8=True)
            minibatch = sliced_wasserstein.downscale_minibatch(
                minibatch, base_lod)
            for lod, level in enumerate(
                    sliced_wasserstein.generate_laplacian_pyramid(
                        minibatch, len(resolutions))):
                desc_fake[lod].append(
                    sliced_wasserstein.get_descriptors_for_minibatch(
                        level, nhood_size, nhoods_per_image))

        # Evaluate scores.
        scores = []
        for lod, res in enumerate(resolutions):
            desc_fake[lod] = sliced_wasserstein.finalize_descriptors(
                desc_fake[lod])
            scores.append(
                sliced_wasserstein.sliced_wasserstein(desc_real[lod],
                                                      desc_fake[lod],
                                                      dir_repeats,
                                                      dirs_per_repeat))
        del desc_fake

        # Report results.
        for lod, res in enumerate(resolutions):
            print '%-12.6f' % scores[lod],
        print '%.6f' % np.mean(scores)
    print
    print 'Done.'
Exemplo n.º 28
0
    def run(self):
        self.parent.clear_plots()
        if self.data is None:
            length, tMin, tMax, fMin, fMax, lMin, lMax, peakF, peakL, peakT = (
                '', ) * 10
        else:
            length = len(self.data)
            tMin = format_time(self.extent.tMin, True)
            tMax = format_time(self.extent.tMax, True)
            fMin = format_precision(self.settings,
                                    freq=self.extent.fMin,
                                    fancyUnits=True)
            fMax = format_precision(self.settings,
                                    freq=self.extent.fMax,
                                    fancyUnits=True)
            lMin = format_precision(self.settings,
                                    level=self.extent.lMin,
                                    fancyUnits=True)
            lMax = format_precision(self.settings,
                                    level=self.extent.lMax,
                                    fancyUnits=True)
            peak = self.extent.get_peak_flt()
            peakF = format_precision(self.settings,
                                     freq=peak[0],
                                     fancyUnits=True)
            peakL = format_precision(self.settings,
                                     level=peak[1],
                                     fancyUnits=True)
            peakT = format_time(peak[2], True)

        text = [
            ['Sweeps', '', length],
            ['Extents', '', ''],
            ['', 'Start', tMin],
            ['', 'End', tMax],
            ['', 'Min frequency', fMin],
            ['', 'Max frequency', fMax],
            ['', 'Min level', lMin],
            ['', 'Max level', lMax],
            ['Peak', '', ''],
            ['', 'Level', peakL],
            ['', 'Frequency', peakF],
            ['', 'Time', peakT],
        ]

        table = Table(self.axes, loc='center')
        table.set_gid('table')

        rows = len(text)
        cols = len(text[0])
        for row in xrange(rows):
            for col in xrange(cols):
                table.add_cell(row,
                               col,
                               text=text[row][col],
                               width=1.0 / cols,
                               height=1.0 / rows)

        if self.settings.grid:
            colour = 'LightGray'
        else:
            colour = 'w'
        set_table_colour(table, colour)

        for i in range(3):
            table.auto_set_column_width(i)

        self.axes.add_table(table)
        self.parent.redraw_plot()

        self.parent.threadPlot = None
Exemplo n.º 29
0
def evaluate_metrics(run_id,
                     log,
                     metrics,
                     num_images,
                     real_passes,
                     minibatch_size=None):
    metric_class_names = {
        'swd': 'metrics.sliced_wasserstein.API',
        'fid': 'metrics.frechet_inception_distance.API',
        'is': 'metrics.inception_score.API',
        'msssim': 'metrics.ms_ssim.API',
    }

    # Locate training run and initialize logging.
    result_subdir = misc.locate_result_subdir(run_id)
    snapshot_pkls = misc.list_network_pkls(result_subdir, include_final=False)
    assert len(snapshot_pkls) >= 1
    log_file = os.path.join(result_subdir, log)
    print('Logging output to', log_file)
    misc.set_output_log_file(log_file)

    # Initialize dataset and select minibatch size.
    dataset_obj, mirror_augment = misc.load_dataset_for_previous_run(
        result_subdir, verbose=True, shuffle_mb=0)
    if minibatch_size is None:
        minibatch_size = np.clip(8192 // dataset_obj.shape[1], 4, 256)

    # Initialize metrics.
    metric_objs = []
    for name in metrics:
        class_name = metric_class_names.get(name, name)
        print('Initializing %s...' % class_name)
        class_def = tfutil.import_obj(class_name)
        image_shape = [3] + dataset_obj.shape[1:]
        obj = class_def(num_images=num_images,
                        image_shape=image_shape,
                        image_dtype=np.uint8,
                        minibatch_size=minibatch_size)
        tfutil.init_uninited_vars()
        mode = 'warmup'
        obj.begin(mode)
        for idx in range(10):
            obj.feed(
                mode,
                np.random.randint(0,
                                  256,
                                  size=[minibatch_size] + image_shape,
                                  dtype=np.uint8))
        obj.end(mode)
        metric_objs.append(obj)

    # Print table header.
    print()
    print('%-10s%-12s' % ('Snapshot', 'Time_eval'), end='')
    for obj in metric_objs:
        for name, fmt in zip(obj.get_metric_names(),
                             obj.get_metric_formatting()):
            print('%-*s' % (len(fmt % 0), name), end='')
    print()
    print('%-10s%-12s' % ('---', '---'), end='')
    for obj in metric_objs:
        for fmt in obj.get_metric_formatting():
            print('%-*s' % (len(fmt % 0), '---'), end='')
    print()

    # Feed in reals.
    for title, mode in [('Reals', 'reals'), ('Reals2', 'fakes')][:real_passes]:
        print('%-10s' % title, end='')
        time_begin = time.time()
        labels = np.zeros([num_images, dataset_obj.label_size],
                          dtype=np.float32)
        [obj.begin(mode) for obj in metric_objs]
        for begin in range(0, num_images, minibatch_size):
            end = min(begin + minibatch_size, num_images)
            images, labels[begin:end] = dataset_obj.get_minibatch_np(end -
                                                                     begin)
            if mirror_augment:
                images = misc.apply_mirror_augment(images)
            if images.shape[1] == 1:
                images = np.tile(images, [1, 3, 1, 1])  # grayscale => RGB
            [obj.feed(mode, images) for obj in metric_objs]
        results = [obj.end(mode) for obj in metric_objs]
        print('%-12s' % misc.format_time(time.time() - time_begin), end='')
        for obj, vals in zip(metric_objs, results):
            for val, fmt in zip(vals, obj.get_metric_formatting()):
                print(fmt % val, end='')
        print()

    # Evaluate each network snapshot.
    for snapshot_idx, snapshot_pkl in enumerate(reversed(snapshot_pkls)):
        prefix = 'network-snapshot-'
        postfix = '.pkl'
        snapshot_name = os.path.basename(snapshot_pkl)
        assert snapshot_name.startswith(prefix) and snapshot_name.endswith(
            postfix)
        snapshot_kimg = int(snapshot_name[len(prefix):-len(postfix)])

        print('%-10d' % snapshot_kimg, end='')
        mode = 'fakes'
        [obj.begin(mode) for obj in metric_objs]
        time_begin = time.time()
        with tf.Graph().as_default(), tfutil.create_session(
                config.tf_config).as_default():
            G, D, Gs = misc.load_pkl(snapshot_pkl)
            for begin in range(0, num_images, minibatch_size):
                end = min(begin + minibatch_size, num_images)
                latents = misc.random_latents(end - begin, Gs)
                images = Gs.run(latents,
                                labels[begin:end],
                                num_gpus=config.num_gpus,
                                out_mul=127.5,
                                out_add=127.5,
                                out_dtype=np.uint8)
                if images.shape[1] == 1:
                    images = np.tile(images, [1, 3, 1, 1])  # grayscale => RGB
                [obj.feed(mode, images) for obj in metric_objs]
        results = [obj.end(mode) for obj in metric_objs]
        print('%-12s' % misc.format_time(time.time() - time_begin), end='')
        for obj, vals in zip(metric_objs, results):
            for val, fmt in zip(vals, obj.get_metric_formatting()):
                print(fmt % val, end='')
        print()
    print()
Exemplo n.º 30
0
def train_detector(
    minibatch_repeats=4,  # Number of minibatches to run before adjusting training parameters.
    total_kimg=1,  # Total length of the training, measured in thousands of real images.
    drange_net=[
        -1, 1
    ],  # Dynamic range used when feeding image data to the networks.
    snapshot_size=16,  # Size of the snapshot image
    snapshot_ticks=2**13,  # Number of images before maintenance
    image_snapshot_ticks=1,  # How often to export image snapshots?
    network_snapshot_ticks=1,  # How often to export network snapshots?
    save_tf_graph=True,  # Include full TensorFlow computation graph in the tfevents file?
    save_weight_histograms=False,  # Include weight histograms in the tfevents file?
    resume_run_id=None,  # Run ID or network pkl to resume training from, None = start from scratch.
    resume_snapshot=None,  # Snapshot index to resume training from, None = autodetect.
    resume_kimg=0.0,  # Assumed training progress at the beginning. Affects reporting and training schedule.
    resume_time=0.0
):  # Assumed wallclock time at the beginning. Affects reporting.

    maintenance_start_time = time.time()

    # Load the datasets
    training_set = dataset.load_dataset(tfrecord=config.tfrecord_train,
                                        verbose=True,
                                        **config.dataset)
    testing_set = dataset.load_dataset(tfrecord=config.tfrecord_test,
                                       verbose=True,
                                       repeat=False,
                                       shuffle_mb=0,
                                       **config.dataset)
    testing_set_len = len(testing_set)

    # TODO: data augmentation
    # TODO: testing set

    # Construct networks.
    with tf.device('/gpu:0'):
        if resume_run_id is not None:  # TODO: save methods
            network_pkl = misc.locate_network_pkl(resume_run_id,
                                                  resume_snapshot)
            print('Loading networks from "%s"...' % network_pkl)
            N = misc.load_pkl(network_pkl)
        else:
            print('Constructing the network...'
                  )  # TODO: better network (like lod-wise network)
            N = tfutil.Network('N',
                               num_channels=training_set.shape[0],
                               resolution=training_set.shape[1],
                               **config.N)
    N.print_layers()

    print('Building TensorFlow graph...')
    # Training set up
    with tf.name_scope('Inputs'):
        lrate_in = tf.placeholder(tf.float32, name='lrate_in', shape=[])
        # minibatch_in            = tf.placeholder(tf.int32, name='minibatch_in', shape=[])
        reals, labels, bboxes = training_set.get_minibatch_tf(
        )  # TODO: increase the size of the batch by several loss computation and mean
    N_opt = tfutil.Optimizer(name='TrainN',
                             learning_rate=lrate_in,
                             **config.N_opt)

    with tf.device('/gpu:0'):
        reals, labels, gt_outputs, gt_ref = pre_process(
            reals, labels, bboxes, training_set.dynamic_range,
            [0, training_set.shape[-2]], drange_net)
        with tf.name_scope('N_loss'):  # TODO: loss inadapted
            N_loss = tfutil.call_func_by_name(N=N,
                                              reals=reals,
                                              gt_outputs=gt_outputs,
                                              gt_ref=gt_ref,
                                              **config.N_loss)

        N_opt.register_gradients(tf.reduce_mean(N_loss), N.trainables)
    N_train_op = N_opt.apply_updates()

    # Testing set up
    with tf.device('/gpu:0'):
        test_reals_tf, test_labels_tf, test_bboxes_tf = testing_set.get_minibatch_tf(
        )
        test_reals_tf, test_labels_tf, test_gt_outputs_tf, test_gt_ref_tf = pre_process(
            test_reals_tf, test_labels_tf, test_bboxes_tf,
            testing_set.dynamic_range, [0, testing_set.shape[-2]], drange_net)
        with tf.name_scope('N_test_loss'):
            test_loss = tfutil.call_func_by_name(N=N,
                                                 reals=test_reals_tf,
                                                 gt_outputs=test_gt_outputs_tf,
                                                 gt_ref=test_gt_ref_tf,
                                                 is_training=False,
                                                 **config.N_loss)

    print('Setting up result dir...')
    result_subdir = misc.create_result_subdir(config.result_dir, config.desc)
    summary_log = tf.summary.FileWriter(result_subdir)
    if save_tf_graph:
        summary_log.add_graph(tf.get_default_graph())
    if save_weight_histograms:
        N.setup_weight_histograms()

    test_reals, _, test_bboxes = testing_set.get_minibatch_np(snapshot_size)
    misc.save_img_bboxes(test_reals,
                         test_bboxes,
                         os.path.join(result_subdir, 'reals.png'),
                         snapshot_size,
                         adjust_range=False)

    test_reals = misc.adjust_dynamic_range(test_reals,
                                           training_set.dynamic_range,
                                           drange_net)
    test_preds, _ = N.run(test_reals, minibatch_size=snapshot_size)
    misc.save_img_bboxes(test_reals, test_preds,
                         os.path.join(result_subdir, 'fakes.png'),
                         snapshot_size)

    print('Training...')
    if resume_run_id is None:
        tfutil.run(tf.global_variables_initializer())

    cur_nimg = 0
    cur_tick = 0
    tick_start_nimg = cur_nimg
    tick_start_time = time.time()
    train_start_time = tick_start_time

    # Choose training parameters and configure training ops.
    sched = TrainingSchedule(cur_nimg, training_set, **config.sched)
    training_set.configure(sched.minibatch)

    _train_loss = 0

    while cur_nimg < total_kimg * 1000:

        # Run training ops.
        # for _ in range(minibatch_repeats):
        _, loss = tfutil.run([N_train_op, N_loss], {lrate_in: sched.N_lrate})
        _train_loss += loss
        cur_nimg += sched.minibatch

        # Perform maintenance tasks once per tick.
        if (cur_nimg >= total_kimg * 1000) or (cur_nimg % snapshot_ticks == 0
                                               and cur_nimg > 0):

            cur_tick += 1
            cur_time = time.time()
            # tick_kimg = (cur_nimg - tick_start_nimg) / 1000.0
            _train_loss = _train_loss / (cur_nimg - tick_start_nimg)
            tick_start_nimg = cur_nimg
            tick_time = cur_time - tick_start_time
            total_time = cur_time - train_start_time
            maintenance_time = tick_start_time - maintenance_start_time
            maintenance_start_time = cur_time

            testing_set.configure(sched.minibatch)
            _test_loss = 0
            # testing_set_len = 1 # TMP
            for _ in range(0, testing_set_len, sched.minibatch):
                _test_loss += tfutil.run(test_loss)
            _test_loss /= testing_set_len

            # Report progress. # TODO: improved report display
            print(
                'tick %-5d kimg %-6.1f time %-10s sec/tick %-3.1f maintenance %-7.2f train_loss %.4f test_loss %.4f'
                % (tfutil.autosummary('Progress/tick', cur_tick),
                   tfutil.autosummary('Progress/kimg', cur_nimg / 1000),
                   misc.format_time(
                       tfutil.autosummary('Timing/total_sec', total_time)),
                   tfutil.autosummary('Timing/sec_per_tick', tick_time),
                   tfutil.autosummary('Timing/maintenance', maintenance_time),
                   tfutil.autosummary('TrainN/train_loss', _train_loss),
                   tfutil.autosummary('TrainN/test_loss', _test_loss)))

            tfutil.autosummary('Timing/total_hours',
                               total_time / (60.0 * 60.0))
            tfutil.autosummary('Timing/total_days',
                               total_time / (24.0 * 60.0 * 60.0))
            tfutil.save_summaries(summary_log, cur_nimg)

            if cur_tick % image_snapshot_ticks == 0:
                test_bboxes, test_refs = N.run(test_reals,
                                               minibatch_size=snapshot_size)
                misc.save_img_bboxes_ref(
                    test_reals, test_bboxes, test_refs,
                    os.path.join(result_subdir,
                                 'fakes%06d.png' % (cur_nimg // 1000)),
                    snapshot_size)
            if cur_tick % network_snapshot_ticks == 0:
                misc.save_pkl(
                    N,
                    os.path.join(
                        result_subdir,
                        'network-snapshot-%06d.pkl' % (cur_nimg // 1000)))

            _train_loss = 0

            # Record start time of the next tick.
            tick_start_time = time.time()

    # Write final results.
    # misc.save_pkl(N, os.path.join(result_subdir, 'network-final.pkl'))
    summary_log.close()
    open(os.path.join(result_subdir, '_training-done.txt'), 'wt').close()
Exemplo n.º 31
0
def train_progressive_gan(
    G_smoothing=0.999,  # Exponential running average of generator weights.
    D_repeats=1,  # How many times the discriminator is trained per G iteration.
    minibatch_repeats=4,  # Number of minibatches to run before adjusting training parameters.
    reset_opt_for_new_lod=True,  # Reset optimizer internal state (e.g. Adam moments) when new layers are introduced?
    total_kimg=15000,  # Total length of the training, measured in thousands of real images.
    mirror_augment=False,  # Enable mirror augment?
    drange_net=[
        -1, 1
    ],  # Dynamic range used when feeding image data to the networks.
    image_snapshot_ticks=1,  # How often to export image snapshots?
    network_snapshot_ticks=10,  # How often to export network snapshots?
    save_tf_graph=False,  # Include full TensorFlow computation graph in the tfevents file?
    save_weight_histograms=False,  # Include weight histograms in the tfevents file?
    resume_run_id=None,  # Run ID or network pkl to resume training from, None = start from scratch.
    resume_snapshot=None,  # Snapshot index to resume training from, None = autodetect.
    resume_kimg=0.0,  # Assumed training progress at the beginning. Affects reporting and training schedule.
    resume_time=0.0
):  # Assumed wallclock time at the beginning. Affects reporting.

    maintenance_start_time = time.time()
    training_set = dataset.load_dataset(data_dir=config.data_dir,
                                        verbose=True,
                                        **config.dataset)

    # Construct networks.
    with tf.device('/gpu:0'):
        if resume_run_id is not None:
            network_pkl = misc.locate_network_pkl(resume_run_id,
                                                  resume_snapshot)
            print('Loading networks from "%s"...' % network_pkl)
            G, D, Gs, E = misc.load_pkl(network_pkl)
        else:
            print('Constructing networks...')
            G = tfutil.Network('G',
                               num_channels=training_set.shape[0],
                               resolution=training_set.shape[1],
                               label_size=training_set.label_size,
                               **config.G)
            D = tfutil.Network('D',
                               num_channels=training_set.shape[0],
                               resolution=training_set.shape[1],
                               label_size=training_set.label_size,
                               **config.D)

            E = tfutil.Network('E',
                               num_channels=training_set.shape[0],
                               resolution=training_set.shape[1],
                               label_size=training_set.label_size,
                               **config.E)

            Gs = G.clone('Gs')
        Gs_update_op = Gs.setup_as_moving_average_of(G, beta=G_smoothing)
    G.print_layers()
    D.print_layers()
    E.print_layers()

    print('Building TensorFlow graph...')
    with tf.name_scope('Inputs'):
        lod_in = tf.placeholder(tf.float32, name='lod_in', shape=[])
        lrate_in = tf.placeholder(tf.float32, name='lrate_in', shape=[])
        minibatch_in = tf.placeholder(tf.int32, name='minibatch_in', shape=[])
        minibatch_split = minibatch_in // config.num_gpus
        reals, labels = training_set.get_minibatch_tf()
        reals_split = tf.split(reals, config.num_gpus)
        labels_split = tf.split(labels, config.num_gpus)
    G_opt = tfutil.Optimizer(name='TrainG',
                             learning_rate=lrate_in,
                             **config.G_opt)
    D_opt = tfutil.Optimizer(name='TrainD',
                             learning_rate=lrate_in,
                             **config.D_opt)
    E_opt = tfutil.Optimizer(name='TrainE',
                             learning_rate=lrate_in,
                             **config.E_opt)
    for gpu in range(config.num_gpus):
        with tf.name_scope('GPU%d' % gpu), tf.device('/gpu:%d' % gpu):
            G_gpu = G if gpu == 0 else G.clone(G.name + '_shadow')
            D_gpu = D if gpu == 0 else D.clone(D.name + '_shadow')
            E_gpu = E if gpu == 0 else E.clone(E.name + '_shadow')
            lod_assign_ops = [
                tf.assign(G_gpu.find_var('lod'), lod_in),
                tf.assign(D_gpu.find_var('lod'), lod_in),
                tf.assign(E_gpu.find_var('lod'), lod_in)
            ]
            reals_gpu = process_reals(reals_split[gpu], lod_in, mirror_augment,
                                      training_set.dynamic_range, drange_net)
            labels_gpu = labels_split[gpu]
            with tf.name_scope('G_loss'), tf.control_dependencies(
                    lod_assign_ops):
                G_loss = tfutil.call_func_by_name(
                    G=G_gpu,
                    D=D_gpu,
                    E=E_gpu,
                    opt=G_opt,
                    training_set=training_set,
                    minibatch_size=minibatch_split,
                    **config.G_loss)
            with tf.name_scope('D_loss'), tf.control_dependencies(
                    lod_assign_ops):
                D_loss = tfutil.call_func_by_name(
                    G=G_gpu,
                    D=D_gpu,
                    E=E_gpu,
                    opt=D_opt,
                    training_set=training_set,
                    minibatch_size=minibatch_split,
                    reals=reals_gpu,
                    labels=labels_gpu,
                    **config.D_loss)
            with tf.name_scope('E_loss'), tf.control_dependencies(
                    lod_assign_ops):
                E_loss = tfutil.call_func_by_name(
                    G=G_gpu,
                    D=D_gpu,
                    E=E_gpu,
                    opt=E_opt,
                    training_set=training_set,
                    reals=reals_gpu,
                    minibatch_size=minibatch_split,
                    **config.E_loss)
            G_opt.register_gradients(tf.reduce_mean(G_loss), G_gpu.trainables)
            D_opt.register_gradients(tf.reduce_mean(D_loss), D_gpu.trainables)
            E_opt.register_gradients(tf.reduce_mean(E_loss), E_gpu.trainables)
    G_train_op = G_opt.apply_updates()
    D_train_op = D_opt.apply_updates()
    E_train_op = E_opt.apply_updates()

    #sys.exit(0)

    print('Setting up snapshot image grid...')
    grid_size, grid_reals, grid_labels, grid_latents = setup_snapshot_image_grid(
        G, training_set, **config.grid)
    sched = TrainingSchedule(total_kimg * 1000, training_set, **config.sched)
    grid_fakes = Gs.run(grid_latents,
                        grid_labels,
                        minibatch_size=sched.minibatch // config.num_gpus)

    print('Setting up result dir...')
    result_subdir = misc.create_result_subdir(config.result_dir, config.desc)
    misc.save_image_grid(grid_reals,
                         os.path.join(result_subdir, 'reals.png'),
                         drange=training_set.dynamic_range,
                         grid_size=grid_size)
    misc.save_image_grid(grid_fakes,
                         os.path.join(result_subdir, 'fakes%06d.png' % 0),
                         drange=drange_net,
                         grid_size=grid_size)
    summary_log = tf.summary.FileWriter(result_subdir)
    if save_tf_graph:
        summary_log.add_graph(tf.get_default_graph())
    if save_weight_histograms:
        G.setup_weight_histograms()
        D.setup_weight_histograms()

    print('Training...')
    cur_nimg = int(resume_kimg * 1000)
    cur_tick = 0
    tick_start_nimg = cur_nimg
    tick_start_time = time.time()
    train_start_time = tick_start_time - resume_time
    prev_lod = -1.0
    while cur_nimg < total_kimg * 1000:

        # Choose training parameters and configure training ops.
        sched = TrainingSchedule(cur_nimg, training_set, **config.sched)
        training_set.configure(sched.minibatch, sched.lod)
        if reset_opt_for_new_lod:
            if np.floor(sched.lod) != np.floor(prev_lod) or np.ceil(
                    sched.lod) != np.ceil(prev_lod):
                G_opt.reset_optimizer_state()
                D_opt.reset_optimizer_state()
                E_opt.reset_optimizer_state()
        prev_lod = sched.lod

        # Run training ops.
        for repeat in range(minibatch_repeats):
            for _ in range(D_repeats):
                tfutil.run(
                    [D_train_op, Gs_update_op], {
                        lod_in: sched.lod,
                        lrate_in: sched.D_lrate,
                        minibatch_in: sched.minibatch
                    })
                cur_nimg += sched.minibatch
            tfutil.run(
                [G_train_op, E_train_op], {
                    lod_in: sched.lod,
                    lrate_in: sched.G_lrate,
                    minibatch_in: sched.minibatch
                })

        # Perform maintenance tasks once per tick.
        done = (cur_nimg >= total_kimg * 1000)
        if cur_nimg >= tick_start_nimg + sched.tick_kimg * 1000 or done:
            cur_tick += 1
            cur_time = time.time()
            tick_kimg = (cur_nimg - tick_start_nimg) / 1000.0
            tick_start_nimg = cur_nimg
            tick_time = cur_time - tick_start_time
            total_time = cur_time - train_start_time
            maintenance_time = tick_start_time - maintenance_start_time
            maintenance_start_time = cur_time

            # Report progress.
            print(
                'tick %-5d kimg %-8.1f lod %-5.2f minibatch %-4d time %-12s sec/tick %-7.1f sec/kimg %-7.2f maintenance %.1f'
                % (tfutil.autosummary('Progress/tick', cur_tick),
                   tfutil.autosummary('Progress/kimg', cur_nimg / 1000.0),
                   tfutil.autosummary('Progress/lod', sched.lod),
                   tfutil.autosummary('Progress/minibatch', sched.minibatch),
                   misc.format_time(
                       tfutil.autosummary('Timing/total_sec', total_time)),
                   tfutil.autosummary('Timing/sec_per_tick', tick_time),
                   tfutil.autosummary('Timing/sec_per_kimg',
                                      tick_time / tick_kimg),
                   tfutil.autosummary('Timing/maintenance_sec',
                                      maintenance_time)))
            tfutil.autosummary('Timing/total_hours',
                               total_time / (60.0 * 60.0))
            tfutil.autosummary('Timing/total_days',
                               total_time / (24.0 * 60.0 * 60.0))
            tfutil.save_summaries(summary_log, cur_nimg)

            # Save snapshots.
            if cur_tick % image_snapshot_ticks == 0 or done:
                grid_fakes = Gs.run(grid_latents,
                                    grid_labels,
                                    minibatch_size=sched.minibatch //
                                    config.num_gpus)
                misc.save_image_grid(grid_fakes,
                                     os.path.join(
                                         result_subdir,
                                         'fakes%06d.png' % (cur_nimg // 1000)),
                                     drange=drange_net,
                                     grid_size=grid_size)

                misc.save_all_res(training_set.shape[1],
                                  Gs,
                                  result_subdir,
                                  50,
                                  minibatch_size=sched.minibatch //
                                  config.num_gpus)

            if cur_tick % network_snapshot_ticks == 0 or done:
                misc.save_pkl(
                    (G, D, Gs, E),
                    os.path.join(
                        result_subdir,
                        'network-snapshot-%06d.pkl' % (cur_nimg // 1000)))

            # Record start time of the next tick.
            tick_start_time = time.time()

    # Write final results.
    misc.save_pkl((G, D, Gs, E),
                  os.path.join(result_subdir, 'network-final.pkl'))
    summary_log.close()
    open(os.path.join(result_subdir, '_training-done.txt'), 'wt').close()
Exemplo n.º 32
0
    def run(self):
        self.parent.clear_plots()
        if self.data is None:
            length, tMin, tMax, fMin, fMax, lMin, lMax, peakF, peakL, peakT = ('-',) * 10
        else:
            length = len(self.data)
            tMin = format_time(self.extent.tMin, True)
            tMax = format_time(self.extent.tMax, True)
            fMin = format_precision(self.settings, freq=self.extent.fMin,
                                    fancyUnits=True)
            fMax = format_precision(self.settings, freq=self.extent.fMax,
                                    fancyUnits=True)
            lMin = format_precision(self.settings, level=self.extent.lMin,
                                    fancyUnits=True)
            lMax = format_precision(self.settings, level=self.extent.lMax,
                                    fancyUnits=True)
            peak = self.extent.get_peak_flt()
            peakF = format_precision(self.settings, freq=peak[0],
                                     fancyUnits=True)
            peakL = format_precision(self.settings, level=peak[1],
                                     fancyUnits=True)
            peakT = format_time(peak[2], True)

        text = [['Sweeps', '', length],
                ['Extent', '', ''],
                ['', 'Start', tMin],
                ['', 'End', tMax],
                ['', 'Min frequency', fMin],
                ['', 'Max frequency', fMax],
                ['', 'Min level', lMin],
                ['', 'Max level', lMax],
                ['Peak', '', ''],
                ['', 'Level', peakL],
                ['', 'Frequency', peakF],
                ['', 'Time', peakT],
                ]

        table = Table(self.axes, loc='center')
        table.set_gid('table')

        rows = len(text)
        cols = len(text[0])
        fontProperties = FontProperties()
        fontProperties.set_weight('semibold')
        for row in xrange(rows):
            for col in xrange(cols):
                fp = fontProperties if col == 0 else None
                table.add_cell(row, col,
                               text=text[row][col],
                               fontproperties=fp,
                               width=1.0 / cols, height=1.0 / rows)

        if self.settings.grid:
            colour = 'LightGray'
        else:
            colour = 'w'
        set_table_colour(table, colour)

        for i in range(2):
            table.auto_set_column_width(i)

        self.axes.add_table(table)
        self.parent.redraw_plot()

        self.parent.threadPlot = None