Пример #1
0
def custom_NMF(V,
               W=None,
               H=None,
               num_components=10,
               init="random",
               num_iter=20,
               beta=2):
    '''
    parameters:
    V - matrix to be factorised
    init - set initialisation,must be a string in:{"random","custom"}
           if "custom", W,H must be given
    num_iter - number of iterations
    beta - beta of divergence, must be an integer(beta>=0)
    
    return:
    updated W,H
    '''
    np.seterr(all='ignore')
    n_samples = V.shape[0]
    n_features = V.shape[1]
    #initialisation
    if init == "random":
        W = np.random.random((n_samples, num_components))
        H = np.random.random((num_components, n_features))
    if init == "custom":
        if W.shape != (n_samples, num_components) or H.shape != (
                num_components, n_features):
            print("error!W or H does not have a right shape!")
    #multiplicative update
    #visulize progress
    pro = IntProgress(max=num_iter)
    pro.description = "updating"
    display(pro)
    start = time()
    b_div = []
    for ite in range(num_iter):
        #update H
        H = update_H(W, H, V, beta)
        #update W
        W = update_W(W, H, V, beta)
        #limit W and H to avoid overflow
        H = np.clip(H, 10**-150, None)
        W = np.clip(W, 10**-150, None)
        #show error
        #print("divergence of iteration %d: %.6f"%(ite,beta_divergence(V,np.dot(W,H),beta)))
        b_div.append(beta_divergence(V, np.dot(W, H), beta))
        pro.value += 1
    end = time()
    pro.description = "done(%.1fs)" % (end - start)
    #plot the curve of divergence
    iterations = np.linspace(1, num_iter, num_iter)
    plt.figure(figsize=(6, 4))
    plt.plot(iterations, b_div)
    plt.title("beta=%d" % (beta))
    plt.xlabel("iterations")
    plt.ylabel("beta divergence")
    plt.show()

    return W, H
Пример #2
0
def perceptron_pass2(lines, classifier):
    n = len(lines)
    count = 0
    p = IntProgress(max=n)
    display(p)
    c = classifier
    random.shuffle(lines)
    for line in lines:
        count += 1
        label, feature = get_label_feature(line)
        value = get_value(classifier, feature)
        if label * value > 0:
            pass
        else:
            for word in feature:
                if None == classifier.get(word):
                    classifier[word] = label * feature[word]
                    c[word] = label * feature[word] * (n - count) / n
                else:
                    classifier[word] += label * feature[word]
                    c[word] += label * feature[word] * (n - count) / n
        p.value = count
        p.description = "{}%".format(round(100 * p.value / p.max, 2))
    p.description = "Done"
    return c
Пример #3
0
def progress_bar(job):
    '''Display a status bar showing how many tasks are completed'''
    
    status=get_status(job)
    f = IntProgress(min=0, max=status['Total'], bar_style='success')
    f.value = status['Total'] - status['Pending'] - status['Running']
    f.description =  "{:1.0f} tasks left ".format(status['Pending']+status['Running'])
    display(f)
    
    while f.value <  status['Total']:
        status=get_status(job)
        f.value = status['Total'] - status['Pending'] - status['Running']
        f.description =  "{:1.0f} tasks left ".format(status['Pending']+status['Running'])
        if status['Failed'] > 0:
            f.bar_style='warning'
        sleep(1)
Пример #4
0
def progress_bar(dirs, path, extension, data):
    progress = IntProgress()
    progress.max = len(mydirs)
    progress.description = '(Init)'
    display(progress)
    for mydir in dirs:
        os.chdir(path + mydir)
        all_filenames = [i for i in glob.glob('*.{}'.format(extension))]
        for file in all_filenames:
            df_temp = pd.read_csv(file, encoding='utf8')
            try:
                data = pd.merge(data, df_temp, how='outer', on='SEQN')
            except:
                print(mypath + mydir + '/' + file)
        progress.value += 1
        progress.description = mydir
    progress.description = '(Done)'
Пример #5
0
    def create_movie(self,
                     path,
                     size,
                     image_format='png',
                     animation_frames=list(),
                     quality=100,
                     samples_per_pixel=1,
                     start_frame=0,
                     end_frame=0,
                     interpupillary_distance=0.0,
                     exportIntermediateFrames=True):

        from ipywidgets import IntProgress

        application_params = self._client.get_application_parameters()
        renderer_params = self._client.get_renderer()

        old_image_stream_fps = application_params['image_stream_fps']
        old_viewport_size = application_params['viewport']
        old_samples_per_pixel = renderer_params['samples_per_pixel']
        old_max_accum_frames = renderer_params['max_accum_frames']
        self._client.set_renderer(samples_per_pixel=1,
                                  max_accum_frames=samples_per_pixel)
        self._client.set_application_parameters(viewport=size)
        self._client.set_application_parameters(image_stream_fps=0)

        progress_widget = IntProgress(description='In progress...',
                                      min=0,
                                      max=100,
                                      value=0)
        display(progress_widget)

        self.export_frames(path=path,
                           animation_frames=animation_frames,
                           start_frame=start_frame,
                           end_frame=end_frame,
                           size=size,
                           samples_per_pixel=samples_per_pixel,
                           quality=quality,
                           interpupillary_distance=interpupillary_distance,
                           exportIntermediateFrames=exportIntermediateFrames)

        done = False
        while not done:
            import time
            time.sleep(1)
            progress = self.get_export_frames_progress()['progress']
            progress_widget.value = progress * 100
            done = self.get_export_frames_progress()['done']

        self._client.set_application_parameters(
            image_stream_fps=old_image_stream_fps, viewport=old_viewport_size)
        self._client.set_renderer(samples_per_pixel=old_samples_per_pixel,
                                  max_accum_frames=old_max_accum_frames)

        progress_widget.description = 'Done'
        progress_widget.value = 100
Пример #6
0
    def create_snapshot(self, size, path, samples_per_pixel, export_intermediate_frames=False):
        """
        Create a snapshot of the current frame

        :size: Frame buffer size
        :path: Full path of the snapshot image
        :samples_per_pixel: Samples per pixel
        :export_intermediate_frames: If True, intermediate samples are stored to disk. Otherwise, only
        the final accumulation is exported
        """
        application_params = self._client.get_application_parameters()
        renderer_params = self._client.get_renderer()
        old_image_stream_fps = application_params['image_stream_fps']
        old_viewport_size = application_params['viewport']
        old_samples_per_pixel = renderer_params['samples_per_pixel']
        old_max_accum_frames = renderer_params['max_accum_frames']
        old_smoothed_key_frames = copy.deepcopy(self._smoothed_key_frames)

        self._client.set_renderer(samples_per_pixel=1, max_accum_frames=samples_per_pixel)
        self._client.set_application_parameters(viewport=size)
        self._client.set_application_parameters(image_stream_fps=0)

        control_points = [self.get_camera()]
        current_animation_frame = int(self._client.get_animation_parameters()['current'])
        animation_frames = [current_animation_frame]

        self.build_camera_path(
            control_points=control_points, nb_steps_between_control_points=1, smoothing_size=1)

        progress_widget = IntProgress(description='In progress...', min=0, max=100, value=0)
        display(progress_widget)

        base_dir = os.path.dirname(path)
        self.export_frames(
            path=base_dir, animation_frames=animation_frames, size=size,
            samples_per_pixel=samples_per_pixel,
            export_intermediate_frames=export_intermediate_frames)

        done = False
        while not done:
            time.sleep(1)
            progress = self.get_export_frames_progress()['progress']
            progress_widget.value = progress * 100
            done = self.get_export_frames_progress()['done']

        progress_widget.description = 'Done'
        progress_widget.value = 100
        frame_path = base_dir + '/00000.png'
        if os.path.exists(frame_path):
            os.rename(frame_path, path)

        self._client.set_application_parameters(image_stream_fps=old_image_stream_fps,
                                                viewport=old_viewport_size)
        self._client.set_renderer(samples_per_pixel=old_samples_per_pixel,
                                  max_accum_frames=old_max_accum_frames)
        self._smoothed_key_frames = copy.deepcopy(old_smoothed_key_frames)
Пример #7
0
    def create_movie(
            self, path, size, animation_frames=list(), quality=100, samples_per_pixel=1,
            start_frame=0, end_frame=0, interpupillary_distance=0.0, exportIntermediateFrames=True):
        """
        Create and export a set of PNG frames for later movie generation

        :path: Full path of the snapshot folder
        :size: Frame buffer size
        :animation_frames: Optional list of animation frames
        :quality: PNG quality
        :samples_per_pixel: Samples per pixel
        :start_frame: Start frame to export in the provided sequence
        :end_frame: Last frame to export in the provided sequence
        :interpupillary_distance: Interpupillary distance for stereo rendering. If set to 0, stereo
        is disabled
        :exportIntermediateFrames: If True, intermediate samples are stored to disk. Otherwise, only
        the final accumulation is exported
        """
        application_params = self._client.get_application_parameters()
        renderer_params = self._client.get_renderer()

        old_image_stream_fps = application_params['image_stream_fps']
        old_viewport_size = application_params['viewport']
        old_samples_per_pixel = renderer_params['samples_per_pixel']
        old_max_accum_frames = renderer_params['max_accum_frames']
        self._client.set_renderer(samples_per_pixel=1, max_accum_frames=samples_per_pixel)
        self._client.set_application_parameters(viewport=size)
        self._client.set_application_parameters(image_stream_fps=0)

        progress_widget = IntProgress(description='In progress...', min=0, max=100, value=0)
        display(progress_widget)

        self.export_frames(
            path=path, animation_frames=animation_frames, start_frame=start_frame,
            end_frame=end_frame, size=size, samples_per_pixel=samples_per_pixel, quality=quality,
            interpupillary_distance=interpupillary_distance,
            exportIntermediateFrames=exportIntermediateFrames)

        done = False
        while not done:
            time.sleep(1)
            progress = self.get_export_frames_progress()['progress']
            progress_widget.value = progress * 100
            done = self.get_export_frames_progress()['done']

        self._client.set_application_parameters(image_stream_fps=old_image_stream_fps,
                                                viewport=old_viewport_size)
        self._client.set_renderer(samples_per_pixel=old_samples_per_pixel,
                                  max_accum_frames=old_max_accum_frames)

        progress_widget.description = 'Done'
        progress_widget.value = 100
Пример #8
0
def progressify(iterable, n):
    start_time = time.time()
    progress = IntProgress(min=0, max=n, layout=Layout(width='100%'))
    text = Label(layout=Layout(width='100%'))
    display(progress)
    display(text)
    for it in iterable:
        yield it
        progress.value += 1
        elapsed_time = time.time() - start_time
        percent = progress.value * 100.0 / n
        progress.description = '%.1f%% (%s / %s)' % (percent, progress.value,
                                                     n)
        text.value = 'elapsed %s' % datetime.timedelta(seconds=elapsed_time)
    progress.bar_style = 'success'
Пример #9
0
def from_geodataframe_to_map(portal: Portal, gdf: GeoDataFrame, data_name: str, map_title: str, layer_name: str):
    progress = IntProgress()
    progress.max = 100
    progress.value = 0
    progress.description = '上传文件:'
    display(progress)

    def refresh_progress(read, total):
        progress.value = read

    data_id = portal.upload_dataframe_as_json(data_name, gdf, callback=refresh_progress)
    layer = portal.prepare_geojson_layer(data_id, layer_name)
    map_id = portal.create_map([layer], 3857, map_title)
    mr = portal.get_map(map_id)
    pm = PortalThumbnail(mr)
    display(pm)
    return map_id
Пример #10
0
def in_progress(seq, msg="Progress: [%(processed)d / %(total)d]", length=None):
    """ Iterate over sequence, yielding item with progress widget displayed.
        This is useful if you need to precess sequence of items with some
        time consuming operations

        .. note::

            This works only in Jupyter Notebook

        .. note::

            This function requires *ipywidgets* package to be installed

        :param seq: sequence to iterate on.
        :param str msg: (optional) message template to display.
                        available to use 'processed' and 'total' integer vars,
                        where 'processed' is number of items processed and
                        'total' is total number of items in seq.
        :param int length: (optional) if seq is generator, or it is not
                           possible to apply 'len(seq)' function to 'seq',
                           then this argument is required and it's value will
                           be used as total number of items in seq.

        Example example::

            import time
            for i in in_progress(range(10)):
                time.sleep(1)
    """
    from IPython.display import display
    from ipywidgets import IntProgress

    if length is None:
        length = len(seq)

    progress = IntProgress(value=0, min=0, max=length,
                           description=msg % {'processed': 0,
                                              'total': length})
    display(progress)

    for i, item in enumerate(seq, 1):
        progress.value = i
        progress.description = msg % {'processed': i, 'total': length}
        yield item

    progress.close()
Пример #11
0
def perceptron_pass1(lines, classifier):
    p = IntProgress(max=len(lines))
    display(p)
    random.shuffle(lines)
    for line in lines:
        label, feature = get_label_feature(line)
        value = get_value(classifier, feature)
        if label * value > 0:
            pass
        else:
            for word in feature:
                if None == classifier.get(word):
                    classifier[word] = label * feature[word]
                else:
                    classifier[word] += label * feature[word]
        p.value += 1
        p.description = "{}%".format(round(100 * (p.value / p.max), 1))
    return classifier
Пример #12
0
def extract_features(model, filenames, path_to_images, display_bar=True):
    '''
    Extracts output from model on list of filenames.
    
    Parameters
    ----------
    model : Keras model.
    filenames : list
        List of filenames to process.
    path_to_images : path to folder with images
    
    Returns
    -------
    features : ndarray
        Extracted features.
    '''
    if display_bar:
        bar = IntProgress(value=1,
                          min=1,
                          max=len(filenames),
                          step=1,
                          description='Initializing...')
        display(bar)

    features = np.zeros((len(filenames), 4096), dtype=np.float)
    for i in range(len(filenames)):
        img_path = os.path.join(path_to_images, filenames[i])
        img = image.load_img(img_path, target_size=(224, 224))
        x = image.img_to_array(img)
        x = np.expand_dims(x, axis=0)
        x = preprocess_input(x)
        features[i] = feature_extractor.predict(x).squeeze()
        if display_bar:
            bar.value = i + 1
            bar.description = '[{:>{tab}} / {}]'.format(i + 1,
                                                        bar.max,
                                                        tab=len(str(bar.max)))
    if display_bar:
        bar.bar_style = 'success'
    return features
Пример #13
0
    def create_snapshot(self,
                        size,
                        path,
                        samples_per_pixel,
                        exportIntermediateFrames=True):

        from ipywidgets import IntProgress
        import os
        import copy

        application_params = self._client.get_application_parameters()
        renderer_params = self._client.get_renderer()
        old_image_stream_fps = application_params['image_stream_fps']
        old_viewport_size = application_params['viewport']
        old_samples_per_pixel = renderer_params['samples_per_pixel']
        old_max_accum_frames = renderer_params['max_accum_frames']
        old_smoothed_key_frames = copy.deepcopy(self._smoothed_key_frames)

        self._client.set_renderer(samples_per_pixel=1,
                                  max_accum_frames=samples_per_pixel)
        self._client.set_application_parameters(viewport=size)
        self._client.set_application_parameters(image_stream_fps=0)

        control_points = [self.get_camera()]
        current_animation_frame = int(
            self._client.get_animation_parameters()['current'])
        animation_frames = [current_animation_frame]

        self.build_camera_path(control_points=control_points,
                               nb_steps_between_control_points=1,
                               smoothing_size=1)

        progress_widget = IntProgress(description='In progress...',
                                      min=0,
                                      max=100,
                                      value=0)
        display(progress_widget)

        base_dir = os.path.dirname(path)
        self.export_frames(path=base_dir,
                           animation_frames=animation_frames,
                           size=size,
                           samples_per_pixel=samples_per_pixel,
                           exportIntermediateFrames=exportIntermediateFrames)

        done = False
        while not done:
            import time
            time.sleep(1)
            progress = self.get_export_frames_progress()['progress']
            progress_widget.value = progress * 100
            done = self.get_export_frames_progress()['done']

        progress_widget.description = 'Done'
        progress_widget.value = 100
        frame_path = base_dir + '/00000.png'
        if os.path.exists(frame_path):
            os.rename(frame_path, path)

        self._client.set_application_parameters(
            image_stream_fps=old_image_stream_fps, viewport=old_viewport_size)
        self._client.set_renderer(samples_per_pixel=old_samples_per_pixel,
                                  max_accum_frames=old_max_accum_frames)
        self._smoothed_key_frames = copy.deepcopy(old_smoothed_key_frames)
Пример #14
0
def in_progress(seq,
                msg="Progress: [%(processed)d / %(total)d]",
                length=None,
                close=True):
    """ Iterate over sequence, yielding item with progress widget displayed.
        This is useful if you need to precess sequence of items with some
        time consuming operations

        .. note::

            This works only in Jupyter Notebook

        .. note::

            This function requires *ipywidgets* package to be installed

        :param seq: sequence to iterate on.
        :param str msg: (optional) message template to display.
                        Following variables could be used in this template:
                            - processed
                            - total
                            - time_total
                            - time_per_item
        :param int length: (optional) if seq is generator, or it is not
                           possible to apply 'len(seq)' function to 'seq',
                           then this argument is required and it's value will
                           be used as total number of items in seq.

        Example example::

            import time
            for i in in_progress(range(10)):
                time.sleep(1)
    """
    from IPython.display import display
    from ipywidgets import IntProgress
    import time

    if length is None:
        length = len(seq)

    start_time = time.time()

    progress = IntProgress(value=0,
                           min=0,
                           max=length,
                           description=msg % {
                               'processed': 0,
                               'total': length,
                               'time_total': 0.0,
                               'time_per_item': 0.0,
                               'time_remaining': 0.0,
                           })
    display(progress)

    for i, item in enumerate(seq, 1):
        progress.value = i

        # i_start_time = time.time()

        yield item  # Do the job

        i_end_time = time.time()

        progress.description = msg % {
            'processed': i,
            'total': length,
            'time_total': i_end_time - start_time,
            'time_per_item': (i_end_time - start_time) / i,
            'time_remaining': ((i_end_time - start_time) / i) * (length - i),
        }

    if close:
        progress.close()
Пример #15
0
    def train(self,
              batch_size=64,
              learning_rate=1e-3,
              num_epochs=5,
              max_num=-1,
              best_path='keyboard_model_best.tar',
              current_path='keyboard_model_latest.tar',
              decay_every=10,
              save_model=True,
              dirs=[0]):

        model = self.model
        criterion = nn.MSELoss()
        optimizer = optim.Adam(self.model.parameters(), lr=learning_rate)
        scheduler = optim.lr_scheduler.StepLR(optimizer,
                                              step_size=decay_every,
                                              gamma=0.05)

        since = time.time()

        best_model_wts = copy.deepcopy(model.state_dict())
        best_loss = None

        for epoch in range(num_epochs):

            print('Epoch {}/{}'.format(epoch + 1, num_epochs))

            # Each epoch has a training and validation phase
            for phase in ['train', 'val']:
                if phase == 'train':
                    scheduler.step()
                    model.train()  # Set model to training mode
                else:
                    model.eval()  # Set model to evaluate mode

                running_loss = 0.0
                # Iterate over data.

                max_num_for_this_epoch = max_num if phase == 'train' else -1

                total = dataset.get_num_of_data(
                    phase) if max_num == -1 else max_num

                bar = IntProgress(max=total)

                display(bar)

                for inputs, labels in dataset.data_batch(
                        type=phase,
                        batch_size=batch_size,
                        max_num=max_num_for_this_epoch,
                        dirs=dirs):

                    inputs = torch.Tensor(inputs)
                    labels = torch.Tensor(labels)

                    inputs = inputs.to(device)
                    labels = labels.to(device)

                    # zero the parameter gradients
                    optimizer.zero_grad()

                    # forward
                    outputs = model(inputs)
                    labels = torch.reshape(labels, [-1, 8])
                    loss = criterion(outputs, labels)

                    # backward + optimize only if in training phase
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()

                    # statistics
                    running_loss += loss.item() * batch_size

                    # free unoccupied memory
                    if torch.cuda.is_available():
                        torch.cuda.empty_cache()
                    else:
                        torch.cpu.empty_cache()

                    bar.value += batch_size
                    bar.description = f'{bar.value} / {total}'

                bar.close()

                epoch_loss = running_loss / dataset.get_num_of_data(phase)

                print('{} Loss: {:.4f}'.format(phase, epoch_loss))

            # deep copy the model
            if phase == 'val' and (best_loss == None
                                   or epoch_loss < best_loss):
                best_loss = epoch_loss
                best_model_wts = copy.deepcopy(model.state_dict())
                torch.save(model.state_dict(), best_path)
                print(f'The best model has been saved to {best_path} ...')

            torch.save(model.state_dict(), current_path)
            print(f'Current mode has been saved to {current_path} ...')

            print()

        time_elapsed = time.time() - since
        print('Training complete in {:.0f}m {:.0f}s'.format(
            time_elapsed // 60, time_elapsed % 60))
        print('Best val loss: {:4f}'.format(best_loss))

        # load best model weights
        model.load_state_dict(best_model_wts)
        self.model = model
Пример #16
0
    def status_printer(file, total=None, desc=None):
        """
        Manage the printing of an IPython/Jupyter Notebook progress bar widget.
        """
        # Fallback to text bar if there's no total
        # DEPRECATED: replaced with an 'info' style bar
        # if not total:
        #    return super(tqdm_notebook, tqdm_notebook).status_printer(file)

        fp = file
        if not getattr(fp, 'flush', False):  # pragma: no cover
            fp.flush = lambda: None

        # Prepare IPython progress bar
        if total:
            pbar = IntProgress(min=0, max=total)
        else:  # No total? Show info style bar with no progress tqdm status
            pbar = IntProgress(min=0, max=1)
            pbar.value = 1
            pbar.bar_style = 'info'
        if desc:
            pbar.description = desc
        # Prepare status text
        ptext = HTML()
        # Only way to place text to the right of the bar is to use a container
        container = HBox(children=[pbar, ptext])
        display(container)

        def print_status(s='', close=False, bar_style=None):
            # Note: contrary to native tqdm, s='' does NOT clear bar
            # goal is to keep all infos if error happens so user knows
            # at which iteration the loop failed.

            # Clear previous output (really necessary?)
            # clear_output(wait=1)

            # Get current iteration value from format_meter string
            if total:
                n = None
                if s:
                    npos = s.find(r'/|/')  # cause we use bar_format=r'{n}|...'
                    # Check that n can be found in s (else n > total)
                    if npos >= 0:
                        n = int(s[:npos])  # get n from string
                        s = s[npos + 3:]  # remove from string

                        # Update bar with current n value
                        if n is not None:
                            pbar.value = n

            # Print stats
            if s:  # never clear the bar (signal: s='')
                s = s.replace('||', '')  # remove inesthetical pipes
                s = escape(s)  # html escape special characters (like '?')
                ptext.value = s

            # Change bar style
            if bar_style:
                # Hack-ish way to avoid the danger bar_style being overriden by
                # success because the bar gets closed after the error...
                if not (pbar.bar_style == 'danger' and bar_style == 'success'):
                    pbar.bar_style = bar_style

            # Special signal to close the bar
            if close and pbar.bar_style != 'danger':  # hide only if no error
                container.visible = False

        return print_status
Пример #17
0
def train(FLAG):
    print("Reading dataset...")
    if FLAG.dataset == 'CIFAR-10':
        train_data = CIFAR10(train=True)
        test_data = CIFAR10(train=False)
    elif FLAG.dataset == 'CIFAR-100':
        train_data = CIFAR100(train=True)
        test_data = CIFAR100(train=False)
    else:
        raise ValueError("dataset should be either CIFAR-10 or CIFAR-100.")

    Xtrain, Ytrain = train_data.train_data, train_data.train_labels
    Xtest, Ytest = test_data.test_data, test_data.test_labels

    print("Build VGG16 models...")
    vgg16 = VGG16(FLAG.init_from, prof_type=FLAG.prof_type)

    # build model using  dp
    dp = [(i + 1) * 0.05 for i in range(1, 20)]
    vgg16.build(dp=dp)

    # define tasks
    tasks = ['100', '50']
    print(tasks)

    saver = tf.train.Saver(tf.global_variables(), max_to_keep=len(tasks))

    checkpoint_path = os.path.join(FLAG.save_dir, 'model.ckpt')

    tvars_trainable = tf.trainable_variables()
    for rm in vgg16.gamma_var:
        tvars_trainable.remove(rm)
        print('%s is not trainable.' % rm)

    # useful function
    def initialize_uninitialized(sess):
        global_vars = tf.global_variables()
        is_not_initialized = sess.run(
            [tf.is_variable_initialized(var) for var in global_vars])
        not_initialized_vars = [
            v for (v, f) in zip(global_vars, is_not_initialized) if not f
        ]
        if len(not_initialized_vars):
            sess.run(tf.variables_initializer(not_initialized_vars))

    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())

        # hyper parameters
        learning_rate = 2e-4
        batch_size = 32
        alpha = 0.5
        early_stop_patience = 4
        min_delta = 0.0001

        # optimizer
        # opt = tf.train.MomentumOptimizer(learning_rate=learning_rate, momentum=0.9)
        opt = tf.train.AdamOptimizer(learning_rate=learning_rate)
        # recorder
        epoch_counter = 0

        # tensorboard writer
        writer = tf.summary.FileWriter(FLAG.log_dir, sess.graph)

        # progress bar
        ptrain = IntProgress()
        pval = IntProgress()
        display(ptrain)
        display(pval)
        ptrain.max = int(Xtrain.shape[0] / batch_size)
        pval.max = int(Xtest.shape[0] / batch_size)

        # initial task
        obj = vgg16.loss_dict[tasks[0]]

        while (len(tasks)):

            # acquire a new task
            cur_task = tasks[0]
            tasks = tasks[1:]
            new_obj = vgg16.loss_dict[cur_task]

            # just finished a task
            if epoch_counter > 0:
                # save models
                saver.save(sess, checkpoint_path, global_step=epoch_counter)

                # task-wise loss aggregation
                # obj = tf.add(tf.multiply(1-alpha,obj), tf.multiply(alpha,new_obj))
                obj = tf.add(obj, new_obj)
            # optimizer
            train_op = opt.minimize(obj, var_list=tvars_trainable)

            # re-initialize
            initialize_uninitialized(sess)

            # reset due to adding a new task
            patience_counter = 0
            current_best_val_loss = 100000  # a large number

            # optimize when the aggregated obj
            while (patience_counter < early_stop_patience):
                stime = time.time()
                bar_train = Bar(
                    'Training',
                    max=int(Xtrain.shape[0] / batch_size),
                    suffix='%(index)d/%(max)d - %(percent).1f%% - %(eta)ds')
                bar_val = Bar(
                    'Validation',
                    max=int(Xtest.shape[0] / batch_size),
                    suffix='%(index)d/%(max)d - %(percent).1f%% - %(eta)ds')

                # training an epoch
                for i in range(int(Xtrain.shape[0] / batch_size)):
                    st = i * batch_size
                    ed = (i + 1) * batch_size
                    sess.run([train_op],
                             feed_dict={
                                 vgg16.x: Xtrain[st:ed, :, :, :],
                                 vgg16.y: Ytrain[st:ed, :]
                             })
                    ptrain.value += 1
                    ptrain.description = "Training %s/%s" % (i, ptrain.max)
                    bar_train.next()

                # validation
                val_loss = 0
                val_accu = 0
                for i in range(int(Xtest.shape[0] / 200)):
                    st = i * 200
                    ed = (i + 1) * 200
                    loss, accu, epoch_summary = sess.run(
                        [obj, vgg16.accu_dict[cur_task], vgg16.summary_op],
                        feed_dict={
                            vgg16.x: Xtest[st:ed, :],
                            vgg16.y: Ytest[st:ed, :]
                        })
                    val_loss += loss
                    val_accu += accu
                    pval.value += 1
                    pval.description = "Testing %s/%s" % (i, pval.value)
                val_loss = val_loss / pval.value
                val_accu = val_accu / pval.value

                # early stopping check
                if (current_best_val_loss - val_loss) > min_delta:
                    current_best_val_loss = val_loss
                    patience_counter = 0
                else:
                    patience_counter += 1

                # shuffle Xtrain and Ytrain in the next epoch
                idx = np.random.permutation(Xtrain.shape[0])
                Xtrain, Ytrain = Xtrain[idx, :, :, :], Ytrain[idx, :]

                # epoch end
                writer.add_summary(epoch_summary, epoch_counter)
                epoch_counter += 1

                ptrain.value = 0
                pval.value = 0
                bar_train.finish()
                bar_val.finish()

                print(
                    "Epoch %s (%s), %s sec >> obj loss: %.4f, task at %s: %.4f"
                    % (epoch_counter, patience_counter,
                       round(time.time() - stime,
                             2), val_loss, cur_task, val_accu))
        saver.save(sess, checkpoint_path, global_step=epoch_counter)

        writer.close()
Пример #18
0
    def status_printer(file, total=None, desc=None):
        """
        Manage the printing of an IPython/Jupyter Notebook progress bar widget.
        """
        # Fallback to text bar if there's no total
        # DEPRECATED: replaced with an 'info' style bar
        # if not total:
        #    return super(tqdm_notebook, tqdm_notebook).status_printer(file)

        fp = file
        if not getattr(fp, 'flush', False):  # pragma: no cover
            fp.flush = lambda: None

        # Prepare IPython progress bar
        if total:
            pbar = IntProgress(min=0, max=total)
        else:  # No total? Show info style bar with no progress tqdm status
            pbar = IntProgress(min=0, max=1)
            pbar.value = 1
            pbar.bar_style = 'info'
        if desc:
            pbar.description = desc
        # Prepare status text
        ptext = HTML()
        # Only way to place text to the right of the bar is to use a container
        container = HBox(children=[pbar, ptext])
        display(container)

        def print_status(s='', close=False, bar_style=None):
            # Note: contrary to native tqdm, s='' does NOT clear bar
            # goal is to keep all infos if error happens so user knows
            # at which iteration the loop failed.

            # Clear previous output (really necessary?)
            # clear_output(wait=1)

            # Get current iteration value from format_meter string
            if total:
                n = None
                if s:
                    npos = s.find(r'/|/')  # cause we use bar_format=r'{n}|...'
                    # Check that n can be found in s (else n > total)
                    if npos >= 0:
                        n = int(s[:npos])  # get n from string
                        s = s[npos + 3:]  # remove from string

                        # Update bar with current n value
                        if n is not None:
                            pbar.value = n

            # Print stats
            if s:  # never clear the bar (signal: s='')
                s = s.replace('||', '')  # remove inesthetical pipes
                s = escape(s)  # html escape special characters (like '?')
                ptext.value = s

            # Change bar style
            if bar_style:
                # Hack-ish way to avoid the danger bar_style being overriden by
                # success because the bar gets closed after the error...
                if not (pbar.bar_style == 'danger' and bar_style == 'success'):
                    pbar.bar_style = bar_style

            # Special signal to close the bar
            if close and pbar.bar_style != 'danger':  # hide only if no error
                container.visible = False

        return print_status
Пример #19
0
    def train(self,
              phase=['train', 'val'],
              color='black',
              learning_rate=1e-3,
              weight_lambda=0.0005,
              num_epoch=5,
              max_num=-1,
              best_path='model_best.tar',
              current_path='model_latest.tar',
              tsb_writer=None,
              tag='',
              decay_every=10,
              save_model=True):
        model = self.model
        criterion = nn.MSELoss()
        optimizer = optim.Adam(self.model.parameters(),
                               lr=learning_rate,
                               weight_decay=weight_lambda)
        scheduler = optim.lr_scheduler.StepLR(optimizer,
                                              step_size=decay_every,
                                              gamma=0.05)
        since = time.time()
        best_model = None
        best_loss = None

        best_path = time.strftime('[%Y%m%d]%H-%M-%S') + best_path
        current_path = time.strftime('[%Y%m%d]%H-%M-%S') + current_path

        print(f'The best model will be saved to {best_path} ...')
        print(f'Thhe latest model will be saved to {current_path} ...')

        for epoch in range(num_epoch):

            print('Epoch {}/{}'.format(epoch + 1, num_epoch), end='')
            self.epoch_total += 1
            _loss = dict()
            _diff = dict()

            for phase in phase:
                if phase == 'train':
                    scheduler.step()
                    model.train()
                else:
                    model.eval()

                running_loss = 0.0
                running_diff = 0.0
                total = dataset.get_lstm_data_num(
                    phase, color) if max_num == -1 else max_num

                bar = IntProgress(max=total)
                display(bar)

                for i, (inputs, labels) in enumerate(
                        dataset.lstm_data_batch(type=phase,
                                                color=color,
                                                max_num=total,
                                                need_bar=False)):

                    _labels = labels
                    inputs = torch.Tensor(inputs)
                    labels = torch.Tensor(np.array([labels]) / 63.5 - 1)
                    inputs = inputs.to(device)
                    labels = labels.to(device)

                    optimizer.zero_grad()

                    outputs = model(inputs)
                    labels = torch.reshape(labels, [1])
                    loss = criterion(outputs, labels)

                    if phase == 'train':
                        loss.backward()
                        optimizer.step()

                    running_loss += loss.item()
                    running_diff += np.abs(
                        (outputs.cpu().detach().numpy()[0] + 1) * 63.5 -
                        _labels)

                    if torch.cuda.is_available():
                        torch.cuda.empty_cache()

                    bar.value += 1
                    if i % 32 == 0:
                        bar.description = f'{bar.value} / {total}'

                bar.close()

                epoch_loss = running_loss / total
                epoch_diff = running_diff / total

                if epoch % 5 == 0:
                    print('{} Loss: {:.4f}, L1 Diff: {:.4f}'.format(
                        phase, epoch_loss, epoch_diff))

                _loss[phase] = epoch_loss
                _diff[phase] = epoch_diff

                if phase == 'val' and (best_loss == None
                                       or epoch_loss < best_loss):
                    best_loss = epoch_loss
                    best_model = copy.deepcopy(model.state_dict())
                    if save_model:
                        torch.save(model.state_dict(), best_path)

                if save_model:
                    torch.save(model.state_dict(), current_path)

            if tsb_writer and 'val' in phase and 'train' in phase:
                tsb_writer.add_scalars(f'{tag}/Loss', {
                    'val': _loss['val'],
                    'train': _loss['train']
                }, self.epoch_total)
                tsb_writer.add_scalars(f'{tag}/L1 Diff', {
                    'val': _diff['val'],
                    'train': _diff['train']
                }, self.epoch_total)

        time_elapsed = time.time() - since
        print('Training complete in {:.0f}m {:.0f}s'.format(
            time_elapsed // 60, time_elapsed % 60))
        print('Best val loss: {:4f}'.format(best_loss))

        self.model = model
Пример #20
0
def in_progress(seq, msg="Progress: [%(processed)d / %(total)d]",
                length=None, close=True):
    """ Iterate over sequence, yielding item with progress widget displayed.
        This is useful if you need to precess sequence of items with some
        time consuming operations

        .. note::

            This works only in Jupyter Notebook

        .. note::

            This function requires *ipywidgets* package to be installed

        :param seq: sequence to iterate on.
        :param str msg: (optional) message template to display.
                        Following variables could be used in this template:
                            - processed
                            - total
                            - time_total
                            - time_per_item
        :param int length: (optional) if seq is generator, or it is not
                           possible to apply 'len(seq)' function to 'seq',
                           then this argument is required and it's value will
                           be used as total number of items in seq.

        Example example::

            import time
            for i in in_progress(range(10)):
                time.sleep(1)
    """
    from IPython.display import display
    from ipywidgets import IntProgress
    import time

    if length is None:
        length = len(seq)

    start_time = time.time()

    progress = IntProgress(
        value=0, min=0, max=length, description=msg % {
            'processed': 0,
            'total': length,
            'time_total': 0.0,
            'time_per_item': 0.0,
            'time_remaining': 0.0,
        }
    )
    display(progress)

    for i, item in enumerate(seq, 1):
        progress.value = i

        # i_start_time = time.time()

        yield item  # Do the job

        i_end_time = time.time()

        progress.description = msg % {
            'processed': i,
            'total': length,
            'time_total': i_end_time - start_time,
            'time_per_item': (i_end_time - start_time) / i,
            'time_remaining': ((i_end_time - start_time) / i) * (length - i),
        }

    if close:
        progress.close()
Пример #21
0
    def interpolate(self,
                    tile_size=256,
                    n_workers=1,
                    threads_per_worker=8,
                    memory_limit='14GB',
                    progressBar=None):
        """
        Interpolates the data of a time series object using
        the method or methods provided
        :param method: list of interpolation methods
        """
        if self.mask is None:
            pass

        if self.isNotebook is True:
            # Set up progress bar
            _items = len(self.interpolation_methods.value)
            # For every interpol method selected by the user
            _item = 0
            progress_bar = IntProgress(
                value=0,
                min=0,
                max=_items,
                step=1,
                description='',
                bar_style='',  # 'success', 'info', 'warning', 'danger' or ''
                orientation='horizontal',
                style={'description_width': 'initial'},
                layout={'width': '75%'})
            display(progress_bar)
            progress_bar.value = _item

            # Get temp dataset to perform the interpolation
            data_var = self.data_vars.value

            # Interpolation methods
            interpolation_methods = self.interpolation_methods.value

        else:
            data_var = self.selected_data_var
            interpolation_methods = [self.selected_interpolation_method]

        tmp_ds = getattr(self.ts.data, data_var).copy(deep=True)

        # Store original data type
        dtype = tmp_ds.data.dtype

        # Get fill value and idx
        fill_value = tmp_ds.attrs['nodatavals'][0]
        mask_fill_value = (tmp_ds == fill_value)
        mask_fill_value = (mask_fill_value * fill_value).astype(dtype)
        #idx_no_data = np.where(tmp_ds.data == fill_value)

        # Apply mask
        tmp_ds *= self.mask
        # Set NaN where there are zeros
        tmp_ds = tmp_ds.where(tmp_ds != 0)

        # Where there were fill values, set the value again to
        # fill value to avoid not having data to interpolate
        #tmp_ds.data[idx_no_data] = fill_value
        tmp_ds += mask_fill_value

        #tmp_ds[idx_no_data] = fill_value

        # Where are less than 20% of observations, use fill value
        min_n_obs = int(tmp_ds.shape[0] * 0.2)
        #idx_lt_two_obs = np.where(self.mask.sum(axis=0) < min_n_obs)
        tmp_ds = tmp_ds.where(self.mask.sum(axis=0) > min_n_obs, fill_value)

        #tmp_ds.data[:, idx_lt_two_obs[0],
        #            idx_lt_two_obs[1]] = fill_value
        #tmp_ds[:, idx_lt_two_obs[0], idx_lt_two_obs[1]] = fill_value

        for method in interpolation_methods:
            if self.isNotebook is True:
                progress_bar.value = _item
                progress_bar.description = (f"Interpolation of {data_var}"
                                            f" using {method}")

            if method == 'smoothn':
                # First, we need a linear interpolation
                tmp_interpol_ds = tmp_ds.interpolate_na(dim='time',
                                                        method='linear')

                # Weigth obs
                #idx = np.nonzero(tmp_interpol_ds.data)
                #w = tmp_ds.copy(deep=True).data
                #w[idx] *= 2
                # Smoothing
                s = float(self.smooth_factor.value)
                tmp_masked = np.ma.masked_equal(
                    tmp_interpol_ds.data * self.mask, 0)

                tmp_smoothed = smoothn(
                    tmp_masked,
                    #W=tmp_masked * 2, isrobust=True,
                    isrobust=True,
                    s=s,
                    TolZ=1e-6,
                    axis=0)[0]

                tmp_masked = None
                del (tmp_masked)
                # Overwrite data
                tmp_interpol_ds.data = tmp_smoothed

            else:
                tmp_interpol_ds = tmp_ds.interpolate_na(dim='time',
                                                        method=method)

            # Set data type to match the original (non-interpolated)
            tmp_interpol_ds.data = tmp_interpol_ds.data.astype(dtype)
            # Copy metadata attributes
            tmp_interpol_ds.attrs = tmp_ds.attrs

            # Save to file
            fname = f"{self.product}.{self.version}.{data_var}.{method}.tif"
            output_dir = os.path.join(self.source_dir, data_var[1::],
                                      'interpolated')

            if os.path.exists(output_dir) is False:
                os.mkdir(output_dir)
            fname = os.path.join(output_dir, fname)

            save_dask_array(fname=fname,
                            data=tmp_interpol_ds,
                            data_var=data_var,
                            method=method,
                            tile_size=tile_size,
                            n_workers=n_workers,
                            threads_per_worker=threads_per_worker,
                            memory_limit=memory_limit,
                            progressBar=progressBar)

            if self.isNotebook is True:
                _item += 1

        if self.isNotebook is True:
            # Remove progress bar
            progress_bar.close()
            del progress_bar
Пример #22
0
def process_folder(path_in, path_out, path_to_annotations, predictor_path):
    '''
    Detects faces and landmarks on every image in the folder,
    and saves cropped faces to the output folder. Landmarks are
    saves to a DataFrame, or appended to an existing one.
    
    Parameters
    ----------
    path_in : str
        Folder with input images.
    path_out : str
        Ouput folder for extracted faces.
    path_to_annotations : str
        Path to annotations csv file.
    predictor_path : str
        Path to a trained dlib face detector.
    Returns
    -------
    '''
    # Create output dir, if needed.
    if not os.path.exists(os.path.abspath(path_out)):
        os.makedirs(os.path.abspath(path_out))

    # Get list of image pathes.
    imgs = [
        os.path.join(path_in, filename) for filename in os.listdir(path_in)
    ]

    # Configure dlib face detector and landmarks predictor
    face_detector = dlib.get_frontal_face_detector()
    # Read trained landmarks predictor model
    predictor = dlib.shape_predictor(predictor_path)

    landmarks = {}
    count = 0

    # Progress bar
    bar = IntProgress(value=1, min=1, max=len(imgs), step=1)
    print('Processing folder: ' + path_in)
    display(bar)

    # Read the annotations file
    if os.path.exists(path_to_annotations):
        annotations = True
        df = pd.read_csv(path_to_annotations, header=0, index_col=0)
    else:
        annotations = False

    for filename in imgs:
        # Update progress bar
        bar.description = '%d / %d' % (bar.value, len(imgs))

        # Try to find the image in already processed files
        img_processed = False
        if annotations:
            img_basename = os.path.splitext(os.path.basename(filename))[0]
            if df.index.str.startswith(img_basename).sum() > 0:
                img_processed = True

        if not img_processed or not annotations:
            # Read the image
            img = imread(filename)
            # Detect faces
            dets = face_detector(img, 1)

            # Iterate over all detected faces
            for i, d in enumerate(dets):
                count += 1
                # Predict landmark points
                shape = predictor(img, d)

                # Crop face
                face, points = crop_by_landmarks(img,
                                                 shape_to_array(shape),
                                                 pad=10)

                # Save face and landmarks
                base, ext = os.path.splitext(filename)
                face_filename = os.path.basename(base) + '_' + str(i) + ext
                imsave(os.path.join(path_out, face_filename), face)
                landmarks[face_filename] = array_to_str(points)

        bar.value += 1

    # Convert landmarks dictionary to DataFrame and add to csv
    landmarks_df = pd.DataFrame.from_dict(landmarks,
                                          columns=['points'],
                                          orient='index')
    landmarks_df.index.name = 'filename'
    landmarks_df['smile'] = 0
    landmarks_df['mouth_open'] = 0
    landmarks_df['labeled'] = False

    if annotations:
        df = df.append(landmarks_df, sort=False)
        df.to_csv(path_to_annotations)
    else:
        landmarks_df.to_csv(path_to_annotations)

    # Print statistics
    print('Found %d faces' % count)
Пример #23
0
    def _analytics(self, b):
        """
        Uses the self.user_qa_selection OrderedDictionary to extract
        the corresponding QA values and create a mask of dimensions:
            (number of qa layers, time steps, cols(lat), rows(lon))
        Additionally computes the temporal mask and the max gap length
        """
        if not type(b) == QProgressBar:
            progress_bar = IntProgress(
                value=0,
                min=0,
                max=len(self.user_qa_selection),
                step=1,
                description='',
                bar_style='', # 'success', 'info', 'warning', 'danger' or ''
                orientation='horizontal',
                style = {'description_width': 'initial'},
                layout={'width': '50%'}
            )
            display(progress_bar)

        n_qa_layers = len(self.user_qa_selection)

        # Get the name of the first data var to extract its shape
        for k, v in self.ts.data.data_vars.items():
            break

        # Create mask xarray
        _time, _latitude, _longitude = self.ts.data.data_vars[k].shape
        mask = np.zeros((n_qa_layers, _time, _latitude, _longitude),
                        np.int8)

        qa_layer = self.qa_def.QualityLayer.unique()

        # QA layer user to create mask
        _qa_layer = getattr(self.ts.qa, f"qa{qa_layer[0]}")

        for i, user_qa in enumerate(self.user_qa_selection):

            if type(b) == QProgressBar:
                b.setValue(i)
                b.setFormat(f"Masking by QA {user_qa}")
            else:
                progress_bar.value = i
                progress_bar.description = f"Masking by QA {user_qa}"

            user_qa_fieldname = user_qa.replace(" ", "_").replace("/", "_")

            for j, qa_value in enumerate(self.user_qa_selection[user_qa]):
                qa_value_field_name = qa_value.replace(" ", "_")

                qa_flag_val = self.qa_def[(self.qa_def.Name == user_qa) & 
                        (self.qa_def.Description == qa_value)].Value.iloc[0]

                if j == 0 :
                    mask[i] = (_qa_layer[user_qa_fieldname] == qa_flag_val)
                else:
                    mask[i] = np.logical_or(
                            mask[i], _qa_layer[user_qa_fieldname] == qa_flag_val)

        if type(b) == QProgressBar:
            b.setValue(0)
            b.setEnabled(False)
        else:
            # Remove progress bar
            progress_bar.close()
            del progress_bar

        #self.__temp_mask = mask
        #mask = xr.DataArray(np.all(self.__temp_mask, axis=0),
        mask = xr.DataArray(np.all(mask, axis=0),
                            coords=[v.time.data,
                                    v.latitude.data,
                                    v.longitude.data],
                            dims=['time', 'latitude', 'longitude'])

        mask.attrs = v.attrs

        self.mask = mask
        # Remove local multi-layer mask variable
        mask = None
        del(mask)

        # Create the percentage of data available mask
        # Get the per-pixel per-time step binary mask
        pct_data_available = (self.mask.sum(axis=0) * 100.0) / _time
        pct_data_available.latitude.data = v.latitude.data
        pct_data_available.longitude.data = v.longitude.data
        # Set the pct_data_available object
        self.pct_data_available = pct_data_available

        # Using the computed mask get the max gap length
        self.__get_max_gap_length(b)
Пример #24
0
def vgg16_train(model, train, test, init_from, save_dir, batch_size=64, epoch=300, early_stop_patience=25):
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)
    
    checkpoint_path = os.path.join(save_dir, 'model.ckpt')

    with tf.Session() as sess:
        print(tf.trainable_variables())
        
        # hyper parameters
        learning_rate =  5e-4 #adam
        min_delta = 0.0001

        # recorder
        epoch_counter = 0
        loss_history = []
        val_loss_history = []

        # optimizer
        opt = tf.train.AdamOptimizer(learning_rate=learning_rate)
        train_op = opt.minimize(model.loss)
        
        # saver 
        saver = tf.train.Saver(tf.global_variables(), max_to_keep=2)
        
        sess.run(tf.global_variables_initializer())

        # progress bar
        ptrain = IntProgress()
        pval = IntProgress()
        display(ptrain)
        display(pval)
        ptrain.max = int(train.images.shape[0]/batch_size)
        pval.max = int(test.images.shape[0]/batch_size)

        # reset due to adding a new task
        patience_counter = 0
        current_best_val_loss = 100000 # a large number
        

        # train start
        while(patience_counter < early_stop_patience):
            stime = time.time()
            bar_train = Bar('Training', max=int(train.images.shape[0]/batch_size), suffix='%(index)d/%(max)d - %(percent).1f%% - %(eta)ds')
            bar_val =  Bar('Validation', max=int(test.images.shape[0]/batch_size), suffix='%(index)d/%(max)d - %(percent).1f%% - %(eta)ds')
            
            # training an epoch
            train_loss = 0
            for i in range(int(train.images.shape[0]/batch_size)):
                st = i*batch_size
                ed = (i+1)*batch_size
                
                _, loss = sess.run([train_op, model.loss],
                                   feed_dict={model.x: train.images[st:ed,:],
                                              model.y: train.labels[st:ed,:],
                                              model.w: train.weights[st:ed,:]
                                             })
                train_loss += loss
                ptrain.value +=1
                ptrain.description = "Training %s/%s" % (i, ptrain.max)
                bar_train.next()
            
            train_loss /= ptrain.max
            
            val_loss = 0

            for i in range(int(test.images.shape[0]/batch_size)):
                st = i*batch_size
                ed = (i+1)*batch_size
                
                loss = sess.run(model.loss,
                                   feed_dict={model.x: test.images[st:ed,:],
                                              model.y: test.labels[st:ed,:],
                                              model.w: np.expand_dims(np.repeat(1.0,batch_size),axis=1)
                                             })
                val_loss += loss
                pval.value +=1
                pval.description = "Training %s/%s" % (i, pval.max)
                bar_val.next()
                
            val_loss /= pval.max
            
            if (current_best_val_loss - val_loss) > min_delta:
                current_best_val_loss = val_loss
                patience_counter = 0
                saver.save(sess, checkpoint_path, global_step=epoch_counter)
                print("reset early stopping and save model into %s at epoch %s" % (checkpoint_path,epoch_counter))
            else:
                patience_counter += 1

            # shuffle Xtrain and Ytrain in the next epoch
            train.shuffle()
            
            loss_history.append(train_loss)
            val_loss_history.append(val_loss)

            ptrain.value = 0
            pval.value = 0
            bar_train.finish()
            bar_val.finish()
            print("Epoch %s (%s), %s sec >> train-loss: %.4f, val-loss: %.4f" % (epoch_counter, patience_counter, round(time.time()-stime,2), train_loss, val_loss))
            
            # epoch end
            epoch_counter += 1
            if epoch_counter >= epoch:
                break
        res = pd.DataFrame({"epoch":range(0,len(loss_history)), "loss":loss_history, "val_loss":val_loss_history})
        res.to_csv(os.path.join(save_dir,"history.csv"), index=False)
        print("end training")
Пример #25
0
    def getTS(self, startDate, endDate):

        ndays = (endDate - startDate).days + 1
        if self.init:
            ndays += 1
        currentDate = startDate
        delta = timedelta(days=1)

        self.df = pd.DataFrame(columns=[
            'datetime', 'AOD', 'DUST_PM', 'SALT_PM', 'ORG_CARB', 'BLK_CARB',
            'SO4', 'PM2.5'
        ])

        with self.out_cp:
            self.out_cp.clear_output()

            pbar = IntProgress(min=0, max=int(ndays))
            pbar.description = 'Progress:'
            info1 = Label('0%')
            info2 = Label(' ')
            display(
                VBox([
                    HBox([pbar, info1]),
                    HBox([info2], layout=Layout(justify_content='center'))
                ]))

            progVal = 0
            if self.init:
                info2.value = 'Initializing NASA Earth Data Connection..'
                self.initSession()
                self.init = False
                pbar.value += 1
                progVal += 1
                info1.value = '{:.1f}%'.format(
                    (float(progVal) / float(ndays)) * 100.0)

            self.lonlatToIndex(self.plon, self.plat)

            while currentDate <= endDate:
                url = self.getUrlMERRA(currentDate)
                info2.value = 'Accessing data for {}'.format(currentDate)

                dataset = open_url(url, session=self.session)
                aod = np.squeeze(dataset['TOTEXTTAU'][:, self.ilat, self.ilon])
                dust_pm = np.squeeze(
                    dataset['DUSMASS25'][:, self.ilat,
                                         self.ilon]) * 1000000000.0
                salt_pm = np.squeeze(
                    dataset['SSSMASS25'][:, self.ilat,
                                         self.ilon]) * 1000000000.0
                org_carb = np.squeeze(
                    dataset['OCSMASS'][:, self.ilat, self.ilon]) * 1000000000.0
                blk_carb = np.squeeze(
                    dataset['BCSMASS'][:, self.ilat, self.ilon]) * 1000000000.0
                so4 = np.squeeze(dataset['SO4SMASS'][:, self.ilat,
                                                     self.ilon]) * 1000000000.0
                pm25 = (1.375 * so4 + 1.6 * org_carb + blk_carb + dust_pm +
                        salt_pm)
                dt = pd.date_range(currentDate, periods=24, freq='H')
                vardict = {
                    'datetime': dt,
                    'AOD': aod,
                    'DUST_PM': dust_pm,
                    'SALT_PM': salt_pm,
                    'ORG_CARB': org_carb,
                    'BLK_CARB': blk_carb,
                    'SO4': so4,
                    'PM2.5': pm25
                }
                df_add = pd.DataFrame(vardict)
                self.df = pd.concat([self.df, df_add])
                currentDate += delta
                progVal += 1
                info1.value = '{:.1f}%'.format(
                    (float(progVal) / float(ndays)) * 100.0)
                pbar.value += 1
        self.stateChange = False
Пример #26
0
def train(FLAG):
    print("Reading dataset...")
    # load data
    Xtrain, Ytrain = read_images(TRAIN_DIR), read_masks(TRAIN_DIR, onehot=True)
    Xtest, Ytest = read_images(VAL_DIR), read_masks(VAL_DIR, onehot=True)
    track = [
        "hw3-train-validation/validation/0008",
        "hw3-train-validation/validation/0097",
        "hw3-train-validation/validation/0107"
    ]
    Xtrack, Ytrack = read_list(track)

    vgg16 = VGG16(classes=7, shape=(256, 256, 3))
    vgg16.build(vgg16_npy_path=FLAG.init_from,
                mode=FLAG.mode,
                keep_prob=FLAG.keep_prob)

    saver = tf.train.Saver(tf.global_variables(), max_to_keep=1)
    checkpoint_path = os.path.join(FLAG.save_dir, 'model.ckpt')

    def initialize_uninitialized(sess):
        global_vars = tf.global_variables()
        is_not_initialized = sess.run(
            [tf.is_variable_initialized(var) for var in global_vars])
        not_initialized_vars = [
            v for (v, f) in zip(global_vars, is_not_initialized) if not f
        ]
        if len(not_initialized_vars):
            sess.run(tf.variables_initializer(not_initialized_vars))

    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())

        # hyper parameters
        batch_size = 32
        epoch = 500
        early_stop_patience = 50
        min_delta = 0.0001
        opt_type = 'adam'

        # recorder
        epoch_counter = 0

        # optimizer
        global_step = tf.Variable(0, trainable=False)

        # Passing global_step to minimize() will increment it at each step.
        if opt_type is 'sgd':
            start_learning_rate = FLAG.lr
            half_cycle = 2000
            learning_rate = tf.train.exponential_decay(start_learning_rate,
                                                       global_step,
                                                       half_cycle,
                                                       0.5,
                                                       staircase=True)
            opt = tf.train.MomentumOptimizer(learning_rate=learning_rate,
                                             momentum=0.9,
                                             use_nesterov=True)
        else:
            start_learning_rate = FLAG.lr
            half_cycle = 2000
            learning_rate = tf.train.exponential_decay(start_learning_rate,
                                                       global_step,
                                                       half_cycle,
                                                       0.5,
                                                       staircase=True)
            opt = tf.train.AdamOptimizer(learning_rate=learning_rate)

        obj = vgg16.loss
        train_op = opt.minimize(obj, global_step=global_step)

        # progress bar
        ptrain = IntProgress()
        pval = IntProgress()
        display(ptrain)
        display(pval)
        ptrain.max = int(Xtrain.shape[0] / batch_size)
        pval.max = int(Xtest.shape[0] / batch_size)

        # re-initialize
        initialize_uninitialized(sess)

        # reset due to adding a new task
        patience_counter = 0
        current_best_val_loss = np.float('Inf')

        # optimize when the aggregated obj
        while (patience_counter < early_stop_patience
               and epoch_counter < epoch):

            # start training
            stime = time.time()
            bar_train = Bar(
                'Training',
                max=int(Xtrain.shape[0] / batch_size),
                suffix='%(index)d/%(max)d - %(percent).1f%% - %(eta)ds')
            bar_val = Bar(
                'Validation',
                max=int(Xtest.shape[0] / batch_size),
                suffix='%(index)d/%(max)d - %(percent).1f%% - %(eta)ds')

            train_loss, train_accu = 0.0, 0.0
            for i in range(int(Xtrain.shape[0] / batch_size)):
                st = i * batch_size
                ed = (i + 1) * batch_size
                loss, accu, _ = sess.run(
                    [obj, vgg16.accuracy, train_op],
                    feed_dict={
                        vgg16.x: Xtrain[st:ed, :],
                        vgg16.y: Ytrain[st:ed, :],
                        vgg16.is_train: True
                    })
                train_loss += loss
                train_accu += accu
                ptrain.value += 1
                ptrain.description = "Training %s/%s" % (ptrain.value,
                                                         ptrain.max)
            train_loss = train_loss / ptrain.value
            train_accu = train_accu / ptrain.value

            # validation
            val_loss = 0
            val_accu = 0
            for i in range(int(Xtest.shape[0] / batch_size)):
                st = i * batch_size
                ed = (i + 1) * batch_size
                loss, accu = sess.run(
                    [obj, vgg16.accuracy],
                    feed_dict={
                        vgg16.x: Xtest[st:ed, :],
                        vgg16.y: Ytest[st:ed, :],
                        vgg16.is_train: False
                    })
                val_loss += loss
                val_accu += accu
                pval.value += 1
                pval.description = "Testing %s/%s" % (pval.value, pval.value)
            val_loss = val_loss / pval.value
            val_accu = val_accu / pval.value

            # plot
            if epoch_counter % 10 == 0:
                Xplot = sess.run(vgg16.pred,
                                 feed_dict={
                                     vgg16.x: Xtrack[:, :],
                                     vgg16.y: Ytrack[:, :],
                                     vgg16.is_train: False
                                 })

                for i, fname in enumerate(track):
                    saveimg = skimage.transform.resize(Xplot[i],
                                                       output_shape=(512, 512),
                                                       order=0,
                                                       preserve_range=True,
                                                       clip=False)
                    saveimg = label2rgb(saveimg)
                    imageio.imwrite(
                        os.path.join(
                            FLAG.save_dir,
                            os.path.basename(fname) + "_pred_" +
                            str(epoch_counter) + ".png"), saveimg)
                    print(
                        os.path.join(
                            FLAG.save_dir,
                            os.path.basename(fname) + "_pred_" +
                            str(epoch_counter) + ".png"))

            # early stopping check
            if (current_best_val_loss - val_loss) > min_delta:
                current_best_val_loss = val_loss
                patience_counter = 0
                saver.save(sess, checkpoint_path, global_step=epoch_counter)
                print("save in %s" % checkpoint_path)
            else:
                patience_counter += 1

            # shuffle Xtrain and Ytrain in the next epoch
            idx = np.random.permutation(Xtrain.shape[0])
            Xtrain, Ytrain = Xtrain[idx, :, :, :], Ytrain[idx, :]

            # epoch end
            epoch_counter += 1

            ptrain.value = 0
            pval.value = 0
            bar_train.finish()
            bar_val.finish()

            print(
                "Epoch %s (%s), %s sec >> train loss: %.4f, train accu: %.4f, val loss: %.4f, val accu: %.4f"
                % (epoch_counter, patience_counter,
                   round(time.time() - stime,
                         2), train_loss, train_accu, val_loss, val_accu))
Пример #27
0
def train(FLAG):
    print("Reading dataset...")
    if FLAG.dataset == 'CIFAR-10':
        train_data = CIFAR10(train=True)
        test_data = CIFAR10(train=False)
        vgg16 = VGG16(classes=10)
    elif FLAG.dataset == 'CIFAR-100':
        train_data = CIFAR100(train=True)
        test_data = CIFAR100(train=False)
        vgg16 = VGG16(classes=100)
    else:
        raise ValueError("dataset should be either CIFAR-10 or CIFAR-100.")
    print("Build VGG16 models for %s..." % FLAG.dataset)

    Xtrain, Ytrain = train_data.train_data, train_data.train_labels
    Xtest, Ytest = test_data.test_data, test_data.test_labels

    vgg16.build(vgg16_npy_path=FLAG.init_from,
                prof_type=FLAG.prof_type,
                conv_pre_training=True,
                fc_pre_training=False)
    vgg16.sparsity_train(l1_gamma=FLAG.lambda_s,
                         l1_gamma_diff=FLAG.lambda_m,
                         decay=FLAG.decay,
                         keep_prob=FLAG.keep_prob)

    # define tasks
    tasks = ['var_dp']
    print(tasks)

    # initial task
    cur_task = tasks[0]
    obj = vgg16.loss_dict[tasks[0]]

    saver = tf.train.Saver(tf.global_variables(), max_to_keep=len(tasks))

    checkpoint_path = os.path.join(FLAG.save_dir, 'model.ckpt')
    tvars_trainable = tf.trainable_variables()

    #for rm in vgg16.gamma_var:
    #    tvars_trainable.remove(rm)
    #    print('%s is not trainable.'% rm)

    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())

        # hyper parameters
        batch_size = 64
        epoch = 500
        early_stop_patience = 50
        min_delta = 0.0001
        opt_type = 'adam'

        # recorder
        epoch_counter = 0

        # optimizer
        global_step = tf.Variable(0, trainable=False)

        # Passing global_step to minimize() will increment it at each step.
        if opt_type is 'sgd':
            start_learning_rate = 1e-4  # adam # 4e-3 #sgd
            half_cycle = 20000
            learning_rate = tf.train.exponential_decay(start_learning_rate,
                                                       global_step,
                                                       half_cycle,
                                                       0.5,
                                                       staircase=True)
            opt = tf.train.MomentumOptimizer(learning_rate=learning_rate,
                                             momentum=0.9,
                                             use_nesterov=True)
        else:
            start_learning_rate = 1e-4  # adam # 4e-3 #sgd
            half_cycle = 10000
            learning_rate = tf.train.exponential_decay(start_learning_rate,
                                                       global_step,
                                                       half_cycle,
                                                       0.5,
                                                       staircase=True)
            opt = tf.train.AdamOptimizer(learning_rate=learning_rate)

        train_op = opt.minimize(obj,
                                global_step=global_step,
                                var_list=tvars_trainable)

        # progress bar
        ptrain = IntProgress()
        pval = IntProgress()
        display(ptrain)
        display(pval)
        ptrain.max = int(Xtrain.shape[0] / batch_size)
        pval.max = int(Xtest.shape[0] / batch_size)

        spareness = vgg16.spareness(thresh=0.05)
        print("initial spareness: %s" % sess.run(spareness))

        # re-initialize
        initialize_uninitialized(sess)

        # reset due to adding a new task
        patience_counter = 0
        current_best_val_accu = 0

        # optimize when the aggregated obj
        while (patience_counter < early_stop_patience
               and epoch_counter < epoch):

            def load_batches():
                for i in range(int(Xtrain.shape[0] / batch_size)):
                    st = i * batch_size
                    ed = (i + 1) * batch_size
                    batch = ia.Batch(images=Xtrain[st:ed, :, :, :],
                                     data=Ytrain[st:ed, :])
                    yield batch

            batch_loader = ia.BatchLoader(load_batches)
            bg_augmenter = ia.BackgroundAugmenter(batch_loader=batch_loader,
                                                  augseq=transform,
                                                  nb_workers=4)

            # start training
            stime = time.time()
            bar_train = Bar(
                'Training',
                max=int(Xtrain.shape[0] / batch_size),
                suffix='%(index)d/%(max)d - %(percent).1f%% - %(eta)ds')
            bar_val = Bar(
                'Validation',
                max=int(Xtest.shape[0] / batch_size),
                suffix='%(index)d/%(max)d - %(percent).1f%% - %(eta)ds')
            train_loss, train_accu = 0.0, 0.0
            while True:
                batch = bg_augmenter.get_batch()
                if batch is None:
                    print("Finished epoch.")
                    break
                x_images_aug = batch.images_aug
                y_images = batch.data
                loss, accu, _ = sess.run(
                    [obj, vgg16.accu_dict[cur_task], train_op],
                    feed_dict={
                        vgg16.x: x_images_aug,
                        vgg16.y: y_images,
                        vgg16.is_train: True
                    })
                bar_train.next()
                train_loss += loss
                train_accu += accu
                ptrain.value += 1
                ptrain.description = "Training %s/%s" % (ptrain.value,
                                                         ptrain.max)
            train_loss = train_loss / ptrain.value
            train_accu = train_accu / ptrain.value
            batch_loader.terminate()
            bg_augmenter.terminate()

            # # training an epoch
            # for i in range(int(Xtrain.shape[0]/batch_size)):
            #     st = i*batch_size
            #     ed = (i+1)*batch_size

            #     augX = transform.augment_images(Xtrain[st:ed,:,:,:])

            #     sess.run([train_op], feed_dict={vgg16.x: augX,
            #                                     vgg16.y: Ytrain[st:ed,:],
            #                                     vgg16.is_train: False})
            #     ptrain.value +=1
            #     ptrain.description = "Training %s/%s" % (i, ptrain.max)
            #     bar_train.next()

            # validation
            val_loss = 0
            val_accu = 0
            for i in range(int(Xtest.shape[0] / 200)):
                st = i * 200
                ed = (i + 1) * 200
                loss, accu = sess.run(
                    [obj, vgg16.accu_dict[cur_task]],
                    feed_dict={
                        vgg16.x: Xtest[st:ed, :],
                        vgg16.y: Ytest[st:ed, :],
                        vgg16.is_train: False
                    })
                val_loss += loss
                val_accu += accu
                pval.value += 1
                pval.description = "Testing %s/%s" % (pval.value, pval.value)
            val_loss = val_loss / pval.value
            val_accu = val_accu / pval.value

            print("\nspareness: %s" % sess.run(spareness))
            # early stopping check
            if (val_accu - current_best_val_accu) > min_delta:
                current_best_val_accu = val_accu
                patience_counter = 0

                para_dict = sess.run(vgg16.para_dict)
                np.save(os.path.join(FLAG.save_dir, "para_dict.npy"),
                        para_dict)
                print("save in %s" %
                      os.path.join(FLAG.save_dir, "para_dict.npy"))
            else:
                patience_counter += 1

            # shuffle Xtrain and Ytrain in the next epoch
            idx = np.random.permutation(Xtrain.shape[0])
            Xtrain, Ytrain = Xtrain[idx, :, :, :], Ytrain[idx, :]

            # epoch end
            # writer.add_summary(epoch_summary, epoch_counter)
            epoch_counter += 1

            ptrain.value = 0
            pval.value = 0
            bar_train.finish()
            bar_val.finish()

            print(
                "Epoch %s (%s), %s sec >> train loss: %.4f, train accu: %.4f, val loss: %.4f, val accu at %s: %.4f"
                % (epoch_counter, patience_counter,
                   round(time.time() - stime, 2), train_loss, train_accu,
                   val_loss, cur_task, val_accu))
        saver.save(sess, checkpoint_path, global_step=epoch_counter)

        sp, rcut = gammaSparsifyVGG16(para_dict, thresh=0.02)
        np.save(os.path.join(FLAG.save_dir, "sparse_dict.npy"), sp)
        print("sparsify %s in %s" % (np.round(
            1 - rcut, 3), os.path.join(FLAG.save_dir, "sparse_dict.npy")))

        #writer.close()
        arr_spareness.append(1 - rcut)
        np.save(os.path.join(FLAG.save_dir, "sprocess.npy"), arr_spareness)
    FLAG.optimizer = opt_type
    FLAG.lr = start_learning_rate
    FLAG.batch_size = batch_size
    FLAG.epoch_end = epoch_counter
    FLAG.val_accu = current_best_val_accu

    header = ''
    row = ''
    for key in sorted(vars(FLAG)):
        if header is '':
            header = key
            row = str(getattr(FLAG, key))
        else:
            header += "," + key
            row += "," + str(getattr(FLAG, key))
    row += "\n"
    header += "\n"
    if os.path.exists("/home/cmchang/new_CP_CNN/model.csv"):
        with open("/home/cmchang/new_CP_CNN/model.csv", "a") as myfile:
            myfile.write(row)
    else:
        with open("/home/cmchang/new_CP_CNN/model.csv", "w") as myfile:
            myfile.write(header)
            myfile.write(row)
Пример #28
0
def train(FLAG):
    print("Reading dataset...")
    # load data
    Xtrain, df_train = read_dataset(TRAIN_CSV, TRAIN_DIR)
    Xtest, df_test = read_dataset(TEST_CSV, TEST_DIR)

    vae = VAE()
    vae.build(lambda_KL=FLAG.lambda_KL,
              n_dim=FLAG.n_dim,
              batch_size=FLAG.batch_size,
              shape=Xtrain.shape[1:])

    saver = tf.train.Saver(tf.global_variables(), max_to_keep=1)
    checkpoint_path = os.path.join(FLAG.save_dir, 'model.ckpt')

    def initialize_uninitialized(sess):
        global_vars = tf.global_variables()
        is_not_initialized = sess.run(
            [tf.is_variable_initialized(var) for var in global_vars])
        not_initialized_vars = [
            v for (v, f) in zip(global_vars, is_not_initialized) if not f
        ]
        if len(not_initialized_vars):
            sess.run(tf.variables_initializer(not_initialized_vars))

    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())

        # hyper parameters
        batch_size = FLAG.batch_size
        epoch = 500
        early_stop_patience = 50
        min_delta = 0.0001
        opt_type = 'adam'

        # recorder
        epoch_counter = 0

        # optimizer
        global_step = tf.Variable(0, trainable=False)

        # Passing global_step to minimize() will increment it at each step.
        if opt_type is 'sgd':
            start_learning_rate = FLAG.lr
            half_cycle = 2000
            learning_rate = tf.train.exponential_decay(start_learning_rate,
                                                       global_step,
                                                       half_cycle,
                                                       0.5,
                                                       staircase=True)
            opt = tf.train.MomentumOptimizer(learning_rate=learning_rate,
                                             momentum=0.9,
                                             use_nesterov=True)
        else:
            start_learning_rate = FLAG.lr
            half_cycle = 2000
            learning_rate = tf.train.exponential_decay(start_learning_rate,
                                                       global_step,
                                                       half_cycle,
                                                       0.5,
                                                       staircase=True)
            opt = tf.train.AdamOptimizer(learning_rate=learning_rate)

        obj = vae.train_op
        train_op = opt.minimize(obj, global_step=global_step)

        # progress bar
        ptrain = IntProgress()
        pval = IntProgress()
        display(ptrain)
        display(pval)
        ptrain.max = int(Xtrain.shape[0] / batch_size)
        pval.max = int(Xtest.shape[0] / batch_size)

        # re-initialize
        initialize_uninitialized(sess)

        # reset due to adding a new task
        patience_counter = 0
        current_best_val_loss = np.float('Inf')

        # optimize when the aggregated obj
        while (patience_counter < early_stop_patience
               and epoch_counter < epoch):

            # start training
            stime = time.time()
            bar_train = Bar(
                'Training',
                max=int(Xtrain.shape[0] / batch_size),
                suffix='%(index)d/%(max)d - %(percent).1f%% - %(eta)ds')
            bar_val = Bar(
                'Validation',
                max=int(Xtest.shape[0] / batch_size),
                suffix='%(index)d/%(max)d - %(percent).1f%% - %(eta)ds')

            train_loss = 0.0
            train_reconstruction_loss = 0.0
            train_kl_loss = 0.0
            for i in range(int(Xtrain.shape[0] / batch_size)):
                st = i * batch_size
                ed = (i + 1) * batch_size
                loss, reconstruction_loss, kl_loss, _ = sess.run(
                    [
                        obj, vae.loss['reconstruction'], vae.loss['KL_loss'],
                        train_op
                    ],
                    feed_dict={
                        vae.x: Xtrain[st:ed, :],
                        vae.y: Xtrain[st:ed, :],
                        vae.is_train: True
                    })
                print(loss)
                print(reconstruction_loss)
                print(kl_loss)
                train_loss += loss
                train_reconstruction_loss += reconstruction_loss
                train_kl_loss += kl_loss
                ptrain.value += 1
                ptrain.description = "Training %s/%s" % (ptrain.value,
                                                         ptrain.max)

                output = sess.run(
                    [vae.output],
                    feed_dict={
                        vae.x: Xtrain[0:64, :],
                        vae.y: Xtrain[0:64, :],
                        vae.is_train: False
                    })

                print("=== train data ====")
                print(output)
                #print((Xtrain[0,:]-128.0)/128.0)
            train_loss = train_loss / ptrain.value
            train_reconstruction_loss = train_reconstruction_loss / ptrain.value
            train_kl_loss = train_kl_loss / ptrain.value

            # validation
            val_loss = 0
            val_reconstruction_loss = 0.0
            val_kl_loss = 0.0
            for i in range(int(Xtest.shape[0] / batch_size)):
                st = i * batch_size
                ed = (i + 1) * batch_size
                loss, reconstruction_loss, kl_loss = sess.run(
                    [obj, vae.loss['reconstruction'], vae.loss['KL_loss']],
                    feed_dict={
                        vae.x: Xtest[st:ed, :],
                        vae.y: Xtest[st:ed, :],
                        vae.is_train: False
                    })
                val_loss += loss
                val_reconstruction_loss += reconstruction_loss
                val_kl_loss += kl_loss
                pval.value += 1
                pval.description = "Testing %s/%s" % (pval.value, pval.value)
            val_loss = val_loss / pval.value
            val_reconstruction_loss = val_reconstruction_loss / pval.value
            val_kl_loss = val_kl_loss / pval.value

            # plot
            # if epoch_counter%10 == 0:
            #     Xplot = sess.run(vae.output,
            #             feed_dict={vae.x: Xtest[:,:],
            #                         vae.y: Xtest[:,:],
            #                         vae.is_train: False})
            #     for i, fname in enumerate(track):
            #         imageio.imwrite(os.path.join(FLAG.save_dir,os.path.basename(fname)+"_pred_"+str(epoch_counter)+".png"), saveimg)
            #         print(os.path.join(FLAG.save_dir,os.path.basename(fname)+"_pred_"+str(epoch_counter)+".png"))

            # early stopping check
            if (current_best_val_loss - val_loss) > min_delta:
                current_best_val_loss = val_loss
                patience_counter = 0
                saver.save(sess, checkpoint_path, global_step=epoch_counter)
                print("save in %s" % checkpoint_path)
            else:
                patience_counter += 1

            # shuffle Xtrain and Ytrain in the next epoch
            idx = np.random.permutation(Xtrain.shape[0])
            Xtrain = Xtrain[idx, :, :, :]

            # epoch end
            epoch_counter += 1

            ptrain.value = 0
            pval.value = 0
            bar_train.finish()
            bar_val.finish()

            print(
                "Epoch %s (%s), %s sec >> train loss: %.4f, train recon loss: %.4f, train kl loss: %.4f, val loss: %.4f, val recon loss: %.4f, val kl loss: %.4f"
                % (epoch_counter, patience_counter,
                   round(time.time() - stime, 2), train_loss,
                   train_reconstruction_loss, train_kl_loss, val_loss,
                   val_reconstruction_loss, val_kl_loss))

        # para_dict = sess.run(vgg16.para_dict)
        # np.save(os.path.join(FLAG.save_dir, "para_dict.npy"), para_dict)
        # print("save in %s" % os.path.join(FLAG.save_dir, "para_dict.npy"))

        FLAG.optimizer = opt_type
        FLAG.lr = start_learning_rate
        FLAG.batch_size = batch_size
        FLAG.epoch_end = epoch_counter
        FLAG.val_loss = current_best_val_loss

        header = ''
        row = ''
        for key in sorted(vars(FLAG)):
            if header is '':
                header = key
                row = str(getattr(FLAG, key))
            else:
                header += "," + key
                row += "," + str(getattr(FLAG, key))
        row += "\n"
        if os.path.exists("/home/cmchang/DLCV2018SPRING/hw4/model.csv"):
            with open("/home/cmchang/DLCV2018SPRING/hw4/model.csv",
                      "a") as myfile:
                myfile.write(row)
        else:
            with open("/home/cmchang/DLCV2018SPRING/hw4/model.csv",
                      "w") as myfile:
                myfile.write(header)
                myfile.write(row)
Пример #29
0
def get_press_series(spliter, color, difference, paddings=2):
    global tmp_imgs

    white_width = 17 + 2 * paddings
    black_width = 16 + 2 * paddings
    height = 106
    width = 884

    print('Start extracting keypress series ...')
    print(f'  White width: {white_width}px')
    print(f'  Black width: {black_width}px')
    print(f'  Height: {height}px')
    print('')

    for name in spliter:
        black_coor = None
        N = y_org[name].shape[0]
        for p in X_path[name]:
            img = cv2.imread(p)
            black_coor = get_black_boundaries(img)
            if len(black_coor) == 36:
                break
        y_trans = np.transpose(y_org[name], (1, 0))

        print('Pre-loading images ...')

        bar = IntProgress(max=N)
        display(bar)
        for i in range(N):
            img = pad_img(cv2.imread(X_path[name][i]), paddings)
            tmp_imgs.append(img)
            bar.value += 1
        bar.close()

        bar = IntProgress(max=88)
        display(bar)

        for k in range(88):
            if k in black_mask:
                col = 'black'
            else:
                col = 'white'
            if col not in color:
                continue
            _y = y_trans[k]
            _y = np.argwhere(_y > 0).flatten()
            if _y.shape[0] == 0:
                continue
            last = _y[0]
            _n = len(_y)
            for i in range(_n):
                if i % 32 == 0:
                    bar.description = f'{i}/{_n}'
                if i != 0 and _y[i] != _y[i - 1] + 1:
                    if col == 'black':
                        add_series(name, col, last, _y[i - 1], k, paddings,
                                   black_coor, difference)
                    else:
                        add_series(name, col, last, _y[i - 1], k, paddings,
                                   difference)
                    last = _y[i]
                if i == _n - 1 and last != -1:
                    if col == 'black':
                        add_series(name, col, last, _y[i], k, paddings,
                                   black_coor, difference)
                    else:
                        add_series(name, col, last, _y[i], k, paddings,
                                   difference)
            bar.value += 1

        bar.close()

        del tmp_imgs
        tmp_imgs = []

        print(f'{name} set loading finished ...')
        print('  Pressed white keys: ' + str(len(X_series[name]['white'])))
        print('  Pressed black keys: ' + str(len(X_series[name]['black'])))
Пример #30
0

# 2015-2016
mypath = "/Users/Tim/Desktop/scor_test/Data/2015-2016/"
mydirs = [f for f in listdir(mypath) if isdir(join(mypath, f))]
mydirs.remove("Demographics")
mydirs.remove("Dietary")
df9 = pd.read_csv(
    "/Users/Tim/Desktop/scor_test/Data/2015-2016/Demographics/DEMO_I.csv",
    encoding='utf8')
extension = 'csv'

# Initialize a progess bar
progress = IntProgress()
progress.max = len(mydirs)
progress.description = '(Init)'
display(progress)
for mydir in mydirs:
    os.chdir(mypath + mydir)
    all_filenames = [i for i in glob.glob('*.{}'.format(extension))]
    for file in all_filenames:
        df_temp = pd.read_csv(file, encoding='utf8')
        try:
            df9 = pd.merge(df9, df_temp, how='outer', on='SEQN')
        except:
            print(mypath + mydir + '/' + file)
    progress.value += 1
    progress.description = mydir
progress.description = '(Done)'

k = 0