Exemple #1
0
def train_model_wrap(train_model, shared_x, shared_y, rand_arr, img_mean,
                     count, minibatch_index, minibatch_range, batch_size,
                     train_filenames, train_labels,
                     flag_para_load, 
                     flag_batch,
                     send_queue=None, recv_queue=None):

    # load by self or the other process

    if flag_para_load:
        # wait for the copying to finish
        msg = recv_queue.get()
        assert msg == 'copy_finished'

        if count < len(minibatch_range):
            ind_to_read = minibatch_range[count]
            name_to_read = str(train_filenames[ind_to_read])
            send_queue.put(name_to_read)
            send_queue.put(get_rand3d())

    else:
        batch_img = hkl.load(str(train_filenames[minibatch_index])) - img_mean
        param_rand = get_rand3d()           
        batch_img = crop_and_mirror(batch_img, param_rand, flag_batch=flag_batch)         
        shared_x.set_value(batch_img)

    batch_label = train_labels[minibatch_index * batch_size:
                               (minibatch_index + 1) * batch_size]
    shared_y.set_value(batch_label)

    cost_ij = train_model()

    return cost_ij
def train_model_wrap(train_model, shared_xquery, shared_xp, shared_xns, rand_arrs, img_mean,
                     count, minibatch_index, minibatch_range, batch_size,
                     filenamesA, filenamesB,
                     flag_para_load, flag_batch,
                     send_queue=None, recv_queue=None):

    if flag_para_load:
        # load by self or the other process

        # wait for the copying to finish
        msg = recv_queue.get()
        assert msg == 'copy_finished'

        if count < len(minibatch_range):
            ind_to_read = minibatch_range[count]
            name_to_read1 = str(filenamesA[ind_to_read])
            send_queue.put(name_to_read1)
            send_queue.put(get_rand3d())

            name_to_read2 = str(filenamesB[ind_to_read])
            send_queue.put(name_to_read2)
            send_queue.put(get_rand3d())

    else:
        img1 = hkl.load(str(filenamesA[minibatch_index])) - img_mean
        img2 = hkl.load(str(filenamesB[minibatch_index])) - img_mean
        param_rand1 = get_rand3d()
        param_rand2 = get_rand3d()
        img1 = crop_and_mirror(img1, param_rand1, flag_batch=flag_batch)         
        img2 = crop_and_mirror(img2, param_rand2, flag_batch=flag_batch)         
        shared_xquery.set_value(img1)
        shared_xp.set_value(img2)
        # set false colour variant matches
        for i in xrange(len(shared_xns)):
            shared_xn = shared_xns[i]
            N = img2.shape[3]
            n_indexes = np.arange(N) + np.random.randint(1,N-5)
            n_indexes %= N
            imgN = img2[:,:,:,n_indexes]
            shared_xn.set_value(imgN)

    cost_ij = train_model()

    return cost_ij
Exemple #3
0
def sample_minibatch(train_set, batch_size):
    minibatch_list = choice(train_set, batch_size)
    im_label = np.zeros(batch_size, dtype=int)
    for (pi, (img, label)) in enumerate(minibatch_list):
        if pi == 0:
            im_input = np.zeros(img.shape + (batch_size,), dtype=theano.config.floatX)
        im_input[:, :, :, pi] = img
        im_label[pi] = label

    param_rand = get_rand3d()
    im_input = crop_and_mirror(im_input, param_rand, flag_batch=False)

    return (im_input, im_label)
Exemple #4
0
def train_model_wrap(train_model,
                     shared_x,
                     shared_y,
                     rand_arr,
                     img_mean,
                     count,
                     minibatch_index,
                     minibatch_range,
                     batch_size,
                     train_filenames,
                     train_labels,
                     flag_para_load,
                     flag_batch,
                     send_queue=None,
                     recv_queue=None):

    # load by self or the other process

    if flag_para_load:
        # wait for the copying to finish
        msg = recv_queue.get()
        assert msg == 'copy_finished'

        if count < len(minibatch_range):
            ind_to_read = minibatch_range[count]
            name_to_read = str(train_filenames[ind_to_read])
            send_queue.put(name_to_read)
            send_queue.put(get_rand3d())

    else:
        batch_img = hkl.load(str(train_filenames[minibatch_index])) - img_mean
        param_rand = get_rand3d()
        batch_img = crop_and_mirror(batch_img,
                                    param_rand,
                                    flag_batch=flag_batch)
        shared_x.set_value(batch_img)

    batch_label = train_labels[minibatch_index *
                               batch_size:(minibatch_index + 1) * batch_size]
    shared_y.set_value(batch_label)

    cost_ij = train_model()
    #cost_ij = train_model(shared_x,shared_y)

    return cost_ij
def get_val_error_loss(rand_arrs, shared_xquery, shared_xp, shared_xns,
                       filenamesA, filenamesB,
                       flag_para_load, img_mean,
                       batch_size, validate_model,
                       send_queue=None, recv_queue=None,
                       flag_top_5=False):

    validation_losses = []
    validation_errors = []
    if flag_top_5:
        validation_errors_top_5 = []

    n_val_batches = len(filenamesA)

    if flag_para_load:
        # send the initial message to load data, before each epoch
        send_queue.put(str(filenamesA[0]))
        send_queue.put(str(filenamesB[0]))
        send_queue.put(np.float32([0.5, 0.5, 0]))
        send_queue.put('calc_finished')

    for val_index in xrange(n_val_batches):

        if flag_para_load:
            # load by self or the other process

            # wait for the copying to finish
            msg = recv_queue.get()
            assert msg == 'copy_finished'

            if val_index + 1 < n_val_batches:
                name_to_read1 = str(filenamesA[val_index + 1])
                send_queue.put(name_to_read1)
                send_queue.put(np.float32([0.5, 0.5, 0]))

                name_to_read2 = str(filenamesB[val_index + 1])
                send_queue.put(name_to_read2)
                send_queue.put(np.float32([0.5, 0.5, 0]))
        else:
            img1 = hkl.load(str(filenamesA[val_index])) - img_mean            
            img2 = hkl.load(str(filenamesB[val_index])) - img_mean            
            param_rand = [0.5,0.5,0]              
            img1 = crop_and_mirror(img1, param_rand, flag_batch=True)
            img2 = crop_and_mirror(img2, param_rand, flag_batch=True)
            shared_xquery.set_value(img1)
            shared_xp.set_value(img2)
            # set false colour variant matches
            for i in xrange(len(shared_xns)):
                shared_xn = shared_xns[i]
                N = img2.shape[3]
                n_indexes = np.arange(N) + np.random.randint(1,N-5)
                n_indexes %= N
                imgN = img2[:,:,:,n_indexes]
                shared_xn.set_value(imgN)

        if flag_top_5:
            loss, error, error_top_5 = validate_model()
        else:
            loss, error = validate_model()


        if flag_para_load and (val_index + 1 < n_val_batches):
            send_queue.put('calc_finished')

        validation_losses.append(loss)
        validation_errors.append(error)

        if flag_top_5:
            validation_errors_top_5.append(error_top_5)

    this_validation_loss = np.mean(validation_losses)
    this_validation_error = np.mean(validation_errors)

    #if flag_top_5:
    #    this_validation_error_top_5 = np.mean(validation_errors_top_5)
    #    return this_validation_error, this_validation_error_top_5, this_validation_loss
    #else:
    return this_validation_error, this_validation_loss
Exemple #6
0
def get_val_error_loss(rand_arr,
                       shared_x,
                       shared_y,
                       val_filenames,
                       val_labels,
                       flag_para_load,
                       img_mean,
                       batch_size,
                       validate_model,
                       send_queue=None,
                       recv_queue=None,
                       flag_top_5=False):

    validation_losses = []
    validation_errors = []
    if flag_top_5:
        validation_errors_top_5 = []

    n_val_batches = len(val_filenames)

    if flag_para_load:
        # send the initial message to load data, before each epoch
        send_queue.put(str(val_filenames[0]))
        send_queue.put(np.float32([0.5, 0.5, 0]))
        send_queue.put('calc_finished')
    print('n_val_batches ', n_val_batches)
    for val_index in range(n_val_batches):

        if flag_para_load:
            # load by self or the other process

            # wait for the copying to finish
            msg = recv_queue.get()
            assert msg == 'copy_finished'

            if val_index + 1 < n_val_batches:
                name_to_read = str(val_filenames[val_index + 1])
                send_queue.put(name_to_read)
                send_queue.put(np.float32([0.5, 0.5, 0]))
        else:
            val_img = hkl.load(str(val_filenames[val_index])) - img_mean
            param_rand = [0.5, 0.5, 0]
            val_img = crop_and_mirror(val_img, param_rand, flag_batch=True)
            shared_x.set_value(val_img)

        shared_y.set_value(val_labels[val_index * batch_size:(val_index + 1) *
                                      batch_size])

        if flag_top_5:
            loss, error, error_top_5 = validate_model()
        else:
            loss, error = validate_model()

        if flag_para_load and (val_index + 1 < n_val_batches):
            send_queue.put('calc_finished')
        # print loss, error
        validation_losses.append(loss)
        validation_errors.append(error)

        if flag_top_5:
            validation_errors_top_5.append(error_top_5)

    this_validation_loss = np.mean(validation_losses)
    this_validation_error = np.mean(validation_errors)
    if flag_top_5:
        this_validation_error_top_5 = np.mean(validation_errors_top_5)
        return this_validation_error, this_validation_error_top_5, this_validation_loss
    else:
        return this_validation_error, this_validation_loss
Exemple #7
0
def get_val_error_loss(rand_arr, shared_x, shared_y,
                       val_filenames, val_labels,
                       flag_para_load, img_mean,
                       batch_size, validate_model,
                       send_queue=None, recv_queue=None,
                       flag_top_5=False):

    validation_losses = []
    validation_errors = []
    if flag_top_5:
        validation_errors_top_5 = []

    n_val_batches = len(val_filenames)

    if flag_para_load:
        # send the initial message to load data, before each epoch
        send_queue.put(str(val_filenames[0]))
        send_queue.put(np.float32([0.5, 0.5, 0]))
        send_queue.put('calc_finished')
    print ('n_val_batches ', n_val_batches)
    for val_index in range(n_val_batches):

        if flag_para_load:
            # load by self or the other process

            # wait for the copying to finish
            msg = recv_queue.get()
            assert msg == 'copy_finished'

            if val_index + 1 < n_val_batches:
                name_to_read = str(val_filenames[val_index + 1])
                send_queue.put(name_to_read)
                send_queue.put(np.float32([0.5, 0.5, 0]))
        else:
            val_img = hkl.load(str(val_filenames[val_index])) - img_mean            
            param_rand = [0.5,0.5,0]              
            val_img = crop_and_mirror(val_img, param_rand, flag_batch=True)
            shared_x.set_value(val_img)

        shared_y.set_value(val_labels[val_index * batch_size:
                                      (val_index + 1) * batch_size])

        if flag_top_5:
            loss, error, error_top_5 = validate_model()
        else:
            loss, error = validate_model()


        if flag_para_load and (val_index + 1 < n_val_batches):
            send_queue.put('calc_finished')
        # print loss, error
        validation_losses.append(loss)
        validation_errors.append(error)

        if flag_top_5:
            validation_errors_top_5.append(error_top_5)

    this_validation_loss = np.mean(validation_losses)
    this_validation_error = np.mean(validation_errors)
    if flag_top_5:
        this_validation_error_top_5 = np.mean(validation_errors_top_5)
        return this_validation_error, this_validation_error_top_5, this_validation_loss
    else:
        return this_validation_error, this_validation_loss