Exemple #1
0
def test(instances, theta, word_vectors, isPrint=False):
    if isPrint:
        outfile = open('./output/test_result.txt', 'w')
    total_lines = len(instances)
    total_true = 0

    # init rae
    rae = RecursiveAutoencoder.build(theta, embsize)

    offset = RecursiveAutoencoder.compute_parameter_num(embsize)
    delta = ReorderClassifer.compute_parameter_num(embsize)
    rms = []
    for i in range(0, worker_num):
        rm = ReorderClassifer.build(theta[offset:offset+delta], embsize, rae)
        offset += delta
        rms.append(rm)

    for instance in instances:
        words_embedded = word_vectors[instance.preWords]
        root_prePhrase, rec_error = rae.forward(words_embedded)

        words_embedded = word_vectors[instance.aftWords]
        root_aftPhrase, rec_error = rae.forward(words_embedded)

        if isPrint:
            outfile.write("%f" %instance.order)
        prediction = 0
        avg_softmaxLayer = zeros(2)
        for i in range(0, worker_num):
            softmaxLayer, reo_error = rms[i].forward(instance, root_prePhrase.p, root_aftPhrase.p, embsize)
            if isPrint:
                outfile.write("  [%f,%f]" % (softmaxLayer[0], softmaxLayer[1]))
            avg_softmaxLayer += softmaxLayer

        avg_softmaxLayer /= worker_num
        if isPrint:
            outfile.write("\n")

        if instance.order == 1 and avg_softmaxLayer[0] > avg_softmaxLayer[1]:
            total_true += 1
        if instance.order == 0 and avg_softmaxLayer[0] < avg_softmaxLayer[1]:
            total_true += 1

    if isPrint:
        outfile.write("Total instances: %f\tTotal true predictions: %f\t" % (total_lines, total_true))
        outfile.write("Precision: %f" % (float(total_true / total_lines)))
    print("Total instances: %f\tToral true predictions: %f\tPrecision: %f\n" %(total_lines, total_true, float(total_true / total_lines)))
Exemple #2
0
def compute_cost_and_grad(theta, instances, instances_of_Unlabel, word_vectors, embsize, total_internal_node, lambda_rec, lambda_reg, lambda_reo,
                          lambda_unlabel, instances_of_News, is_Test):
    '''Compute the value and gradients of the objective function at theta

    Args:
    theta: model parameter
    instances: training instances
    embsize: word embedding vector size
    lambda_reg: the weight of regularizer
    lambda_reo: the weight of reo

    Returns:
    total_cost: the value of the objective function at theta
    total_grad: the gradients of the objective function at theta
    '''
    if rank == 0:
        # send working signal
        send_working_signal()

        if is_Test:
            #test per iteration
            instances_of_test,_ = prepare_test_data(word_vectors, instances_of_News)
            instances_of_test = random.sample(instances_of_test, 500)
            test(instances_of_test, theta, word_vectors, isPrint=True)
        # init rae
        rae = RecursiveAutoencoder.build(theta, embsize)

        offset = RecursiveAutoencoder.compute_parameter_num(embsize)
        delta = ReorderClassifer.compute_parameter_num(embsize)

        rms = []
        local_rm = ReorderClassifer.build(theta[offset:offset+delta], embsize, rae)
        rms.append(local_rm)
        offset += delta
        for i in range(1, worker_num):
            rm = ReorderClassifer.build(theta[offset:offset + delta], embsize,
                                        rae)
            offset += delta
            comm.send(rae, dest=i)
            comm.send(rm, dest=i)
            rms.append(rm)
        comm.barrier()

        total_rae_rec_grad = zeros(RecursiveAutoencoder.compute_parameter_num(embsize))
        total_rae_grad = zeros(RecursiveAutoencoder.compute_parameter_num(embsize))
        total_rm_grad = zeros(ReorderClassifer.compute_parameter_num(embsize)*worker_num)
        # compute local reconstruction error, reo and gradients
        local_rae_error, local_rm_error,rae_rec_gradient, rae_gradient, rm_gradient = process_local_batch(rm, rae, word_vectors, instances, lambda_rec, lambda_reo)
        local_rm_error /= len(instances)
        rm_gradient /= len(instances)
        rae_gradient /= len(instances)

        total_rae_error = comm.reduce(local_rae_error, op=MPI.SUM, root=0)
        total_rm_error = comm.reduce(local_rm_error, op=MPI.SUM, root=0)
        comm.Reduce([rae_rec_gradient, MPI.DOUBLE], [total_rae_rec_grad, MPI.DOUBLE],
                    op=MPI.SUM, root=0)
        comm.Reduce([rae_gradient, MPI.DOUBLE], [total_rae_grad, MPI.DOUBLE],
                    op=MPI.SUM, root=0)

        total_error = total_rm_error + total_rae_error/total_internal_node
        total_rae_rec_grad /= total_internal_node
        total_rae_grad += total_rae_rec_grad

        total_rm_grad[0:delta] += rm_gradient
        for i in range(1, worker_num):
            local_rm_gradient = comm.recv(source=i)
            total_rm_grad[i*delta:(i+1)*delta] += local_rm_gradient
        comm.barrier()

        # compute unlabeled error and gradients
        local_unlabel_error, unlabel_rae_gradient, unlabel_rm_gradient = process_unlabeled_batch(rms, rae, word_vectors,
                                                                                                 instances_of_Unlabel)

        # compute total cost
        reg = 0
        for i in range(0, worker_num):
            reg += rms[i].get_weights_square()
        reg += rae.get_weights_square()
        final_cost = total_error + lambda_unlabel * local_unlabel_error / len(instances_of_Unlabel) + lambda_reg / 2 * reg

        unlabel_rae_gradient /= len(instances_of_Unlabel)
        unlabel_rm_gradient /= len(instances_of_Unlabel)

        total_rae_grad += lambda_unlabel * unlabel_rae_gradient
        total_rm_grad += lambda_unlabel * unlabel_rm_gradient

        # gradients related to regularizer
        reg_grad = rae.get_zero_gradients()
        reg_grad.gradWi1 += rae.Wi1
        reg_grad.gradWi2 += rae.Wi2
        reg_grad.gradWo1 += rae.Wo1
        reg_grad.gradWo2 += rae.Wo2
        reg_grad *= lambda_reg

        total_rae_grad += reg_grad.to_row_vector()

        for i in range(0, worker_num):
            reg_grad = local_rm.get_zero_gradients()
            reg_grad.gradW1 += rms[i].W1
            reg_grad.gradW2 += rms[i].W2
            reg_grad.gradb1 += rms[i].b1
            reg_grad.gradb2 += rms[i].b2
            reg_grad *= lambda_reg
            total_rm_grad[i*delta:(i+1)*delta] += reg_grad.to_row_vector()

        return final_cost, concatenate((total_rae_grad, total_rm_grad))
    else:
        while True:
            signal = comm.bcast(root=0)
            if isinstance(signal, TerminatorSignal):
                return
            if isinstance(signal, ForceQuitSignal):
                exit(-1)

            rae = comm.recv(source=0)
            local_rm = comm.recv(source=0)
            comm.barrier()

            local_rae_error, local_rm_error,rae_rec_gradient, rae_gradient, rm_gradient = process_local_batch(local_rm, rae, word_vectors, instances, lambda_rec, lambda_reo)
            local_rm_error /= len(instances)
            rae_gradient /= len(instances)
            rm_gradient /= len(instances)

            comm.reduce(local_rae_error, op=MPI.SUM, root=0)
            comm.reduce(local_rm_error, op=MPI.SUM, root=0)
            comm.Reduce([rae_rec_gradient, MPI.DOUBLE], None, op=MPI.SUM, root=0)
            comm.Reduce([rae_gradient, MPI.DOUBLE], None, op=MPI.SUM, root=0)
            comm.send(rm_gradient, dest=0)
            comm.barrier()