Exemplo n.º 1
0
def evaluateForAllThresholds(path_to_dataset, path_to_checkpoint, thresh_params, model_type="multipleRNNs"):
    """
	Input: 
	path_to_dataset: Complete path to the pickle file containing data
	path_to_checkpoint: Complete path to the checkpoint to evaluate at
	thresh_params: The list of threshold values to  evaluate at
	model_type: the kind of model to load. This is the same model using which the checkpoint was created

	This function evaluates (data,checkpoint) for all the values of threshold specified in thresh_params
	"""
    test_data = cPickle.load(open(path_to_dataset))
    Y_te = test_data["labels"]
    X_te = test_data["features"]
    actions = []
    if test_data.has_key("actions"):
        actions = test_data["actions"]
    else:
        actions = ["end_action", "lchange", "rchange", "lturn", "rturn"]

    rnn = []
    if model_type == "multipleRNNs":
        rnn = loadMultipleRNNsCombined(path_to_checkpoint)
    else:
        rnn = load(path_to_checkpoint)

    precision = []
    recall = []
    time_before_maneuver = []

    for th in thresh_params:
        with open("settings.py", "w") as f:
            f.write("OUTPUT_THRESH = %f \n" % th)
        print "Generating results for th= ", th
        predictions = []
        errors = 0
        N = 0
        P = []
        Y = []
        Time_before_maneuver = []

        for xte, yte in zip(X_te, Y_te):
            inputD = xte.shape[2]
            road_feature_dimension = 4
            prediction = []

            if model_type == "multipleRNNs":
                prediction = rnn.predict_output(
                    [xte[:, :, (inputD - road_feature_dimension) :], xte[:, :, : (inputD - road_feature_dimension)]],
                    OutputActionThresh,
                )
                # [:,:,:(inputD-road_feature_dimension)]
            else:
                prediction = rnn.predict_output(xte, OutputActionThresh)

                # print prediction.T
            predictions.append(prediction)
            t = np.nonzero(yte - prediction)

            # Label 0 is the dummy label. Currently maneuvers are labeled [1..n]
            prediction = prediction[:, 0]
            yte_ = copy.deepcopy(yte)
            actual = yte_[:, 0]
            prediction[prediction > 0] -= 1
            actual[actual > 0] -= 1

            p, anticipation_time = predictManeuver(prediction, actions)
            y = actual[-1]
            P.append(p)
            Y.append(y)
            Time_before_maneuver.append(anticipation_time)
            result = {"actual": y, "prediction": p, "timeseries": list(prediction)}
            # print result.values()
            errors += len(t[0])
            N += yte.shape[0]

        P = np.array(P)
        Y = np.array(Y)
        Time_before_maneuver = np.array(Time_before_maneuver)
        [conMat, p_mat, re_mat, time_mat] = confusionMat(P, Y, Time_before_maneuver)
        precision.append(np.mean(np.diag(p_mat)[1:]))
        recall.append(np.mean(np.diag(re_mat)[1:]))
        time_before_maneuver.append(np.mean(np.divide(np.diag(time_mat)[1:], np.diag(conMat)[1:])))

    return np.array(precision), np.array(recall), np.array(time_before_maneuver)
Exemplo n.º 2
0
def evaluate(path_to_dataset, path_to_checkpoint, model_type="multipleRNNs"):
    """
	Input: 
	path_to_dataset: Complete path to the pickle file containing data
	path_to_checkpoint: Complete path to the checkpoint to evaluate at
	model_type: the kind of model to load. This is the same model using which the checkpoint was created

	Before running this function make sure that the threshold at which you want to evaluate is written in settings.py file. 
	"""

    test_data = cPickle.load(open(path_to_dataset))
    Y_te = test_data["labels"]
    X_te = test_data["features"]
    actions = []
    if test_data.has_key("actions"):
        actions = test_data["actions"]
    else:
        actions = ["end_action", "lchange", "rchange", "lturn", "rturn"]

    rnn = []
    if model_type == "multipleRNNs":
        rnn = loadMultipleRNNsCombined(path_to_checkpoint)
    else:
        rnn = load(path_to_checkpoint)

    print "Number of parameters {0}".format(rnn.num_params)

    conMat = {}
    p_mat = {}
    re_mat = {}
    time_mat = {}

    P = {}
    Y = {}
    for i in range(10):
        P[i] = []
        Y[i] = []
        conMat[i] = []
        p_mat[i] = []
        re_mat[i] = []
        time_mat[i] = []
    i = "best"
    P[i] = []
    Y[i] = []
    conMat[i] = []
    p_mat[i] = []
    re_mat[i] = []
    time_mat[i] = []

    Time_before_maneuver = []
    for xte, yte in zip(X_te, Y_te):
        inputD = xte.shape[2]
        road_feature_dimension = 4
        prediction = []

        if model_type == "multipleRNNs":
            prediction = rnn.predict_output(
                [xte[:, :, (inputD - road_feature_dimension) :], xte[:, :, : (inputD - road_feature_dimension)]],
                OutputActionThresh,
            )
            # [:,:,:(inputD-road_feature_dimension)]
        else:
            prediction = rnn.predict_output(xte, OutputActionThresh)

            # Label 0 is the dummy label. Currently maneuvers are labeled [1..n]
        prediction = prediction[:, 0]
        actual = yte[:, 0]
        prediction[prediction > 0] -= 1
        actual[actual > 0] -= 1
        y = actual[-1]

        iter_ = 10 - len(prediction)
        for p in prediction:
            P[iter_].append(p)
            Y[iter_].append(y)
            iter_ += 1

        p, anticipation_time = predictManeuver(prediction, actions)
        P["best"].append(p)
        Y["best"].append(y)
        Time_before_maneuver.append(anticipation_time)

    Time_before_maneuver = np.array(Time_before_maneuver)
    for k in P.keys():
        if len(P[k]) == 0:
            continue
        P[k] = np.array(P[k])
        Y[k] = np.array(Y[k])
        [conMat_, p_mat_, re_mat_, time_mat_] = confusionMat(P[k], Y[k], Time_before_maneuver)

        conMat[k] = conMat_
        p_mat[k] = p_mat_
        re_mat[k] = re_mat_
        time_mat[k] = time_mat_

    return conMat, p_mat, re_mat, time_mat
Exemplo n.º 3
0
def evaluate(path_to_dataset, path_to_checkpoint, model_type='multipleRNNs'):
    '''
	Input: 
	path_to_dataset: Complete path to the pickle file containing data
	path_to_checkpoint: Complete path to the checkpoint to evaluate at
	model_type: the kind of model to load. This is the same model using which the checkpoint was created

	Before running this function make sure that the threshold at which you want to evaluate is written in settings.py file. 
	'''

    test_data = cPickle.load(open(path_to_dataset))
    Y_te = test_data['labels']
    X_te = test_data['features']
    actions = []
    if test_data.has_key('actions'):
        actions = test_data['actions']
    else:
        actions = ['end_action', 'lchange', 'rchange', 'lturn', 'rturn']

    rnn = []
    if model_type == 'multipleRNNs':
        rnn = loadMultipleRNNsCombined(path_to_checkpoint)
    else:
        rnn = load(path_to_checkpoint)

    print "Number of parameters {0}".format(rnn.num_params)

    conMat = {}
    p_mat = {}
    re_mat = {}
    time_mat = {}

    P = {}
    Y = {}
    for i in range(10):
        P[i] = []
        Y[i] = []
        conMat[i] = []
        p_mat[i] = []
        re_mat[i] = []
        time_mat[i] = []
    i = 'best'
    P[i] = []
    Y[i] = []
    conMat[i] = []
    p_mat[i] = []
    re_mat[i] = []
    time_mat[i] = []

    Time_before_maneuver = []
    for xte, yte in zip(X_te, Y_te):
        inputD = xte.shape[2]
        road_feature_dimension = 4
        prediction = []

        if model_type == 'multipleRNNs':
            prediction = rnn.predict_output([
                xte[:, :, (inputD - road_feature_dimension):],
                xte[:, :, :(inputD - road_feature_dimension)]
            ], OutputActionThresh)
            #[:,:,:(inputD-road_feature_dimension)]
        else:
            prediction = rnn.predict_output(xte, OutputActionThresh)

        # Label 0 is the dummy label. Currently maneuvers are labeled [1..n]
        prediction = prediction[:, 0]
        actual = yte[:, 0]
        prediction[prediction > 0] -= 1
        actual[actual > 0] -= 1
        y = actual[-1]

        iter_ = 10 - len(prediction)
        for p in prediction:
            P[iter_].append(p)
            Y[iter_].append(y)
            iter_ += 1

        p, anticipation_time = predictManeuver(prediction, actions)
        P['best'].append(p)
        Y['best'].append(y)
        Time_before_maneuver.append(anticipation_time)

    Time_before_maneuver = np.array(Time_before_maneuver)
    for k in P.keys():
        if len(P[k]) == 0:
            continue
        P[k] = np.array(P[k])
        Y[k] = np.array(Y[k])
        [conMat_, p_mat_, re_mat_,
         time_mat_] = confusionMat(P[k], Y[k], Time_before_maneuver)

        conMat[k] = conMat_
        p_mat[k] = p_mat_
        re_mat[k] = re_mat_
        time_mat[k] = time_mat_

    return conMat, p_mat, re_mat, time_mat
Exemplo n.º 4
0
def evaluateForAllThresholds(path_to_dataset,
                             path_to_checkpoint,
                             thresh_params,
                             model_type='multipleRNNs'):
    '''
	Input: 
	path_to_dataset: Complete path to the pickle file containing data
	path_to_checkpoint: Complete path to the checkpoint to evaluate at
	thresh_params: The list of threshold values to  evaluate at
	model_type: the kind of model to load. This is the same model using which the checkpoint was created

	This function evaluates (data,checkpoint) for all the values of threshold specified in thresh_params
	'''
    test_data = cPickle.load(open(path_to_dataset))
    Y_te = test_data['labels']
    X_te = test_data['features']
    actions = []
    if test_data.has_key('actions'):
        actions = test_data['actions']
    else:
        actions = ['end_action', 'lchange', 'rchange', 'lturn', 'rturn']

    rnn = []
    if model_type == 'multipleRNNs':
        rnn = loadMultipleRNNsCombined(path_to_checkpoint)
    else:
        rnn = load(path_to_checkpoint)

    precision = []
    recall = []
    time_before_maneuver = []

    for th in thresh_params:
        with open('settings.py', 'w') as f:
            f.write('OUTPUT_THRESH = %f \n' % th)
        print "Generating results for th= ", th
        predictions = []
        errors = 0
        N = 0
        P = []
        Y = []
        Time_before_maneuver = []

        for xte, yte in zip(X_te, Y_te):
            inputD = xte.shape[2]
            road_feature_dimension = 4
            prediction = []

            if model_type == 'multipleRNNs':
                prediction = rnn.predict_output([
                    xte[:, :, (inputD - road_feature_dimension):],
                    xte[:, :, :(inputD - road_feature_dimension)]
                ], OutputActionThresh)
                #[:,:,:(inputD-road_feature_dimension)]
            else:
                prediction = rnn.predict_output(xte, OutputActionThresh)

            #print prediction.T
            predictions.append(prediction)
            t = np.nonzero(yte - prediction)

            # Label 0 is the dummy label. Currently maneuvers are labeled [1..n]
            prediction = prediction[:, 0]
            yte_ = copy.deepcopy(yte)
            actual = yte_[:, 0]
            prediction[prediction > 0] -= 1
            actual[actual > 0] -= 1

            p, anticipation_time = predictManeuver(prediction, actions)
            y = actual[-1]
            P.append(p)
            Y.append(y)
            Time_before_maneuver.append(anticipation_time)
            result = {
                'actual': y,
                'prediction': p,
                'timeseries': list(prediction)
            }
            #print result.values()
            errors += len(t[0])
            N += yte.shape[0]

        P = np.array(P)
        Y = np.array(Y)
        Time_before_maneuver = np.array(Time_before_maneuver)
        [conMat, p_mat, re_mat,
         time_mat] = confusionMat(P, Y, Time_before_maneuver)
        precision.append(np.mean(np.diag(p_mat)[1:]))
        recall.append(np.mean(np.diag(re_mat)[1:]))
        time_before_maneuver.append(
            np.mean(np.divide(np.diag(time_mat)[1:],
                              np.diag(conMat)[1:])))

    return np.array(precision), np.array(recall), np.array(
        time_before_maneuver)
Exemplo n.º 5
0
def evaluate(index,fold,checkpoint,model_type='lstm_one_layer',path_to_load_from=''):
	path_to_dataset = '/scr/ashesh/brain4cars/dataset/{0}'.format(fold)

	if len(path_to_load_from) > 0:
		path_to_dataset = path_to_load_from

	path_to_checkpoints = '/scr/ashesh/brain4cars/checkpoints/{0}'.format(fold)
	test_data = cPickle.load(open('{1}/test_data_{0}.pik'.format(index,path_to_dataset)))	

	Y_te = test_data['labels']
	X_te = test_data['features']
	actions = []
	if test_data.has_key('actions'):
		actions = test_data['actions']
	else:
		actions = ['end_action','lchange','rchange','lturn','rturn']

	# Prediction
	rnn = []
	if model_type == 'multipleRNNs':
		if len(path_to_load_from) > 0:
			rnn = loadMultipleRNNsCombined('{0}/checkpoint.{1}'.format(path_to_load_from,checkpoint))
		else:
			rnn = loadMultipleRNNsCombined('{2}/{0}/checkpoint.{1}'.format(index,checkpoint,path_to_checkpoints))
	else:
		if len(path_to_load_from) > 0:
			rnn = load('{0}/checkpoint.{1}'.format(path_to_load_from,checkpoint))
		else:
			rnn = load('{2}/{0}/checkpoint.{1}'.format(index,checkpoint,path_to_checkpoints))

	predictions = []
	errors = 0
	N = 0
	P = []
	Y = []
	Time_before_maneuver = []
	for xte,yte in zip(X_te,Y_te):
		inputD = xte.shape[2]
		road_feature_dimension = 4
		prediction = []

		if model_type == 'multipleRNNs':
			prediction = rnn.predict_output([xte[:,:,(inputD-road_feature_dimension):],xte[:,:,:(inputD-road_feature_dimension)]],OutputActionThresh)
			#[:,:,:(inputD-road_feature_dimension)]
		else:
			prediction = rnn.predict_output(xte,OutputActionThresh)

		#print prediction.T
		predictions.append(prediction)
		t = np.nonzero(yte-prediction)
	
		# Label 0 is the dummy label. Currently maneuvers are labeled [1..n]	
		prediction = prediction[:,0]
		actual = yte[:,0]
		prediction[prediction>0] -= 1
		actual[actual>0] -= 1
		
		p,anticipation_time = predictManeuver(prediction,actions)
		y = actual[-1]
		P.append(p)
		Y.append(y)
		Time_before_maneuver.append(anticipation_time)
		result = {'actual':y,'prediction':p,'timeseries':list(prediction)}
		#print result.values()
		errors += len(t[0])
		N += yte.shape[0]
	
	P = np.array(P)
	Y = np.array(Y)
	Time_before_maneuver = np.array(Time_before_maneuver)
	[conMat,p_mat,re_mat,time_mat] = confusionMat(P,Y,Time_before_maneuver)
	return conMat,p_mat,re_mat,time_mat