def ei_reg_scores(lang, submissions, metrics, ref_files):
"""calculates metrics for ei-reg tasks """
anger_scores = evaluate_ei(submissions['ei-reg-' + lang + '-anger'],
ref_files['ei_reg_' + lang + '_anger'])
fear_scores = evaluate_ei(submissions['ei-reg-' + lang + '-fear'],
ref_files['ei_reg_' + lang + '_fear'])
joy_scores = evaluate_ei(submissions['ei-reg-' + lang + '-joy'],
ref_files['ei_reg_' + lang + '_joy'])
sadness_scores = evaluate_ei(submissions['ei-reg-' + lang + '-sadness'],
ref_files['ei_reg_' + lang + '_sadness'])
metrics['r_ei_anger_' + lang] = anger_scores[0]
metrics['r_ei_fear_' + lang] = fear_scores[0]
metrics['r_ei_joy_' + lang] = joy_scores[0]
metrics['r_ei_sadness_' + lang] = sadness_scores[0]
metrics['r_ei_macro_avg_' + lang] = numpy.mean(
[anger_scores[0], fear_scores[0], joy_scores[0], sadness_scores[0]])
metrics['r_05_1_ei_anger_' + lang] = anger_scores[1]
metrics['r_05_1_ei_fear_' + lang] = fear_scores[1]
metrics['r_05_1_ei_joy_' + lang] = joy_scores[1]
metrics['r_05_1_ei_sadness_' + lang] = sadness_scores[1]
metrics['r_05_1_ei_macro_avg_' + lang] = numpy.mean(
[anger_scores[1], fear_scores[1], joy_scores[1], sadness_scores[1]])
def main(argv):
"""main method """
if len(argv)!=3:
raise ValueError('Invalid number of parameters.')
task_type=int(argv[0])
pred=argv[1]
gold=argv[2]
if(task_type==1):
result=evaluate_ei(pred,gold)
print ("Pearson correlation between "+os.path.basename(pred)+" and "+os.path.basename(gold)+":\t"+str(result[0]) )
print ("Pearson correlation for gold scores in range 0.5-1 between "+os.path.basename(pred)+" and "+os.path.basename(gold)+":\t"+str(result[1]))
elif(task_type==2):
result=evaluate_oc(pred,gold)
print ("Pearson correlation between "+os.path.basename(pred)+" and "+os.path.basename(gold)+":\t"+str(result[0]))
print ("Pearson correlation for some emotions between "+os.path.basename(pred)+" and "+os.path.basename(gold)+":\t"+str(result[1]))
print ("Weighted quadratic Kappa between "+os.path.basename(pred)+" and "+os.path.basename(gold)+":\t"+str(result[2]))
print ("Weighted quadratic Kappa for some emotions between "+os.path.basename(pred)+" and "+os.path.basename(gold)+":\t"+str(result[3]))
else:
result=evaluate_multilabel(pred,gold)
print ("Multi-label accuracy (Jaccard index) between "+os.path.basename(pred)+" and "+os.path.basename(gold)+":\t"+str(result[0]))
print ("Micro-averaged F1 score between "+os.path.basename(pred)+" and "+os.path.basename(gold)+":\t"+str(result[1]))
print ("Macro-averaged F1 score between "+os.path.basename(pred)+" and "+os.path.basename(gold)+":\t"+str(result[2]))
def v_reg_scores(lang, submissions, metrics, ref_files):
"""calculates metrics for ei-reg valence tasks """
valence_scores = evaluate_ei(submissions['v-reg-' + lang],
ref_files['valence_reg_' + lang])
metrics['r_reg_valence_' + lang] = valence_scores[0]
metrics['r_05_1_reg_valence_' + lang] = valence_scores[1]