def read_data(filename,filename_v): texts = [] # list of text samples labels_index = {} # dictionary mapping label name to numeric id labels = [] # list of label ids texts_gen=[] # labels_index['0'] = 0 diction=dict() ret=dict() i=0 with open(filename_v, 'r', encoding="utf-8") as fv: for l in fv: if i>0: data = tf.compat.as_str(l.strip()).split(',') diction[int(data[0])]=[data[1],data[2],int(data[3])] i += 1 i = 0 with open(filename, 'r', encoding="utf-8") as f: for line in f: if i>0 and len(line)>100: id = int(line[:line.find('||')]) t_w = text_to_word_sequence(line[line.find('||') + 2:]) text_gen = re.sub('[^a-z^0-9]','',diction[id][0].lower())+' '+re.sub('[^a-z^0-9]','',diction[id][1].lower()) if len(t_w)>MAX_NB_WORDS_IN_TEXT: for text_i in range(0,len(t_w),MAX_NB_WORDS_IN_TEXT): if text_i+MAX_NB_WORDS_IN_TEXT-1<len(t_w): texts.append(' '.join(t_w[text_i:text_i+MAX_NB_WORDS_IN_TEXT-1])) labels.append(diction[id][2]-1) texts_gen.append(text_gen) elif len(t_w)-text_i>50: texts.append(' '.join(t_w[text_i:len(t_w)])) labels.append(diction[id][2]-1) texts_gen.append(text_gen) else: texts.append(' '.join(t_w)) labels.append(diction[id][2]-1) texts_gen.append(text_gen) labels_index[str(diction[id][2])]=diction[id][2]-1 i+=1 if i>=NUM_ROWS_FROM_TEXT : break return texts,texts_gen,labels,labels_index
with open(filename, 'r', encoding="utf-8") as f: with open(os.path.join(SAVE_DIR, 'submissionFile'), 'a') as sf: with open(os.path.join(SAVE_DIR, 'submissionFile_average'), 'a') as sfa: sf.write('ID,class1,class2,class3,class4,class5,class6,class7,class8,class9\n') sfa.write('ID,class1,class2,class3,class4,class5,class6,class7,class8,class9\n') for line in f: if i>0: text= line[line.find('||') + 2:] id =int(line[:line.find('||')]) t_w = text_to_word_sequence(text) outputstr_m = str(id) outputstr_a = str(id) predict_list = [] # ---------- if len(t_w) > MAX_NB_WORDS_IN_TEXT: for text_i in range(0, len(t_w), MAX_NB_WORDS_IN_TEXT): if text_i + MAX_NB_WORDS_IN_TEXT - 1 < len(t_w): t_w_splited=t_w[text_i:text_i + MAX_NB_WORDS_IN_TEXT - 1] predict_list.append(predict(t_w_splited, id, word_index, model, sf,model_shape1)) elif len(t_w) - text_i > 50: t_w_splited=t_w[text_i:len(t_w)] predict_list.append(predict(t_w_splited, id, word_index, model, sf,model_shape1)) else: predict_list.append(predict(t_w, id, word_index, model, sf,model_shape1))