def phrase_search(query, inverted_index): phrase_search_result_list = [] query_info = re.findall(r'(\w+)\s"(\w+)\s(\w+)"', query, re.DOTALL) #phrase_list[0] phrase_list[1] #pre_processing returns a list involving only one element. so pre_processing('str')[0]=='str' serial_number = query_info[0][0] first_term = pre_processing(query_info[0][1])[0] second_term = pre_processing(query_info[0][2])[0] if first_term in inverted_index.keys( ) and second_term in inverted_index.keys(): for doc_info_of_first_term in inverted_index[ first_term]: #doc_info[0] docID; doc_info[1] list of positions for doc_info_of_second_term in inverted_index[second_term]: if doc_info_of_first_term[0] == doc_info_of_second_term[0]: for position_of_second_term in doc_info_of_second_term[1]: for position_of_first_term in doc_info_of_first_term[ 1]: if (position_of_second_term - position_of_first_term) == 1: phrase_search_result_list.append( doc_info_of_first_term[0]) break else: continue break break list_of_docid = [int(docid) for docid in phrase_search_result_list] with open("results.boolean.txt", "a") as written_file: for docid in sorted(list_of_docid): print("%s,%s" % (serial_number, docid), file=written_file)
def load_detection_images(): total_files = glob(DETECTION_PATH + "*.jpg") img1 = pre_processing(total_files[0]) img2 = pre_processing(total_files[1]) return img1, img2
def proximity_search(query, inverted_index): query_info = re.findall(r"(\w+)\s#(\d+)\((\w+),\s?(\w+)\)", query, re.DOTALL) list_of_docid = [] #phrase_list[0] phrase_list[1] #query_info is a list which involve a 3-D tuple [(proximity_num, first_term, second_term)] serial_number = query_info[0][0] proximity_num = int(query_info[0][1]) # str->int first_term = pre_processing(query_info[0][2])[0] second_term = pre_processing(query_info[0][3])[0] if first_term in inverted_index.keys( ) and second_term in inverted_index.keys(): for doc_info_of_first_term in inverted_index[ first_term]: #doc_info[0] docID; doc_info[1] list of positions for doc_info_of_second_term in inverted_index[second_term]: if doc_info_of_first_term[0] == doc_info_of_second_term[0]: for position_of_second_term in doc_info_of_second_term[1]: for position_of_first_term in doc_info_of_first_term[ 1]: if abs(position_of_second_term - position_of_first_term) <= proximity_num: list_of_docid.append(doc_info_of_first_term[0]) #with open("results.boolean.txt", "a") as written_file: #print("%s, %s" %(serial_number, doc_info_of_first_term[0]), file=written_file) break else: continue break break list_of_docid = [int(docid) for docid in list_of_docid] if len(list_of_docid) != 0: with open("results.boolean.txt", "a") as written_file: for docid in sorted(list_of_docid): print("%s,%s" % (serial_number, docid), file=written_file)
def term_boolean_term(query, inverted_index): query_info = re.findall(r"(\w+)\s(\w+)\s(.+)\s(\w+)", query, re.DOTALL) serial_number = query_info[0][0] first_term = pre_processing(query_info[0][1])[0] operator = query_info[0][2] second_term = pre_processing(query_info[0][3])[0] if first_term in inverted_index.keys( ) and second_term in inverted_index.keys(): list_of_first_term = [] list_of_second_term = [] for doc_info_of_first_term in inverted_index[ first_term]: #doc_info_of_X_term [0] docID; doc_info_of_X_term[1] list of positions list_of_first_term.append(doc_info_of_first_term[0]) for doc_info_of_second_term in inverted_index[second_term]: list_of_second_term.append(doc_info_of_second_term[0]) if operator == "AND": list_of_docid = [ docid for docid in list_of_first_term if docid in list_of_second_term ] if operator == "OR": list_of_docid = list(set(list_of_first_term + list_of_second_term)) if operator == "AND NOT": list_of_docid = [ docid for docid in list_of_first_term if docid not in list_of_second_term ] #print the search result if operator == 'OR NOT': all_docid = [] xml_content = extract_xml_file( "trec.5000.xml" ) # single_file[0]: docid; single_file[1]: headline+text for single_file in xml_content: all_docid.append(single_file[0]) not_second_term_docid = [ x for x in all_docid if x not in list_of_second_term ] list_of_docid = list( set(list_of_first_term + not_second_term_docid)) list_of_docid = [int(docid) for docid in list_of_docid] if len(list_of_docid) != 0: with open("results.boolean.txt", "a") as written_file: for docid in sorted(list_of_docid): print("%s,%s" % (serial_number, docid), file=written_file)
def positional_inverted_index(): file_tokens_all = [] file_tokens = [] file_info = [] xml_content = extract_xml_file("trec.5000.xml") for single_file in xml_content: # headline + text format of file_info: [(docid, [tokens of headline + text]),(),(),(),()] file_tokens_all += pre_processing(single_file[1]) + pre_processing( single_file[2]) file_tokens = pre_processing(single_file[1]) + pre_processing( single_file[2]) file_info.append((single_file[0], file_tokens)) inverted_index = dict() set_tokens = set(file_tokens_all) for term in set_tokens: list_of_term_inverted_index = [] for single_file in file_info: #single_file[0]: docID single_file[1]: headline single_file[2]: content #single_file[0]:doc_id single_file[1]: single_file_tokens if term in single_file[1]: list_of_positions = [] for i in range(1, len(single_file[1]) + 1): if term == single_file[1][i - 1]: list_of_positions.append(i) list_of_term_inverted_index.append( (single_file[0], list_of_positions)) #the list of tuples [(), (), ()] inverted_index[term] = list_of_term_inverted_index #Reference: doc_info[0] = docID ; doc_info[1] = list of positions with open("index.txt", "w") as written_file: for key_of_term in sorted(inverted_index): print("%s:%i" % (key_of_term, len(inverted_index[key_of_term])), file=written_file) for doc_info in inverted_index[key_of_term]: print("\t%s:" % doc_info[0], ",".join(str(i) for i in doc_info[1]), file=written_file) #why is %s print(file=written_file) return inverted_index
def browsefunc(): global lines lines = [] filename = filedialog.askopenfilename() x.append(filename) print(x[i]) lines = preprocessing.pre_processing(x[i]) print("preparing image .... Done") global i i += 1
def single_term_search(query, inverted_index): query_info = re.findall(r"(\w+)\s(\w+)", query, re.DOTALL) serial_number = query_info[0][0] search_term = pre_processing(query_info[0][1])[0] search_term_list = [] if search_term in inverted_index.keys(): for doc_info_of_search_term in inverted_index[search_term]: search_term_list.append(doc_info_of_search_term[0]) #if len(search_term_list) != 0: with open("results.boolean.txt", "a") as written_file: for docid in search_term_list: print("%s,%s" % (serial_number, docid), file=written_file)
def main(path,pre,mode): petrinet, init, targets = load_petrinet(path) red_petrinet = petrinet red_init = init pre_processing_res = None # if no pre-preprocessing if pre: time_before = time.time() print "**** pre processing ***********" red_petrinet, red_init, targets,pre_processing_res = pre_processing(petrinet,init,targets) petrinet = red_petrinet init = red_init time_after = time.time() print "pre_processing finished in", time_after - time_before print "*******************************" if pre_processing_res: print "The pre-processing found that one target was coverable" print "Unsafe" exit(0) elif pre_processing_res == False: print "The pre-processing found that all targets were uncoverable" print "Safe" exit(0) # know we will launch the backward coverability algorithm if mode == "qcover": result = coverability(petrinet, init, targets, prune=True,max_iter=None) elif mode == "limit": result = limit_coverability(petrinet, init, targets, prune=True,max_iter=None) elif mode == "comp": result = comparable_coverability(petrinet, init, targets,prune=True,max_iter=None) elif mode == "hfifos": result = limit_coverability_exploration_heuristic(petrinet, init, targets, "fifos",prune=True,max_iter=None) elif mode == "hstacks": result = limit_coverability_exploration_heuristic(petrinet, init, targets, "stacks",prune=True,max_iter=None) else: print "wrong mode" print "choose qcover, limit or comp, hfifos or hstacks" exit(2) if result is None: print 'Unknown' elif result: print 'Unsafe' else: print 'Safe'
def tfidf_process(inverted_index): N = 5000 #total number of documents tfidf = {} file_info = {} xml_content = extract_xml_file("trec.5000.xml") for single_file in xml_content: # headline + text format of file_info: [(docid, [tokens of headline + text]),(),(),(),()] file_tokens = pre_processing(single_file[1]) + pre_processing( single_file[2]) file_info[single_file[0]] = file_tokens # calculate tfidf for term in inverted_index.keys( ): # inverted_index[term] len(inverted_index[term]) is the length idf_part = math.log(N / len(inverted_index[term]), 10) tfidf[term] = {} for term_doc in inverted_index[ term]: # term_doc[0]: docid term_doc[1] list of positions tr: len(term_doc[1])/len(file_info[term_doc[0]]) tf_part = 1 + math.log(len(term_doc[1]), 10) #N/len(inverted_index[term]) idf: math.log(N/len(inverted_index[term]), 10) #w(t.d) = tf_part*idf_part tfidf[term][term_doc[0]] = tf_part * idf_part return tfidf
def tfidf_retrieval(ranked_query, tfidf): query_info = re.findall(r"(\d+)\s([\sA-Za-z]+)", ranked_query, re.DOTALL) serial_number = query_info[0][0] query_content = query_info[0][1] query = pre_processing(query_content) term_doc_list = [] #this part is designed to process the input for query_term in query: #Get a list of docid involing any number of terms of the query term_doc_list += tfidf[query_term].keys() doc_result = list(set(term_doc_list)) docid_score_list = [] for docid in doc_result: docid_score = 0 for query_term in query: if docid in tfidf[query_term].keys(): docid_score += tfidf[query_term][docid] docid_score_list.append((int(docid), docid_score)) docid_score_list = sorted(docid_score_list, key=lambda x: (-x[1], x[0])) # [(,)(,)(,)] if len(docid_score_list) <= 150: with open("results.ranked.txt", "a") as written_file: for doc_score in docid_score_list: #doc_score[0]:docid doc_score[1]: tfidf score print("%s,%s,%.4f" % (serial_number, doc_score[0], doc_score[1]), file=written_file) #if the length of the list is greater than 150, we control it to only print out the first 150 elements. if len(docid_score_list) > 150: with open("results.ranked.txt", "a") as written_file: for i in range(150): print("%s,%s,%.4f" % (serial_number, docid_score_list[i][0], docid_score_list[i][1]), file=written_file)
def phrase_boolean_term(query, inverted_index): phrase_search_result_list = [] term_search_result_list = [] query_info = re.findall(r'(\w+)\s"(\w+)\s(\w+)"\s(.+)\s(\w+)', query, re.DOTALL) serial_number = query_info[0][0] first_term_of_phrase = pre_processing(query_info[0][1])[0] second_term_of_phrase = pre_processing(query_info[0][2])[0] operator = query_info[0][3] second_term = pre_processing(query_info[0][4])[0] #query result for phrase #ATTENTION: NEED TO EDIT AGAIN usage: calculate the phrase search result list phrase_search_result_list = [] if first_term_of_phrase in inverted_index.keys( ) and second_term_of_phrase in inverted_index.keys(): for doc_info_of_first_term in inverted_index[ first_term_of_phrase]: #doc_info[0] docID; doc_info[1] list of positions for doc_info_of_second_term in inverted_index[ second_term_of_phrase]: if doc_info_of_first_term[0] == doc_info_of_second_term[0]: for position_of_second_term in doc_info_of_second_term[1]: for position_of_first_term in doc_info_of_first_term[ 1]: if (position_of_second_term - position_of_first_term) == 1: with open("results.boolean.txt", "a") as written_file: #print("%s, %s" %(serial_number, doc_info_of_first_term[0]), file=written_file) phrase_search_result_list.append( doc_info_of_first_term[0]) break else: continue break break if len(phrase_search_result_list ) != 0 and second_term in inverted_index.keys(): #query result for single term for doc_info_of_second_term in inverted_index[second_term]: term_search_result_list.append(doc_info_of_second_term[0]) if operator == 'AND': list_of_docid = [ docid for docid in phrase_search_result_list if docid in term_search_result_list ] if operator == 'OR': list_of_docid = list( set(phrase_search_result_list + term_search_result_list) ) #set() delete the same element and sort all elements in the list if operator == 'AND NOT': list_of_docid = [ docid for docid in phrase_search_result_list if docid not in term_search_result_list ] if operator == 'OR NOT': all_docid = [] xml_content = extract_xml_file( "trec.5000.xml" ) # single_file[0]: docid; single_file[1]: headline+text for single_file in xml_content: all_docid.append(single_file[0]) not_term_docid = [ x for x in all_docid if x not in term_search_result_list ] list_of_docid = list( set(phrase_search_result_list + not_term_docid)) list_of_docid = [int(docid) for docid in list_of_docid] if len(list_of_docid) != 0: with open("results.boolean.txt", "a") as written_file: for docid in sorted(list_of_docid): print("%s,%s" % (serial_number, docid), file=written_file)
if __name__ == '__main__': train = pd.read_json( "/Users/soyoungkim/Desktop/python_codes/two-sigma/data/train.json") test = pd.read_json( "/Users/soyoungkim/Desktop/python_codes/two-sigma/data/test.json") start_time = time.time() train['interest'] = np.where( train['interest_level'] == 'high', 1, np.where(train['interest_level'] == 'medium', 2, 3)) important_features = [ 'bathrooms', 'bedrooms', 'price', 'price_room', 'latitude', 'longitude', 'nb_images', 'nb_features', 'sentiment', 'nb_description', 'description_len', 'b_counts', 'm_counts', 'b_count_log', 'm_count_log' ] numerical_features = preprocessing.pre_processing(train) processed_test_data = preprocessing.pre_processing(test) print('A set of 15 derived features:{0}\n'.format(important_features)) preprocessing.classification(numerical_features, processed_test_data, train['interest']) ans = rent_interest_classifier() ans_dataframe = pd.DataFrame(ans, columns=['low', 'medium', 'high'], index=processed_test_data.index) ans_dataframe.to_csv('result.csv', index=False) print('--- %s seconds ---' % (time.time() - start_time))
inputData = pickle.load(handle) print 'load the existing raw dataset from pickle file, finished!' ########################################################################################## #################### preprocessing ##################### ########################################################################################## targetName = 'fpd' missingRateHighBound = 0.5 categoryUpLimit = 40 fillna = 'None' # options: 'mean', 'median', '-999', 'None' var_threshold = 0 scale_enable = False # when cross validation, no scale in the preprocessing stage write_en = True inputAfterPre = pre_processing(preprocessing_choice, inputData, targetName, missingRateHighBound, categoryUpLimit, fillna, var_threshold, scale_enable, write_en, preprocess_folder) ########################################################################################## #################### feature selection ##################### ########################################################################################## ranking_method = 'rf' featureNum = 400 inputAfterSelect = feature_select(feature_ranking_choice, ranking_method, inputAfterPre, targetName, featureRank_folder, featureNum) ########################################################################################## #################### hyper parameter search ##################### ##########################################################################################
with open(rawdata_folder+'/rawdata.pickle','rb') as handle: inputData = pickle.load(handle) print 'load the existing raw dataset from pickle file, finished!' ########################################################################################## #################### preprocessing ##################### ########################################################################################## targetName = 'fpd' missingRateHighBound = 0.5 categoryUpLimit = 40 fillna = 'None' # options: 'mean', 'median', '-999', 'None' var_threshold = 0 scale_enable = False # when cross validation, no scale in the preprocessing stage write_en = True inputAfterPre = pre_processing(preprocessing_choice,inputData,targetName,missingRateHighBound,categoryUpLimit,fillna,var_threshold,scale_enable,write_en,preprocess_folder) ########################################################################################## #################### feature selection ##################### ########################################################################################## ranking_method = 'rf' featureNum = 400 inputAfterSelect = feature_select(feature_ranking_choice,ranking_method,inputAfterPre,targetName,featureRank_folder,featureNum) ########################################################################################## #################### hyper parameter search ##################### ########################################################################################## classifierList = ['xgb'] # full list of classifiers: ['xgb','gbt','rf','ERT'], more are coming maxIter = 100
# def calc_accuracy(X,Y): # max_vals, max_indices = torch.max(X, 1) # acc = (max_indices == Y).sum().data.cpu().numpy()/max_indices.size()[0] # return acc if __name__ == "__main__": ##GPU 사용 시 parser = argparse.ArgumentParser() parser.add_argument("--input_text", required = True, help="Input text file") parser.add_argument("--output_text" , required= True, help="output text file") args = parser.parse_args() device = torch.device("cuda:0") pre_processing(args.input_text) test_tsv_file = 'test.tsv' bert_model , vocab = get_pytorch_kobert_model() dataset_test = nlp.data.TSVDataset(test_tsv_file, field_indices=[0], num_discard_samples=0) tokenizer = get_tokenizer() tok = nlp.data.BERTSPTokenizer(tokenizer, vocab, lower=False) ## Setting parameters max_len = 20 batch_size = 64 # data_test = BERTDataset(dataset_test, 0, 1, tok, max_len, True, False)