def extract_features(parsed_file, unified_file, cube_file, cube_context_file, freq_data_file, stop_word_file, freq_pattern_file, base_dir, total_docs, filtered_cell_str):
print filtered_cell_str
freq_patterns = load_freq_patterns(freq_pattern_file)
freq_data = load_freq(freq_data_file)
cubes = load_cube(cube_file)
contexts = load_context(cube_context_file)
unified_list = load_unified_list(unified_file)
print contexts.keys()
#cubes['all'] = [i for i in range(total_docs)]
all_docs = [i for i in range(total_docs)]
total_cnt = agg_phrase_cnt(freq_data, all_docs)
print sum(total_cnt.values())
#extract the features of phrases in each cube
phrase_feature_all = {}
idx = 0
for att in cubes:
if att != filtered_cell_str: #'Topic|Sports;Location|Illinois;':
continue
print "start processing " + att
selected_doc = cubes[att]
selected_context = contexts[att]
#print selected_context
feature_extractor = FeatureExtractor(parsed_file, selected_doc, selected_context, freq_data, stop_word_file, freq_patterns, total_cnt, unified_list)
phrase_features = feature_extractor.extract_features()
for phrase in phrase_features:
norm_phrase = normalize(phrase)
phrase_features[phrase].append(unified_list[norm_phrase])
cell_phrase = "{0}{1}".format(att.replace('|', '_').replace(';', '_').replace(' ', '_').lower(), norm_phrase)
phrase_feature_all[cell_phrase] = phrase_features[phrase]
file_name = "{0}/{1}.fea".format(base_dir, "cells.fea")
save_features(file_name, phrase_feature_all)
def extract_features(parsed_file, cube_file, freq_data_file, stop_word_file, freq_pattern_file, base_dir, total_docs):
freq_patterns = load_freq_patterns(freq_pattern_file)
freq_data = load_freq(freq_data_file)
cubes = load_cube(cube_file)
cubes['all'] = [i for i in range(total_docs)]
#extract the features of phrases in each cube
for att in cubes:
selected_doc = cubes[att]
feature_extractor = FeatureExtractor(parsed_file, selected_doc, freq_data, stop_word_file, freq_patterns)
phrase_features = feature_extractor.extract_features()
file_name = "{0}/{1}.fea".format(base_dir, att.replace('|', '_').replace(';', '_').replace(' ', '_'))
save_features(file_name, phrase_features)
