def parse_sgml(r, table):
start_time = time.time()
total_topics, total_texts = prepare_reuters21578(r)
dict = dict_by_dict(r, total_texts, table)
madlib = open(table + '.madlib', 'w')
j = 1
for key, value in total_texts.iteritems():
#print key
lines = value.split('\n')
words = utility.get_file_words(lines)
if len(words) == 0: continue
#for madlib
line = utility.get_madlib_line(words, dict)
madlib.write(('%s : %s') % (str(j), line))
j = j + 1
madlib.close()
elapsed_time = time.time() - start_time
print r, 'parsing time:', elapsed_time
def dict_by_dict(r, dict, table):
total_words = []
for key, value in dict.iteritems():
total_words += utility.get_file_words(value.split('\n'))
dict = utility.unique(total_words)
utility.write_dict_file(dict, table + '.dict')
return dict
def dict_by_dict(r, dict, table):
total_words = []
for key, value in dict.iteritems():
total_words += utility.get_file_words(value.split('\n'))
dict = utility.unique(total_words)
utility.write_dict_file(dict, table + '.dict')
return dict
def dict_by_text(r, table, parse_method=None):
file_path_list, file_name_list = utility.get_file_list(r)
total_words = []
for file_path in file_path_list:
lines = parse_method(file_path)
total_words += utility.get_file_words(lines)
dict = utility.unique(total_words)
utility.write_dict_file(dict, table + '.dict')
return dict
def dict_by_text(r, table, parse_method = None):
file_path_list, file_name_list = utility.get_file_list(r)
total_words = []
for file_path in file_path_list:
lines = parse_method(file_path)
total_words += utility.get_file_words(lines)
dict = utility.unique(total_words)
utility.write_dict_file(dict, table + '.dict')
return dict
def parse_sgml(r, table):
start_time = time.time()
total_topics, total_texts = prepare_reuters21578(r)
dict = dict_by_dict(r, total_texts, table)
madlib = open(table + '.madlib', 'w')
j = 1
for key, value in total_texts.iteritems():
#print key
lines = value.split('\n')
words = utility.get_file_words(lines)
if len(words) == 0: continue
#for madlib
line = utility.get_madlib_line(words, dict)
madlib.write(('%s : %s') % (str(j), line))
j = j + 1
madlib.close()
elapsed_time = time.time() - start_time
print r, 'parsing time:', elapsed_time
def parse_normal(r, parse_method, dict_method, table):
dataset = table
start_time = time.time()
#topic_method(r)
file_path_list, file_name_list = utility.get_file_list(r)
dict = dict_method(r, table, parse_method)
madlib = open(dataset + '.madlib', 'w')
file_count = len(file_path_list)
for i in range(0, file_count):
lines = parse_method(file_path_list[i])
words = utility.get_file_words(lines)
if len(words) == 0: continue
#for madlib
line = utility.get_madlib_line(words, dict)
madlib.write(('%s : %s') % (str(i + 1), line))
madlib.close()
elapsed_time = time.time() - start_time
print r, 'parsing time:', elapsed_time
def parse_normal(r, parse_method, dict_method, table):
dataset = table
start_time = time.time()
#topic_method(r)
file_path_list, file_name_list = utility.get_file_list(r)
dict = dict_method(r, table, parse_method)
madlib = open(dataset + '.madlib', 'w')
file_count = len(file_path_list)
for i in range(0, file_count):
lines = parse_method(file_path_list[i])
words = utility.get_file_words(lines)
if len(words) == 0: continue
#for madlib
line = utility.get_madlib_line(words, dict)
madlib.write(('%s : %s') % (str(i + 1), line))
madlib.close()
elapsed_time = time.time() - start_time
print r, 'parsing time:', elapsed_time
