from databaseConnection import Database
import math
data = Database()
total_abstract_count = data.getDistinctAbstractNumberForTA('Family Medicine & Internal Medicine')
token_df_info = data.loadTokenTable(total_abstract_count, 0.2, 0.001)
#print("total abstract count: " + str(total_abstract_count))
token_list = []
idf_list = []
count=0
for temp_tuple in token_df_info:
count += 1
token_list.append(str(temp_tuple[0]))
df = temp_tuple[1]
idf = math.log(total_abstract_count/df, 10)
idf = round(idf, 4)
if(idf <0.0001):
idf = 0.0001
idf_list.append(str(idf))
if (len(token_list) == 1000):
idf_str = ",".join(idf_list)
token_str = ",".join(token_list)
#print(idf_str)
#print(token_str)
token_list = []
idf_list = []
