text = re.sub('\$.{1,}\$', '', text) # remove all the latex equations from the text

text= re.sub('\&.{1,5}', '', text) # remove all non ascii characters from text

text_list = [w for w in tokenizer_porter(text) if w not in stop] # remove stop words

output = ""

for w in text_list:

output = output + " " + w

conn = sqlite3.connect(dbpath + df_file_name)

c = conn.cursor()

#c.execute('SELECT * FROM arxiv_papers ORDER BY rowid')

c.execute('SELECT * FROM arxiv_papers')

conn.commit()

results = c.fetchall()

conn.close()

df = pandas.DataFrame(results, columns=['url', 'title', 'authors', 'abstract'])

plt.subplots_adjust(hspace=0.4)

ax = plt.subplot(111)

if y_log == 1:

#ax.set_xscale("log", nonposx='clip')

ax.set_yscale("log", nonposy='clip')

if (line == 1):

ax.plot(x, y, 'b', linewidth=2)

ax.plot(x, y, 'ro')

#ax.plot([0., 100.], [0., 100.], 'b', linewidth=1)

else:

ax.plot(x, y, 'ro')

#ax.plot([0., 100.], [0., 100.], 'b', linewidth=1)

if (xmin < xmax):

ax.set_xlim(xmin, xmax)

if (ymin < ymax):

ax.set_ylim(ymin, ymax)

#ax.axhline(linewidth=axis_width, color="k")

#ax.axvline(linewidth=axis_width, color="k")

plt.title(title)

plt.ylabel(ylabel)

plt.xlabel(xlabel)

#plt.show()

plt.savefig(plot_name, bbox_inches='tight')

plt.subplots_adjust(hspace=0.4)

ax = plt.subplot(111)

ax.bar(x, y, 1.0, color='blue', align='center')

if (xmin < xmax):

ax.set_xlim(xmin, xmax)

if (ymin < ymax):

ax.set_ylim(ymin, ymax)

#ax.axhline(linewidth=axis_width, color="k")

#ax.axvline(linewidth=axis_width, color="k")

plt.title(title)

plt.ylabel(ylabel)

plt.xlabel(xlabel)

#plt.show()

plt.savefig(plot_name, bbox_inches='tight')

filename = 'elbow_plot.dat'

if os.path.exists(cpath + filename):

data = joblib.load(cpath + filename)

K = data[0]

meandistortions = data[1]

else:

X = dataset

print 'X Shape: ', X.shape

#K = range(1, 50, 5)

K = [1, 2, 5, 10, 50, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000]

#K = [1, 2, 5, 10, 50, 100]

meandistortions = []

cluster_centers = []

for k in K:

print k

kmeans = KMeans(n_clusters=k, n_jobs=3)

kmeans.fit(X)

#import ipdb; ipdb.set_trace() # debugging code

#meandistortions.append(sum(np.min(cdist(X, kmeans.cluster_centers_, 'euclidean'), axis=1))/X.shape[0])

meandistortions.append(kmeans.inertia_)

cluster_centers.append(kmeans.cluster_centers_)

#print 'k: ', k, ' Cluster Centers: ', kmeans.cluster_centers_

data = [K, meandistortions]

joblib.dump(data, cpath + filename, compress=8)

plot_name = "elbow_plot.png"

title = 'Selecting k with the Elbow Method'

xlabel = 'Number of Clusters (k)'

ylabel = 'Average Distortion'

filename = 'cluster_num_dist_plot.dat'

if os.path.exists(cpath + filename):

data = joblib.load(cpath + filename)

x = data[0]

y = data[1]

else:

x = []

y = []

for i in range(num_clusters):

wh = np.where(predictions == i)[0]

x.append(i)

y.append(len(wh))

data = [x, y]

joblib.dump(data, cpath + filename, compress=8)

plot_name = "cluster_num_dist_plot.png"

title = 'Cluster Member Distribution'

xlabel = 'Cluster Label'

ylabel = 'Number of Publications'

#import ipdb;ipdb.set_trace() # debugging code

k = num_clusters

print 'Number of Clusters: ', k

filename = "count_vectorizer.dat"

model_filename = "kmeans_" + str(k) + ".dat"

print filename, model_filename

if os.path.exists(cpath + filename):

vec = joblib.load(cpath + filename)

count = vec[0]

tfidf = vec[1]

else:

tfidf = TfidfTransformer(use_idf=True, norm=None, smooth_idf=True)

text = []

for i in range(len(paper_text)):

#print 'Before: ', paper_text[k]

#text.append(paper_text[i])

text.append(preprocessor(paper_text[i]))

#print 'After: ', titles[k]

#import ipdb; ipdb.set_trace() # debugging code

count = CountVectorizer()

count.fit(text)

bag = count.transform(text)

tfidf = TfidfTransformer(use_idf=True, norm='l2', smooth_idf=True)

tfidf.fit(bag)

X = tfidf.transform(bag)

vec = [count, tfidf]

joblib.dump(vec, cpath + filename, compress=8)

print 'X Shape: ', X.shape

# plot elbow plot to look at the effect of different number of clusters

perform_cluster_analysis(X)

#print count.vocabulary_

#perform_cluster_analysis(X)

#print "bag shape: ", bag.shape

#print "dataset shape: ", X.shape

#import ipdb;ipdb.set_trace() # debugging code

if os.path.exists(cpath + model_filename):

model = joblib.load(cpath + model_filename)

kmeans = model[0]

predict = model[1]

else:

print "Clustering the data..."

kmeans = KMeans(n_clusters=k, n_jobs=3)

kmeans.fit(X)

predict = kmeans.predict(X)

model = [kmeans, predict]

joblib.dump(model, cpath + model_filename, compress=8)

print "Data Clustering Completed."

y = tfidf.transform(count.transform([preprocessor(search_text)]))

#for i in range(10):

# print predict[i], paper_titles[i]

plot_cluster_number_distribution(predict, num_clusters)

#import ipdb;ipdb.set_trace() # debugging code

y_cluster = kmeans.predict(y)

print "Predicted cluster: ", y_cluster

wh = np.where(predict == y_cluster)[0]

#import ipdb;ipdb.set_trace() # debugging code

target_papers = paper_titles[wh]

for i in range(len(wh)):

print wh[i], " ", target_papers[i]

#import ipdb;ipdb.set_trace() # debugging code

filename = "count_vectorizer.dat"

if os.path.exists(cpath + filename):

vec = joblib.load(cpath + filename)

count = vec[0]

tfidf = vec[1]

filename = "kmeans_" + str(num_clusters) + ".dat"

if os.path.exists(cpath + filename):

model = joblib.load(cpath + filename)

kmeans = model[0]

predict = model[1]

y = tfidf.transform(count.transform([preprocessor(search_text)]))

y_cluster = kmeans.predict(y)

#print "Predicted cluster: ", y_cluster

wh = np.where(predict == y_cluster)[0]

#import ipdb;ipdb.set_trace() # debugging code