def Stemming():
if request.method == 'POST':
dbmodel = database.DBModel()
stem = dbmodel.get_data_all("DataTA","Filtering")
data_stem = []
for x in stem:
stemm = x.values()
isi_stem = stemm[:-1]
data_stemming = []
for z in isi_stem:
if z <> None:
b = (z.encode("ascii", "ignore"))
stem_w = stemmer.stem(b)
if stem_w <> "":
data_stemming.append(stem_w)
data_stem.append(data_stemming)
data = pd.DataFrame(data_stem)
head = []
for j in data.columns:
head_string = "T" + str(j)
head.append(head_string)
data.columns = head
dbmodel = database.DBModel() # memanggil file model dimodel class DBModel
result_insert_table = dbmodel.insert_stemming_data("DataTA", "Stemming", data)
return render_template('stemming.html', tables=[data.to_html(classes='table table-bordered')])
def tokenisasi():
# if 'username' not in session:
# return render_template('login.html')
dbmodel = x.DBModel()
get_data = dbmodel.get_data_all("Judul_Skripsi", "datanya")
data_all = []
for i in get_data:
get_values = i.values()
a = get_values
for u in a:
data_lower = u.lower()
word_token = word_tokenize(data_lower)
data_all.append(word_token)
data = pd.DataFrame(data_all)
head_token = []
for index in data.columns:
custom_head = "K" + str(index)
head_token.append(custom_head)
data.columns = head_token
dbmodel = x.DBModel() # memanggil file model dimodel class DBModel
dbmodel.insert_tokenisasi("Judul_Skripsi", "Tokenisasi", data)
return render_template(
"tokenisasi.html",
tables=[
data.to_html(
classes='table table-striped table-bordered table-hover')
])
def Filtering():
if request.method == 'POST':
dbmodel = database.DBModel()
filter = dbmodel.get_data_all("DataTA","Tokenisasi")
data_x = []
for x in filter:
fils = x.values()
isi_fils = fils[:-1]
data_filter = []
for z in isi_fils:
if z <> None:
b = (z.encode("ascii","ignore"))
stop_w = remover.remove(b)
if stop_w <> "":
data_filter.append(stop_w)
data_x.append(data_filter)
data = pd.DataFrame(data_x)
head = []
for j in data.columns:
head_string = "T" + str(j)
head.append(head_string)
data.columns = head
dbmodel = database.DBModel() # memanggil file model dimodel class DBModel
result_insert_table = dbmodel.insert_filtering_data("DataTA", "Filtering", data)
return render_template('filtering.html', tables=[data.to_html(classes='table table-bordered')])
def Tokenizing():
if request.method == 'POST':
dbmodel = database.DBModel()
token = dbmodel.get_data_all("DataTA","datanya")
data_s=[]
for i in token :
isi = i.values()
isi_judul = isi[1]
data_baru2 = isi_judul.lower()
word_token2 = word_tokenize(data_baru2)
data_s.append((word_token2))
data = pd.DataFrame(data_s)
head = []
for j in data.columns:
head_string = "T" + str(j)
head.append(head_string)
data.columns = head
dbmodel = database.DBModel() # memanggil file model dimodel class DBModel
result_insert_table = dbmodel.insert_tokenisasi_data("DataTA", "Tokenisasi", data)
return render_template('tokenizing.html', tables=[data.to_html(classes='table table-bordered')])
def allfile():
# if 'username' not in session:
# return render_template('login.html')
dbmodel = x.DBModel()
get_file = dbmodel.get_data_all("Judul_Skripsi", "file")
get_sheet = dbmodel.get_sheet("Judul_Skripsi", "file")
get_nclust = dbmodel.get_nclust("Judul_Skripsi", "file")
coll_name = []
for z in get_sheet:
values = z.values()
for h in values:
h = h
coll_name.append(h)
nclust = []
for z2 in get_nclust:
values2 = z2.values()
for h2 in values2:
h2 = str(h2)
nclust.append(h2)
table = []
for k in get_file:
o = k.values()
table.append(o)
return render_template("allfile.html",
coll_name=coll_name,
tables=table,
nclust=nclust)
def hasil_select():
if request.method == 'POST':
select1 = request.form["select1"]
select2 = request.form["select2"]
selectkolom = request.form["selectkolom"]
namakolom = request.form["namakolom"]
wb = load_workbook(filename='app/upload_data/DATA[CLEANING].xlsx')
sheet_ranges = wb[request.form['sheet']]
data = pd.DataFrame(sheet_ranges.values)
row1 = int(select1)
row2 = int(select2)
cols = selectkolom.split(",") #memisahkan kolom dengan koma
cols = list(map(int,cols)) #convert to list
xname = namakolom.split(",") #memisahkan nama kolom dengan koma
data = data[row1:row2][cols]
data.columns = [xname]
header = {}
for index, head in enumerate(xname):
header[str(index)] = head
dbmodel = database.DBModel() #memanggil file model dimodel class DBModel
result_insert_table = dbmodel.insert_cleaning_data("Komentar","datanya",data)
#result_insert_header = dbmodel.insert_header("DataTA","judulnya",header)
return render_template('hasil_select.html', tables=[data.to_html(classes='table table-bordered')])
def showresult():
# if 'username' not in session:
# return render_template('login.html')
if request.method == 'POST':
collection = request.form['collection']
n_clust = request.form['nclust']
n_clust = int(n_clust)
dbmodel = x.DBModel()
get_file = dbmodel.get_data_all("Judul_Skripsi", collection)
hasil = pd.DataFrame(list(get_file))
pd.options.display.max_colwidth = 999
accordion = []
for p in range(n_clust):
group = dbmodel.find_group("Judul_Skripsi", collection, p + 1)
accordion.append(list(group))
return render_template(
"file_terpilih.html",
tables=[
hasil.to_html(
classes='table table-striped table-bordered table-hover')
],
n=n_clust,
accordion=accordion)
def login():
user = request.form['username']
dbmodel = x.DBModel()
login_user = dbmodel.find_user("Judul_Skripsi", "user", user)
if login_user:
if request.form['pass'].encode('utf-8') == login_user['pass'].encode(
'utf-8'):
session['username'] = request.form['username']
return redirect(url_for('form_lo'))
return render_template("login.html")
def stemming():
# if 'username' not in session:
# return render_template('login.html')
dbmodel = x.DBModel()
get_data = dbmodel.get_data_all("Judul_Skripsi", "Fitering")
data_stemming = []
for i in get_data:
c = i.values()
d = c
data_s = []
for k in d:
if k <> None:
data_stem = (k.encode("ascii", "ignore"))
stemming = stemmer.stem(data_stem)
stemming = stemming.translate(None, string.punctuation)
if stemming <> "":
data_s.append(stemming)
data_stemming.append(data_s)
data = pd.DataFrame(data_stemming)
head_stem = []
for index in data.columns:
custom_head = "K" + str(index)
head_stem.append(custom_head)
data.columns = head_stem
dbmodel = x.DBModel() # memanggil file model dimodel class DBModel
dbmodel.insert_stemming("Judul_Skripsi", "Stemming", data)
return render_template(
"stemming.html",
tables=[
data.to_html(
classes='table table-striped table-bordered table-hover')
])
def filtering():
# if 'username' not in session:
# return render_template('login.html')
dbmodel = x.DBModel()
get_data = dbmodel.get_data_all("Judul_Skripsi", "Tokenisasi")
data_filtering = []
for i in get_data:
a = i.values()
b = a
fil = []
for j in b:
if j <> None:
data_fil = (j.encode("ascii", "ignore"))
stopword = remover.remove(data_fil)
if stopword <> "":
fil.append(stopword)
data_filtering.append(fil)
data = pd.DataFrame(data_filtering)
head_filter = []
for index in data.columns:
custom_head = "K" + str(index)
head_filter.append(custom_head)
data.columns = head_filter
dbmodel = x.DBModel() # memanggil file model dimodel class DBModel
dbmodel.insert_filtering("Judul_Skripsi", "Fitering", data)
return render_template(
"filtering.html",
tables=[
data.to_html(
classes='table table-striped table-bordered table-hover')
])
def hasilupload():
# if 'username' not in session:
# return render_template('login.html')
if request.method == 'POST':
f = request.files['file']
file = f.filename
s = request.form['sheet']
f.save(os.path.join('app/upload_data', file))
dbmodel = x.DBModel()
find_file = dbmodel.find_file("Judul_Skripsi", "file", file, s)
if find_file == True:
dbmodel.delete_same("Judul_Skripsi", "file", file, s)
dbmodel.insert_file("Judul_Skripsi", "file", file, s)
get_file = dbmodel.get_file_desc("Judul_Skripsi", "file")
for w in get_file:
values = w.values()
for y in values:
y = y
wb = load_workbook(filename="app/upload_data/" + file)
for u in wb.get_sheet_names():
if u == y:
sheet_available = True
break
else:
sheet_available = False
if sheet_available == False:
return render_template("upload_data.html")
sheet_ranges = wb[request.form['sheet']]
data = pd.DataFrame(sheet_ranges.values)
return render_template(
'hasilexcel.html',
tables=[
data.to_html(
classes='table table-striped table-bordered table-hover')
])
def Dataframe():
if request.method == 'POST':
dbmodel = database.DBModel()
token = dbmodel.get_data_all("DataTA","datanya")
data_s=[]
for i in token :
isi = i.values()
isi_judul = isi[1]
data_baru2 = isi_judul.lower()
data_s.append(stopword.remove(data_baru2))
data = data_s
shuffle(data)
cluster_paragraphs(data, num_clusters=2)
clusters = cluster_paragraphs(data, num_clusters=2)
data = pd.DataFrame(clusters)
return render_template('dataframe.html', tables=[data.to_html(classes='table table-bordered')])
def cleaning_data():
# Setting title
mainTitle = request.form["mainTitle"]
subTitle = "Data berhasil melalui tahap cleaning"
# Get table from input type selectedColumn
selectedColumn = request.form["selectedColumn"]
table = load_data_selected(selectedColumn)
selectedColumn = '' #reset selected column
# Cleaning data
temp_data_clean = cleaning_str_to_float(table)
data = temp_data_clean[1]
header = {}
for index, head in enumerate(table[0]):
header[str(index)] = head
# return str(data)
# insert table to mongodb
dbmodel = d.DBModel()
result_insert_table = dbmodel.insert_cleaning_data("Alumni_UAD",
"cleaning", data)
result_insert_header = dbmodel.insert_header("Alumni_UAD", "header",
header)
#Setting button and Command
command = "Lanjutkan ke tahap cluster, masukan jumlah cluster"
button = [
'Cluster', 'text',
'%smod_cluster' % (str(request.url_root))
] #request.url_root = ambil root host, format button = [nama button, url]
return render_template('upload/tables.html',
tables=[data.to_html(classes='Alumni')],
text=[mainTitle, subTitle, command, selectedColumn],
button=button)
def harmonic():
# if 'username' not in session:
# return render_template('login.html')
dbmodel = x.DBModel()
get_stemming = dbmodel.get_data_all("Judul_Skripsi", "Stemming")
get_dataawal = dbmodel.get_data_all("Judul_Skripsi", "datanya")
stem = []
for h in get_stemming:
stem1 = []
val1 = h.values()
for k in val1:
if k <> None:
stem1.append(k)
stem.append(stem1)
documents = stem
awal = []
for c in get_dataawal:
val = c.values()
for d in val:
d = d
awal.append(d)
# documents = awal
feature_extraction = FeatureExtraction()
feature = feature_extraction.fit(documents)
score_ec = cosine_similarity(feature) #feature)
#score_ec = euclidean_distances(feature,feature)
print score_ec
total_tf = []
for tf_score in feature:
total_tf.append(round(np.sum(tf_score), 3))
k = int(request.form["kluster"])
data_len = len(feature)
max_iteration = 100
lastoutput = kHarmonic(score_ec, k, data_len, total_tf)
# mengambil nama sheet dari hasil upload terakhir
get_file = dbmodel.get_file_desc("Judul_Skripsi", "file")
for w in get_file:
values = w.values()
for y in values:
y = y
#mengambil nama file dari hasil upload terakhir
get_file2 = dbmodel.get_file_desc2("Judul_Skripsi", "file")
for w2 in get_file2:
values2 = w2.values()
for y2 in values2:
y2 = y2
#mencari sheet didatabase(file) yang nama sheetnya itu = nama sheet yang di uppload terakhir.
find_sheet = dbmodel.find_sheet("Judul_Skripsi", "file", y)
#kondisi jika ketemu sheet yang sama maka dihitung ada berapaa sheet
if find_sheet == True:
count_sheet = dbmodel.count_sheet("Judul_Skripsi", "file", y)
y = y + "(" + (
str(count_sheet -
1)) + ")" # setelah itu nama sheet diubah menjadi (sheet(1))
dbmodel.update_file1(
"Judul_Skripsi", "file", y2, y,
k) #mengupdate database dengan nama sheet yang baru
print y
# update file nambah kolom jumlah kluster
dbmodel.update_file("Judul_Skripsi", "file", y2, y, k)
# mencari collection dengan nama sheet dari hasil upload terakhir
find_collection = dbmodel.find_collection("Judul_Skripsi", y)
if find_collection == True:
dbmodel.delete_collection("Judul_Skripsi", y)
s = 1
all_out = []
for out in lastoutput:
temp_out = []
for o in out:
temp_out.append(awal[o])
dbmodel.insert_hasil("Judul_Skripsi", y, awal[o], s)
all_out.append(temp_out)
s = s + 1
get_hasil = dbmodel.get_data_all("Judul_Skripsi", y)
table_hasil = pd.DataFrame(list(get_hasil))
return render_template(
"k-harmonic.html",
tables=[
table_hasil.to_html(
classes='table table-striped table-bordered table-hover')
])
def cluster():
clusters = request.form["selectedColumn"]
# Setting title
mainTitle = request.form["mainTitle"]
subTitle = "Data berhasil di jadikan %s clusters" % (clusters)
# get table to mongodb
dbmodel = d.DBModel()
getDataCleaning = dbmodel.get_data_all("Alumni_UAD", "cleaning")
getHeader = dbmodel.get_data_all("Alumni_UAD", "header")
header = []
for head in getHeader:
for index in range(len(head) - 1):
header.append(head[str(index)])
data = []
for val in getDataCleaning:
temp_val = []
for head in header:
temp_val.append(val[head])
data.append(temp_val)
result_kmeans = kmeans(data, int(clusters))
# insert table to mongodb
dbmodel = d.DBModel()
dbmodel.drop_collection("Alumni_UAD", "cluster")
list_data = []
for val in result_kmeans['data_cluster']:
temp_data = {}
temp_data['coordinate'] = val['coordinate']
for index, data in enumerate(val['data']):
temp_data[header[index]] = data
temp_data['cluster'] = int(val['cluster'])
dbmodel.insert_one("Alumni_UAD", "cluster", temp_data)
list_data.append(temp_data)
if result_kmeans['plot'] != '':
img = StringIO.StringIO()
result_kmeans['plot'].savefig(img, format='png')
img.seek(0)
plot_url = base64.b64encode(img.getvalue())
else:
plot_url = ''
results = pd.DataFrame(list_data)
results = results.drop('_id', 1)
#Setting button and Command
button = [
'Kembali Upload Data', 'hidden',
'%smod_upload' % (str(request.url_root))
] #request.url_root = ambil root host, format button = [nama button, url]
command = "Tekan tombol %s, untuk kembali melakukan cluster dengan data yang baru" % (
button[0])
return render_template(
'upload/tables.html',
tables=[results.to_html(classes='Alumni', index=False)],
text=[mainTitle, subTitle, command],
button=button,
plot_url=plot_url)
def hasilpemilihan():
# if 'username' not in session:
# return render_template('login.html')
if request.method == 'POST':
select1 = request.form["select1"]
select2 = request.form["select2"]
selectcolom = request.form["selectcolom"]
namacolom = request.form["namacolom"].encode('utf-8')
dbmodel = x.DBModel()
# get sheet dari database, hasil upload terakhir
get_sfile = dbmodel.get_file_desc("Judul_Skripsi", "file")
for w in get_sfile:
values = w.values()
for y in values:
sheet = y
# get nama file dari database, hasil upload terakhir
get_nfile = dbmodel.get_file_desc2("Judul_Skripsi", "file")
for n in get_nfile:
values = n.values()
for q in values:
file = q
wb = load_workbook(filename='app/upload_data/' + file)
sheet_ranges = wb[sheet]
data = pd.DataFrame(sheet_ranges.values)
print(data)
row1 = int(select1)
row2 = int(select2)
cols = selectcolom #memisahkan inputan kolom dipilih berdasarkan koma
cols = list(map(int, cols)) #corvert to int
xname = namacolom #memisahkan inptan nama kolom berdasarkan koma
data = data[row1:row2][
cols] #data terpilih berdasarkan inputan baris dan kolom
print(data)
coba = data[row1:row2][
cols] #data terpilih berdasarkan inputan baris dan kolom
print(coba)
data.columns = [xname]
print(data)
data = data.dropna()
header = {}
for index, head in enumerate(xname):
header[str(index)] = head
pd.options.display.max_colwidth = 999
print(data)
df = pd.DataFrame({'a': ['red', 'yellow', 'blue']})
print(df)
print(df.to_dict('dict'))
print(data.to_dict('dict'))
dbmodel = x.DBModel() #memanggil file model dimodel class DBModel
result_insert_table = dbmodel.insert_cleaning_data(
"Judul_Skripsi", "datanya", data)
result_insert_header = dbmodel.insert_header("Judul_Skripsi",
"judulnya", header)
return render_template(
'masukprosessing.html',
tables=[
data.to_html(
classes='table table-striped table-bordered table-hover')
])
#!/usr/bin/python # -*- coding: iso-8859-15 -*- import dbmodel as d # ===== CONFIG DB ======= dbmodel = d.DBModel() db_fwc = "fwc_db" month_collection = "mar" fwc = "fwc" # ==== END CONFIG ======= documents = dbmodel.
