def Stemming(): if request.method == 'POST': dbmodel = database.DBModel() stem = dbmodel.get_data_all("DataTA","Filtering") data_stem = [] for x in stem: stemm = x.values() isi_stem = stemm[:-1] data_stemming = [] for z in isi_stem: if z <> None: b = (z.encode("ascii", "ignore")) stem_w = stemmer.stem(b) if stem_w <> "": data_stemming.append(stem_w) data_stem.append(data_stemming) data = pd.DataFrame(data_stem) head = [] for j in data.columns: head_string = "T" + str(j) head.append(head_string) data.columns = head dbmodel = database.DBModel() # memanggil file model dimodel class DBModel result_insert_table = dbmodel.insert_stemming_data("DataTA", "Stemming", data) return render_template('stemming.html', tables=[data.to_html(classes='table table-bordered')])
def tokenisasi(): # if 'username' not in session: # return render_template('login.html') dbmodel = x.DBModel() get_data = dbmodel.get_data_all("Judul_Skripsi", "datanya") data_all = [] for i in get_data: get_values = i.values() a = get_values for u in a: data_lower = u.lower() word_token = word_tokenize(data_lower) data_all.append(word_token) data = pd.DataFrame(data_all) head_token = [] for index in data.columns: custom_head = "K" + str(index) head_token.append(custom_head) data.columns = head_token dbmodel = x.DBModel() # memanggil file model dimodel class DBModel dbmodel.insert_tokenisasi("Judul_Skripsi", "Tokenisasi", data) return render_template( "tokenisasi.html", tables=[ data.to_html( classes='table table-striped table-bordered table-hover') ])
def Filtering(): if request.method == 'POST': dbmodel = database.DBModel() filter = dbmodel.get_data_all("DataTA","Tokenisasi") data_x = [] for x in filter: fils = x.values() isi_fils = fils[:-1] data_filter = [] for z in isi_fils: if z <> None: b = (z.encode("ascii","ignore")) stop_w = remover.remove(b) if stop_w <> "": data_filter.append(stop_w) data_x.append(data_filter) data = pd.DataFrame(data_x) head = [] for j in data.columns: head_string = "T" + str(j) head.append(head_string) data.columns = head dbmodel = database.DBModel() # memanggil file model dimodel class DBModel result_insert_table = dbmodel.insert_filtering_data("DataTA", "Filtering", data) return render_template('filtering.html', tables=[data.to_html(classes='table table-bordered')])
def Tokenizing(): if request.method == 'POST': dbmodel = database.DBModel() token = dbmodel.get_data_all("DataTA","datanya") data_s=[] for i in token : isi = i.values() isi_judul = isi[1] data_baru2 = isi_judul.lower() word_token2 = word_tokenize(data_baru2) data_s.append((word_token2)) data = pd.DataFrame(data_s) head = [] for j in data.columns: head_string = "T" + str(j) head.append(head_string) data.columns = head dbmodel = database.DBModel() # memanggil file model dimodel class DBModel result_insert_table = dbmodel.insert_tokenisasi_data("DataTA", "Tokenisasi", data) return render_template('tokenizing.html', tables=[data.to_html(classes='table table-bordered')])
def allfile(): # if 'username' not in session: # return render_template('login.html') dbmodel = x.DBModel() get_file = dbmodel.get_data_all("Judul_Skripsi", "file") get_sheet = dbmodel.get_sheet("Judul_Skripsi", "file") get_nclust = dbmodel.get_nclust("Judul_Skripsi", "file") coll_name = [] for z in get_sheet: values = z.values() for h in values: h = h coll_name.append(h) nclust = [] for z2 in get_nclust: values2 = z2.values() for h2 in values2: h2 = str(h2) nclust.append(h2) table = [] for k in get_file: o = k.values() table.append(o) return render_template("allfile.html", coll_name=coll_name, tables=table, nclust=nclust)
def hasil_select(): if request.method == 'POST': select1 = request.form["select1"] select2 = request.form["select2"] selectkolom = request.form["selectkolom"] namakolom = request.form["namakolom"] wb = load_workbook(filename='app/upload_data/DATA[CLEANING].xlsx') sheet_ranges = wb[request.form['sheet']] data = pd.DataFrame(sheet_ranges.values) row1 = int(select1) row2 = int(select2) cols = selectkolom.split(",") #memisahkan kolom dengan koma cols = list(map(int,cols)) #convert to list xname = namakolom.split(",") #memisahkan nama kolom dengan koma data = data[row1:row2][cols] data.columns = [xname] header = {} for index, head in enumerate(xname): header[str(index)] = head dbmodel = database.DBModel() #memanggil file model dimodel class DBModel result_insert_table = dbmodel.insert_cleaning_data("Komentar","datanya",data) #result_insert_header = dbmodel.insert_header("DataTA","judulnya",header) return render_template('hasil_select.html', tables=[data.to_html(classes='table table-bordered')])
def showresult(): # if 'username' not in session: # return render_template('login.html') if request.method == 'POST': collection = request.form['collection'] n_clust = request.form['nclust'] n_clust = int(n_clust) dbmodel = x.DBModel() get_file = dbmodel.get_data_all("Judul_Skripsi", collection) hasil = pd.DataFrame(list(get_file)) pd.options.display.max_colwidth = 999 accordion = [] for p in range(n_clust): group = dbmodel.find_group("Judul_Skripsi", collection, p + 1) accordion.append(list(group)) return render_template( "file_terpilih.html", tables=[ hasil.to_html( classes='table table-striped table-bordered table-hover') ], n=n_clust, accordion=accordion)
def login(): user = request.form['username'] dbmodel = x.DBModel() login_user = dbmodel.find_user("Judul_Skripsi", "user", user) if login_user: if request.form['pass'].encode('utf-8') == login_user['pass'].encode( 'utf-8'): session['username'] = request.form['username'] return redirect(url_for('form_lo')) return render_template("login.html")
def stemming(): # if 'username' not in session: # return render_template('login.html') dbmodel = x.DBModel() get_data = dbmodel.get_data_all("Judul_Skripsi", "Fitering") data_stemming = [] for i in get_data: c = i.values() d = c data_s = [] for k in d: if k <> None: data_stem = (k.encode("ascii", "ignore")) stemming = stemmer.stem(data_stem) stemming = stemming.translate(None, string.punctuation) if stemming <> "": data_s.append(stemming) data_stemming.append(data_s) data = pd.DataFrame(data_stemming) head_stem = [] for index in data.columns: custom_head = "K" + str(index) head_stem.append(custom_head) data.columns = head_stem dbmodel = x.DBModel() # memanggil file model dimodel class DBModel dbmodel.insert_stemming("Judul_Skripsi", "Stemming", data) return render_template( "stemming.html", tables=[ data.to_html( classes='table table-striped table-bordered table-hover') ])
def filtering(): # if 'username' not in session: # return render_template('login.html') dbmodel = x.DBModel() get_data = dbmodel.get_data_all("Judul_Skripsi", "Tokenisasi") data_filtering = [] for i in get_data: a = i.values() b = a fil = [] for j in b: if j <> None: data_fil = (j.encode("ascii", "ignore")) stopword = remover.remove(data_fil) if stopword <> "": fil.append(stopword) data_filtering.append(fil) data = pd.DataFrame(data_filtering) head_filter = [] for index in data.columns: custom_head = "K" + str(index) head_filter.append(custom_head) data.columns = head_filter dbmodel = x.DBModel() # memanggil file model dimodel class DBModel dbmodel.insert_filtering("Judul_Skripsi", "Fitering", data) return render_template( "filtering.html", tables=[ data.to_html( classes='table table-striped table-bordered table-hover') ])
def hasilupload(): # if 'username' not in session: # return render_template('login.html') if request.method == 'POST': f = request.files['file'] file = f.filename s = request.form['sheet'] f.save(os.path.join('app/upload_data', file)) dbmodel = x.DBModel() find_file = dbmodel.find_file("Judul_Skripsi", "file", file, s) if find_file == True: dbmodel.delete_same("Judul_Skripsi", "file", file, s) dbmodel.insert_file("Judul_Skripsi", "file", file, s) get_file = dbmodel.get_file_desc("Judul_Skripsi", "file") for w in get_file: values = w.values() for y in values: y = y wb = load_workbook(filename="app/upload_data/" + file) for u in wb.get_sheet_names(): if u == y: sheet_available = True break else: sheet_available = False if sheet_available == False: return render_template("upload_data.html") sheet_ranges = wb[request.form['sheet']] data = pd.DataFrame(sheet_ranges.values) return render_template( 'hasilexcel.html', tables=[ data.to_html( classes='table table-striped table-bordered table-hover') ])
def Dataframe(): if request.method == 'POST': dbmodel = database.DBModel() token = dbmodel.get_data_all("DataTA","datanya") data_s=[] for i in token : isi = i.values() isi_judul = isi[1] data_baru2 = isi_judul.lower() data_s.append(stopword.remove(data_baru2)) data = data_s shuffle(data) cluster_paragraphs(data, num_clusters=2) clusters = cluster_paragraphs(data, num_clusters=2) data = pd.DataFrame(clusters) return render_template('dataframe.html', tables=[data.to_html(classes='table table-bordered')])
def cleaning_data(): # Setting title mainTitle = request.form["mainTitle"] subTitle = "Data berhasil melalui tahap cleaning" # Get table from input type selectedColumn selectedColumn = request.form["selectedColumn"] table = load_data_selected(selectedColumn) selectedColumn = '' #reset selected column # Cleaning data temp_data_clean = cleaning_str_to_float(table) data = temp_data_clean[1] header = {} for index, head in enumerate(table[0]): header[str(index)] = head # return str(data) # insert table to mongodb dbmodel = d.DBModel() result_insert_table = dbmodel.insert_cleaning_data("Alumni_UAD", "cleaning", data) result_insert_header = dbmodel.insert_header("Alumni_UAD", "header", header) #Setting button and Command command = "Lanjutkan ke tahap cluster, masukan jumlah cluster" button = [ 'Cluster', 'text', '%smod_cluster' % (str(request.url_root)) ] #request.url_root = ambil root host, format button = [nama button, url] return render_template('upload/tables.html', tables=[data.to_html(classes='Alumni')], text=[mainTitle, subTitle, command, selectedColumn], button=button)
def harmonic(): # if 'username' not in session: # return render_template('login.html') dbmodel = x.DBModel() get_stemming = dbmodel.get_data_all("Judul_Skripsi", "Stemming") get_dataawal = dbmodel.get_data_all("Judul_Skripsi", "datanya") stem = [] for h in get_stemming: stem1 = [] val1 = h.values() for k in val1: if k <> None: stem1.append(k) stem.append(stem1) documents = stem awal = [] for c in get_dataawal: val = c.values() for d in val: d = d awal.append(d) # documents = awal feature_extraction = FeatureExtraction() feature = feature_extraction.fit(documents) score_ec = cosine_similarity(feature) #feature) #score_ec = euclidean_distances(feature,feature) print score_ec total_tf = [] for tf_score in feature: total_tf.append(round(np.sum(tf_score), 3)) k = int(request.form["kluster"]) data_len = len(feature) max_iteration = 100 lastoutput = kHarmonic(score_ec, k, data_len, total_tf) # mengambil nama sheet dari hasil upload terakhir get_file = dbmodel.get_file_desc("Judul_Skripsi", "file") for w in get_file: values = w.values() for y in values: y = y #mengambil nama file dari hasil upload terakhir get_file2 = dbmodel.get_file_desc2("Judul_Skripsi", "file") for w2 in get_file2: values2 = w2.values() for y2 in values2: y2 = y2 #mencari sheet didatabase(file) yang nama sheetnya itu = nama sheet yang di uppload terakhir. find_sheet = dbmodel.find_sheet("Judul_Skripsi", "file", y) #kondisi jika ketemu sheet yang sama maka dihitung ada berapaa sheet if find_sheet == True: count_sheet = dbmodel.count_sheet("Judul_Skripsi", "file", y) y = y + "(" + ( str(count_sheet - 1)) + ")" # setelah itu nama sheet diubah menjadi (sheet(1)) dbmodel.update_file1( "Judul_Skripsi", "file", y2, y, k) #mengupdate database dengan nama sheet yang baru print y # update file nambah kolom jumlah kluster dbmodel.update_file("Judul_Skripsi", "file", y2, y, k) # mencari collection dengan nama sheet dari hasil upload terakhir find_collection = dbmodel.find_collection("Judul_Skripsi", y) if find_collection == True: dbmodel.delete_collection("Judul_Skripsi", y) s = 1 all_out = [] for out in lastoutput: temp_out = [] for o in out: temp_out.append(awal[o]) dbmodel.insert_hasil("Judul_Skripsi", y, awal[o], s) all_out.append(temp_out) s = s + 1 get_hasil = dbmodel.get_data_all("Judul_Skripsi", y) table_hasil = pd.DataFrame(list(get_hasil)) return render_template( "k-harmonic.html", tables=[ table_hasil.to_html( classes='table table-striped table-bordered table-hover') ])
def cluster(): clusters = request.form["selectedColumn"] # Setting title mainTitle = request.form["mainTitle"] subTitle = "Data berhasil di jadikan %s clusters" % (clusters) # get table to mongodb dbmodel = d.DBModel() getDataCleaning = dbmodel.get_data_all("Alumni_UAD", "cleaning") getHeader = dbmodel.get_data_all("Alumni_UAD", "header") header = [] for head in getHeader: for index in range(len(head) - 1): header.append(head[str(index)]) data = [] for val in getDataCleaning: temp_val = [] for head in header: temp_val.append(val[head]) data.append(temp_val) result_kmeans = kmeans(data, int(clusters)) # insert table to mongodb dbmodel = d.DBModel() dbmodel.drop_collection("Alumni_UAD", "cluster") list_data = [] for val in result_kmeans['data_cluster']: temp_data = {} temp_data['coordinate'] = val['coordinate'] for index, data in enumerate(val['data']): temp_data[header[index]] = data temp_data['cluster'] = int(val['cluster']) dbmodel.insert_one("Alumni_UAD", "cluster", temp_data) list_data.append(temp_data) if result_kmeans['plot'] != '': img = StringIO.StringIO() result_kmeans['plot'].savefig(img, format='png') img.seek(0) plot_url = base64.b64encode(img.getvalue()) else: plot_url = '' results = pd.DataFrame(list_data) results = results.drop('_id', 1) #Setting button and Command button = [ 'Kembali Upload Data', 'hidden', '%smod_upload' % (str(request.url_root)) ] #request.url_root = ambil root host, format button = [nama button, url] command = "Tekan tombol %s, untuk kembali melakukan cluster dengan data yang baru" % ( button[0]) return render_template( 'upload/tables.html', tables=[results.to_html(classes='Alumni', index=False)], text=[mainTitle, subTitle, command], button=button, plot_url=plot_url)
def hasilpemilihan(): # if 'username' not in session: # return render_template('login.html') if request.method == 'POST': select1 = request.form["select1"] select2 = request.form["select2"] selectcolom = request.form["selectcolom"] namacolom = request.form["namacolom"].encode('utf-8') dbmodel = x.DBModel() # get sheet dari database, hasil upload terakhir get_sfile = dbmodel.get_file_desc("Judul_Skripsi", "file") for w in get_sfile: values = w.values() for y in values: sheet = y # get nama file dari database, hasil upload terakhir get_nfile = dbmodel.get_file_desc2("Judul_Skripsi", "file") for n in get_nfile: values = n.values() for q in values: file = q wb = load_workbook(filename='app/upload_data/' + file) sheet_ranges = wb[sheet] data = pd.DataFrame(sheet_ranges.values) print(data) row1 = int(select1) row2 = int(select2) cols = selectcolom #memisahkan inputan kolom dipilih berdasarkan koma cols = list(map(int, cols)) #corvert to int xname = namacolom #memisahkan inptan nama kolom berdasarkan koma data = data[row1:row2][ cols] #data terpilih berdasarkan inputan baris dan kolom print(data) coba = data[row1:row2][ cols] #data terpilih berdasarkan inputan baris dan kolom print(coba) data.columns = [xname] print(data) data = data.dropna() header = {} for index, head in enumerate(xname): header[str(index)] = head pd.options.display.max_colwidth = 999 print(data) df = pd.DataFrame({'a': ['red', 'yellow', 'blue']}) print(df) print(df.to_dict('dict')) print(data.to_dict('dict')) dbmodel = x.DBModel() #memanggil file model dimodel class DBModel result_insert_table = dbmodel.insert_cleaning_data( "Judul_Skripsi", "datanya", data) result_insert_header = dbmodel.insert_header("Judul_Skripsi", "judulnya", header) return render_template( 'masukprosessing.html', tables=[ data.to_html( classes='table table-striped table-bordered table-hover') ])
#!/usr/bin/python # -*- coding: iso-8859-15 -*- import dbmodel as d # ===== CONFIG DB ======= dbmodel = d.DBModel() db_fwc = "fwc_db" month_collection = "mar" fwc = "fwc" # ==== END CONFIG ======= documents = dbmodel.