def get_documents_from_folder(path, is_data_preprocessed=False): doc_files = [ f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f)) ] doc = {} if is_data_preprocessed: for f in doc_files: doc[f] = processing.processing(os.path.join(path, f)) else: for f in doc_files: doc[f] = processing.processing(os.path.join(path, f)) return doc
def getDataSet(): cImage = processing.image() cProcess = processing.processing() cExcel = processing.excel() cFile = processing.file() files = cFile.readFolder('Data Batik') for x in files: length = len(x) num = (x[length-5]) extension = x.split('.')[1] if extension != 'jpg': continue if int(num) == 5 or int(num) == 6 : continue else: path = 'Data Batik/'+x image = cImage.readImage(path) imageGray = cImage.toGrayScale(image) imageHSV = cImage.toHSV(image) imageBinary = cImage.toBinary(imageGray) data = cProcess.getFeature(imageBinary,imageHSV) path = x data.append(path) cExcel.write('dataset.xls',data)
def __init__(self): self.horizon = "04" self.satellites = "18" self.file = "planning_data_" + self.satellites + "sat_" + self.horizon + "h" self.doc = minidom.parse(self.file + ".xml") self.totalTime = int(self.horizon) * 60 * 60 self.stations = {} self.processing = processing.processing(self.horizon, self.satellites, self.file)
def analyse(mpd, mels, limit, mel_min, mel_max): _, tracks = fetch_mpd(mpd, limit) track_list = processing(tracks, mels) train_tracks, *_ = split(track_list) x_mean, x_std = analyse_mels(mels, train_tracks, x_min=mel_min, x_max=mel_max) print('mean=%.4f; std=%.4f' % (x_mean, x_std))
def create_naive_document_dictionaries_from_training_files( base_path, is_data_preprocessed=False): dr_path = os.path.join(base_path, 'DR') dt_path = os.path.join(base_path, 'DT') l_path = os.path.join(base_path, 'L') dr_files = [ f for f in os.listdir(dr_path) if os.path.isfile(os.path.join(dr_path, f)) ] dt_files = [ f for f in os.listdir(dt_path) if os.path.isfile(os.path.join(dt_path, f)) ] l_files = [ f for f in os.listdir(l_path) if os.path.isfile(os.path.join(l_path, f)) ] dr_documents = {} dt_documents = {} l_documents = {} if is_data_preprocessed: for f in dr_files: dr_documents[f] = processing.preprocessed_data( os.path.join(dr_path, f)) for f in dt_files: dt_documents[f] = processing.preprocessed_data( os.path.join(dt_path, f)) for f in l_files: l_documents[f] = processing.preprocessed_data( os.path.join(l_path, f)) else: for f in dr_files: dr_documents[f] = processing.processing(os.path.join(dr_path, f)) for f in dt_files: dt_documents[f] = processing.processing(os.path.join(dt_path, f)) for f in l_files: l_documents[f] = processing.processing(os.path.join(l_path, f)) return documentTypes(dr_documents, dt_documents, l_documents)
def stats(mpd, mels, limit): playlists, tracks = fetch_mpd(mpd, limit) size = [len(p) for p in playlists] tracks = processing(tracks, mels) occurencies = [len(t[-1]) for t in tracks] print('%i playlists and %i unique tracks' % (len(playlists), len(tracks))) print('Playlist sizes: min=%i ; avg=%.2f ; max=%i' % (min(size), mean(size), max(size))) print('Track occurences: min=%i ; avg=%.2f ; max=%i' % \ (min(occurencies), mean(occurencies), max(occurencies))) print('Number of interaction (playlist-track pairs): %i' % sum(occurencies))
def get_frequency_file(path, file_list): words = '' for f in file_list: words += processing.processing(os.path.join(path, f)) words = words.split() #splits into list on whitespace total_num_words = len(words) word_dict = dict(Counter(words)) frequency_list = list( {w: c / total_num_words for w, c in word_dict.items()}.items()) #print(frequency_list) sorted_freq_list = sorted(frequency_list, key=lambda x: x[1], reverse=True) return sorted_freq_list[0:20]
def getRetrieve(input): cImage = processing.image() cProcess = processing.processing() cExcel = processing.excel() dataset = cExcel.readDataSet('dataset.xls') image = cImage.readImage(input) imageGray = cImage.toGrayScale(image) imageHSV = cImage.toHSV(image) imageBinary = cImage.toBinary(imageGray) data = cProcess.getFeature(imageBinary,imageHSV) res = cProcess.getRetrieve(data,dataset) for x in range(0,4): print res[x]
def getRetrieveAll(): cImage = processing.image() cProcess = processing.processing() cExcel = processing.excel() cFile = processing.file() files = cFile.readFolder('Data Batik') dataset = cExcel.readDataSet('dataset.xls') counter = 0 for x in files: length = len(x) num = (x[length-5]) extension = x.split('.')[1] if extension != 'jpg': continue if int(num) == 5 or int(num) == 6 : path = 'Data Batik/'+x image = cImage.readImage(path) imageGray = cImage.toGrayScale(image) imageHSV = cImage.toHSV(image) imageBinary = cImage.toBinary(imageGray) data = cProcess.getFeature(imageBinary,imageHSV) res = cProcess.getRetrieve(data,dataset) if counter == 0: name = [] name.append('') for y in res: name.append(y[0]) name.append('') name.append('') name.append('') name.append('') name.append('') name.append('recall') name.append('precission') cExcel.write('hasil.xls',name) counter+=1 inp = [] inp.append(x) for y in res : inp.append(y[1]) res = cProcess.getSort(res) inp.append('') listHasil = [] for y in range(0,4) : inp.append(res[y][0]) listHasil.append(res[y][0]) path = 'Data Batik/' rec,pres = cProcess.getRecallPrecission(x,listHasil,path) inp.append(rec) inp.append(pres) cExcel.write('hasil.xls',inp)
def search(d): """ Search method for classifying a document based on number of times a phrase appears """ my_dict = {} for document in os.listdir(d): list_of_words = processing.processing(os.path.join(d, document)) list_of_words = list_of_words.split() total_num_words = len(list_of_words) word_dict = dict(Counter(list_of_words)) l_freq = 0 dt_freq = 0 dr_freq = 0 if "lien" in word_dict: l_freq = word_dict["lien"] if "trust" in word_dict: dt_freq = word_dict["trust"] if "reconveyance" in word_dict: dr_freq = word_dict["reconveyance"] if dt_freq > l_freq and dt_freq > dr_freq: my_dict[document] = "DT" #print (document + ": " + "Deed of Trust\n") elif dr_freq > l_freq and dr_freq > dt_freq: my_dict[document] = "DR" #print (document + ": " + "Deed of Reconveyance\n") elif l_freq > dt_freq and l_freq > dr_freq: my_dict[document] = "L" #print (document + ": " + "Lien\n") else: choices = ['DT', 'DR', 'L'] my_dict[document] = random.choice(choices) list_of_words = None #print (my_dict) #create_feature.accuracy_of_results(my_dict, "data/test-results.txt") return my_dict
def scanner(): code = np.zeros(12) url = 'http://192.168.42.129:8080/video' cap = cv.VideoCapture(url) ret, frame = cap.read() y_center = int(frame.shape[0] / 2) x_center = int(frame.shape[1] / 2) display_message = False check = False while not check: while True: ret, frame = cap.read() if not ret: raise Exception('Could not read video.') img = frame line = img[y_center, :].copy() cv.circle(img, (x_center, y_center), 10, (0, 255, 0), -1) cv.circle(img, (x_center, y_center), 10, (0, 0, 255), 1) cv.putText(img, 'Press enter to scan. Press esc to exit.', (0, 30), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255)) if display_message: cv.putText(img, 'Could not scan. Please try again.', (0, 60), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255)) img[y_center, :, :] = [0, 0, 255] cv.imshow('Scan', img) k = cv.waitKey(1) if k == 27: cap.release() cv.destroyAllWindows() exit() elif k == 13: break code, check = decoder(processing(line)) if not check: display_message = True cap.release() cv.destroyAllWindows() return (code)
def train(args): playlists, tracks = fetch_mpd(args.mpd, args.limit) all_tracks = processing(tracks, args.mels) train_tracks, valid_tracks, test_tracks = split(all_tracks) mel_enc = MelEncoder(args.mels, mean=args.mel_mean, std=args.mel_std, length=args.mel_length, min=args.mel_min, max=args.mel_max) playlist_train_enc = PlaylistEncoder(train_tracks) train_set = MPDSubset(train_tracks, all_tracks, playlists, playlist_train_enc, mel_enc, nb_negatives=args.nb_negatives) train_dl = DataLoader(train_set, batch_size=args.batch_size, shuffle=True) playlist_valid_enc = PlaylistEncoder(valid_tracks) valid_set = MPDSubset(valid_tracks, all_tracks, playlists, playlist_valid_enc, mel_enc, nb_negatives=args.nb_negatives) valid_dl = DataLoader(valid_set, batch_size=args.batch_size, shuffle=False) model = DCUE(len(playlists), cnn_channels=args.mel_bins, emb_size=args.embedding_size) system = DCUEWrapper(model, train_dl, valid_dl, save=args.save, criterion=args.criterion, margin=args.margin, lr=args.learning_rate) system.fit(args.epochs, load=args.load)
@Author : Yixing Lu @Time : 2021/04/12 16:00:12 @Software : VS Code ''' import os os.environ["CUDA_VISIBLE_DEVICES"] = "1" import cv2 from processing import processing # read video file cap = cv2.VideoCapture('ch01.avi') # write video fourcc = cv2.VideoWriter_fourcc(*'XVID') out = cv2.VideoWriter('output.avi', fourcc, 20.0, (640, 480)) while (cap.isOpened()): ret, frame = cap.read() #捕获一帧图像 if ret == True: frame = processing(frame) # process the img out.write(frame) # store the frame # cv2.imshow('frame',frame) # display it # if cv2.waitKey(1) & 0xFF == ord('q'): # break else: break cap.release() #close camera out.release() cv2.destroyAllWindows()
"""This program recognize handwritten digits using trained NN model. """ import cv2 import numpy as np from keras.models import load_model from processing import processing, segmentation # load data to predict # path = 'IMG_1702.jpg' # IMG_1702.jpg, IMG_1703.jpg i_resize, i_dilate = processing(path) # load model # vanilla_model = load_model('vanilla_model_classic.h5') CNN_model = load_model('cnn_model_classic.h5') def show_prediction(image, prediction, box, model_name): """Show the prediction digits above original digits on the image """ image = image.copy() # cv2.imshow('Original', image) for (x, y, w, h), p in sorted(zip(box, prediction)): cv2.rectangle(image, (x, y), (x + w, y + h), (0, 0, 255), 1) cv2.putText(image, str(p), (x + 2, y - 5), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2) cv2.imshow('Using ' + model_name, image) cv2.waitKey(0) cv2.destroyAllWindows() # recognize # # using vanilla model
def details(request): ''' to display user details and accept the cheque :param request: :return: redirects to appropriate page ''' # get user account details from database profile = bearerBank.objects.filter(user=request.user, registered=True) details = None if profile: details = dict() profile = profile[0] details['accountNumber'] = profile.accountNumber details['name'] = profile.full_name details['fatherName'] = profile.fatherName details['balance'] = profile.balance details['profilePicture'] = profile.profilePicture.url details['dateOfBirth'] = profile.dateOfBirth details['pan'] = profile.pan if not request.user.is_authenticated: # unauthenticated user return redirect('ChequeClearingSystem:main') elif request.POST: # post request form = chequeUpload(request.POST or None, request.FILES or None) if form.is_valid(): # extract details and files from form chequeDetails = form.save(commit=False) chequeDetails.accountNumber = form.cleaned_data['accountNumber'] chequeDetails.amount = form.cleaned_data['amount'] chequeDetails.cheque = request.FILES['cheque'] file_type_sign = chequeDetails.cheque.url.split('.')[-1] file_type_sign = file_type_sign.lower() accountHolder = bearerBank.objects.filter(user=request.user) accountNumberUser = list() for accounts in accountHolder: accountNumberUser.append(accounts.accountNumber) # cheque if account exists in database and user is registered with that account if chequeDetails.accountNumber not in accountNumberUser: context = { 'chequeDetails': chequeDetails, 'form': chequeUpload(), 'msg': 'No account', 'details': details } return render(request, 'ChequeClearingSystem/details.html', context) # Create entry in bearer bank database accountHolder = bearerBank.objects.filter(accountNumber=chequeDetails.accountNumber) accountHolder = accountHolder.values() accountHolder = list(accountHolder) accountHolder = accountHolder[0] temp = accountHolder accountHolder = list() for x in temp: accountHolder.append(temp[x]) accountHolder = bearerBank(*accountHolder) chequeDetails.bearer = accountHolder # check file type if file_type_sign not in IMAGE_FILE_TYPES: context = { 'chequeDetails': chequeDetails, 'form': form, 'msg': 'Image file must be PNG, JPG, or JPEG', } return render(request, 'ChequeClearingSystem/details.html', context) chequeDetails.chequeNumber = -1 # extract details and match them with the database try: chequeDetails.save() acknowledgement = processing(chequeDetails.amount, BASE_DIR + '/ChequeClearingSystem/files/' + str(chequeDetails.cheque), accountHolder.full_name) except: acknowledgement = 'NAK' if acknowledgement == 'NAK': # invalid Format message = 'Transaction Failed' return render(request, 'ChequeClearingSystem/details.html', {'chequeDetails': chequeDetails, 'form': chequeUpload(), 'details': details, 'msg': message}) elif acknowledgement[0] == 'NAK': # Details verification failed try: message = 'Transaction Failed. Balance in A/C ' + str(acknowledgement[3]) + ": " + str( acknowledgement[2]) sendMessage(contactNumber=acknowledgement[1], msg=message) # send the message to payee message = 'Transaction Failed. Balance in A/C ' + str(accountHolder.accountNumber) + ": " + str( accountHolder.balance) sendMessage(contactNumber=accountHolder.contactNumber, msg=message) # send the message to bearer except: message = "Transaction Failed" return render(request, 'ChequeClearingSystem/details.html', {'chequeDetails': chequeDetails, 'form': chequeUpload(), 'details': details, 'msg': message}) else: try: # payeeBank object for entry in payee Bank Database payee = payeeBank.objects.filter(accountNumber=acknowledgement[1]) payee = payee.values() payee = list(payee) payee = payee[0] temp = payee payee = list() for x in temp: payee.append(temp[x]) payee = payeeBank(*payee) chequeDetails.payee = payee chequeDetails.chequeNumber = acknowledgement[3] timeNow = datetime.now() accountHolder.lastTransaction = timeNow payee.lastTransaction = timeNow accountHolder.balance += acknowledgement[2] payee.balance -= acknowledgement[2] # payeeBankCheque object for entry in payeeBank cheque database payeeCheque = payeeBankCheque() payeeCheque.payee = payee payeeCheque.bearer = accountHolder payeeCheque.cheque = chequeDetails.cheque payeeCheque.chequeNumber = acknowledgement[3] payeeCheque.timeDeposited = timeNow payeeCheque.amount = chequeDetails.amount # save chequeDetails.save() accountHolder.save() payee.save() payeeCheque.save() # send message to bearer and payee sendMessage(acknowledgement[2], accountHolder.balance, accountHolder.contactNumber, accountHolder.accountNumber) sendMessage(-acknowledgement[2], payee.balance, payee.contactNumber, payee.accountNumber) message = 'Transaction Successful' except: message = "Transaction Failed" return render(request, 'ChequeClearingSystem/details.html', {'chequeDetails': chequeDetails, 'form': chequeUpload(), 'details': details, 'msg': message}) return redirect('ChequeClearingSystem:profile') else: # for get request if details is not None: # user has account registered message = "WELCOME " + details['name'] else: message = "WELCOME" return render(request, 'ChequeClearingSystem/details.html', {'form': chequeUpload(), 'details': details, 'msg': message})
''' DOCUMENTATION ''' # IMPORTS import entry as en import recording as rec import previous as prev import processing as proc # ENTRY POINT en.animate() ; # START RECORDING rec.record() # IMPORT PREVIOUS CALCULATIONS DATA prev.importPrevious() # BEGIN PROCESSING proc.processing() # DISPLAY RESULTS
def newIniProc(self, procID, rt, procType, params): self.iniProcList.append( processing.processing(procID, rt, self, procType, params))
processed_dir = "../processed_data/" isExists = os.path.exists(processed_dir) if not isExists: os.makedirs(processed_dir) origin_dir = "../webpages/" isExists = os.path.exists(origin_dir) if not isExists: os.makedirs(origin_dir) file_path = "../doc_ku/crawled.txt" download_html(file_path, origin_dir) print("Download html finished.") url_dic = get_url_dic() file_num = len(url_dic) processing(origin_dir, processed_dir) print("Finish processing html files.") start = time.time() indexing(url_dic, processed_dir) end = time.time() print("Finish indexing.") print(end - start) start = time.time() vs_model_pre(file_num) end = time.time() print(end - start) print("Finish vs model preparation.")
def initprocessing(self): self.proc = processing.processing() self.procinitialized = True