def get_documents_from_folder(path, is_data_preprocessed=False):
doc_files = [
f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))
]
doc = {}
if is_data_preprocessed:
for f in doc_files:
doc[f] = processing.processing(os.path.join(path, f))
else:
for f in doc_files:
doc[f] = processing.processing(os.path.join(path, f))
return doc
def getDataSet():
cImage = processing.image()
cProcess = processing.processing()
cExcel = processing.excel()
cFile = processing.file()
files = cFile.readFolder('Data Batik')
for x in files:
length = len(x)
num = (x[length-5])
extension = x.split('.')[1]
if extension != 'jpg':
continue
if int(num) == 5 or int(num) == 6 :
continue
else:
path = 'Data Batik/'+x
image = cImage.readImage(path)
imageGray = cImage.toGrayScale(image)
imageHSV = cImage.toHSV(image)
imageBinary = cImage.toBinary(imageGray)
data = cProcess.getFeature(imageBinary,imageHSV)
path = x
data.append(path)
cExcel.write('dataset.xls',data)
def __init__(self):
self.horizon = "04"
self.satellites = "18"
self.file = "planning_data_" + self.satellites + "sat_" + self.horizon + "h"
self.doc = minidom.parse(self.file + ".xml")
self.totalTime = int(self.horizon) * 60 * 60
self.stations = {}
self.processing = processing.processing(self.horizon, self.satellites, self.file)
def analyse(mpd, mels, limit, mel_min, mel_max):
_, tracks = fetch_mpd(mpd, limit)
track_list = processing(tracks, mels)
train_tracks, *_ = split(track_list)
x_mean, x_std = analyse_mels(mels,
train_tracks,
x_min=mel_min,
x_max=mel_max)
print('mean=%.4f; std=%.4f' % (x_mean, x_std))
def create_naive_document_dictionaries_from_training_files(
base_path, is_data_preprocessed=False):
dr_path = os.path.join(base_path, 'DR')
dt_path = os.path.join(base_path, 'DT')
l_path = os.path.join(base_path, 'L')
dr_files = [
f for f in os.listdir(dr_path)
if os.path.isfile(os.path.join(dr_path, f))
]
dt_files = [
f for f in os.listdir(dt_path)
if os.path.isfile(os.path.join(dt_path, f))
]
l_files = [
f for f in os.listdir(l_path)
if os.path.isfile(os.path.join(l_path, f))
]
dr_documents = {}
dt_documents = {}
l_documents = {}
if is_data_preprocessed:
for f in dr_files:
dr_documents[f] = processing.preprocessed_data(
os.path.join(dr_path, f))
for f in dt_files:
dt_documents[f] = processing.preprocessed_data(
os.path.join(dt_path, f))
for f in l_files:
l_documents[f] = processing.preprocessed_data(
os.path.join(l_path, f))
else:
for f in dr_files:
dr_documents[f] = processing.processing(os.path.join(dr_path, f))
for f in dt_files:
dt_documents[f] = processing.processing(os.path.join(dt_path, f))
for f in l_files:
l_documents[f] = processing.processing(os.path.join(l_path, f))
return documentTypes(dr_documents, dt_documents, l_documents)
def stats(mpd, mels, limit):
playlists, tracks = fetch_mpd(mpd, limit)
size = [len(p) for p in playlists]
tracks = processing(tracks, mels)
occurencies = [len(t[-1]) for t in tracks]
print('%i playlists and %i unique tracks' % (len(playlists), len(tracks)))
print('Playlist sizes: min=%i ; avg=%.2f ; max=%i' %
(min(size), mean(size), max(size)))
print('Track occurences: min=%i ; avg=%.2f ; max=%i' % \
(min(occurencies), mean(occurencies), max(occurencies)))
print('Number of interaction (playlist-track pairs): %i' %
sum(occurencies))
def get_frequency_file(path, file_list):
words = ''
for f in file_list:
words += processing.processing(os.path.join(path, f))
words = words.split() #splits into list on whitespace
total_num_words = len(words)
word_dict = dict(Counter(words))
frequency_list = list(
{w: c / total_num_words
for w, c in word_dict.items()}.items())
#print(frequency_list)
sorted_freq_list = sorted(frequency_list, key=lambda x: x[1], reverse=True)
return sorted_freq_list[0:20]
def getRetrieve(input):
cImage = processing.image()
cProcess = processing.processing()
cExcel = processing.excel()
dataset = cExcel.readDataSet('dataset.xls')
image = cImage.readImage(input)
imageGray = cImage.toGrayScale(image)
imageHSV = cImage.toHSV(image)
imageBinary = cImage.toBinary(imageGray)
data = cProcess.getFeature(imageBinary,imageHSV)
res = cProcess.getRetrieve(data,dataset)
for x in range(0,4):
print res[x]
def getRetrieveAll():
cImage = processing.image()
cProcess = processing.processing()
cExcel = processing.excel()
cFile = processing.file()
files = cFile.readFolder('Data Batik')
dataset = cExcel.readDataSet('dataset.xls')
counter = 0
for x in files:
length = len(x)
num = (x[length-5])
extension = x.split('.')[1]
if extension != 'jpg':
continue
if int(num) == 5 or int(num) == 6 :
path = 'Data Batik/'+x
image = cImage.readImage(path)
imageGray = cImage.toGrayScale(image)
imageHSV = cImage.toHSV(image)
imageBinary = cImage.toBinary(imageGray)
data = cProcess.getFeature(imageBinary,imageHSV)
res = cProcess.getRetrieve(data,dataset)
if counter == 0:
name = []
name.append('')
for y in res:
name.append(y[0])
name.append('')
name.append('')
name.append('')
name.append('')
name.append('')
name.append('recall')
name.append('precission')
cExcel.write('hasil.xls',name)
counter+=1
inp = []
inp.append(x)
for y in res :
inp.append(y[1])
res = cProcess.getSort(res)
inp.append('')
listHasil = []
for y in range(0,4) :
inp.append(res[y][0])
listHasil.append(res[y][0])
path = 'Data Batik/'
rec,pres = cProcess.getRecallPrecission(x,listHasil,path)
inp.append(rec)
inp.append(pres)
cExcel.write('hasil.xls',inp)
def search(d):
"""
Search method for classifying a document based on number of times
a phrase appears
"""
my_dict = {}
for document in os.listdir(d):
list_of_words = processing.processing(os.path.join(d, document))
list_of_words = list_of_words.split()
total_num_words = len(list_of_words)
word_dict = dict(Counter(list_of_words))
l_freq = 0
dt_freq = 0
dr_freq = 0
if "lien" in word_dict:
l_freq = word_dict["lien"]
if "trust" in word_dict:
dt_freq = word_dict["trust"]
if "reconveyance" in word_dict:
dr_freq = word_dict["reconveyance"]
if dt_freq > l_freq and dt_freq > dr_freq:
my_dict[document] = "DT"
#print (document + ": " + "Deed of Trust\n")
elif dr_freq > l_freq and dr_freq > dt_freq:
my_dict[document] = "DR"
#print (document + ": " + "Deed of Reconveyance\n")
elif l_freq > dt_freq and l_freq > dr_freq:
my_dict[document] = "L"
#print (document + ": " + "Lien\n")
else:
choices = ['DT', 'DR', 'L']
my_dict[document] = random.choice(choices)
list_of_words = None
#print (my_dict)
#create_feature.accuracy_of_results(my_dict, "data/test-results.txt")
return my_dict
def scanner():
code = np.zeros(12)
url = 'http://192.168.42.129:8080/video'
cap = cv.VideoCapture(url)
ret, frame = cap.read()
y_center = int(frame.shape[0] / 2)
x_center = int(frame.shape[1] / 2)
display_message = False
check = False
while not check:
while True:
ret, frame = cap.read()
if not ret:
raise Exception('Could not read video.')
img = frame
line = img[y_center, :].copy()
cv.circle(img, (x_center, y_center), 10, (0, 255, 0), -1)
cv.circle(img, (x_center, y_center), 10, (0, 0, 255), 1)
cv.putText(img, 'Press enter to scan. Press esc to exit.', (0, 30),
cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255))
if display_message:
cv.putText(img, 'Could not scan. Please try again.', (0, 60),
cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255))
img[y_center, :, :] = [0, 0, 255]
cv.imshow('Scan', img)
k = cv.waitKey(1)
if k == 27:
cap.release()
cv.destroyAllWindows()
exit()
elif k == 13:
break
code, check = decoder(processing(line))
if not check:
display_message = True
cap.release()
cv.destroyAllWindows()
return (code)
def train(args):
playlists, tracks = fetch_mpd(args.mpd, args.limit)
all_tracks = processing(tracks, args.mels)
train_tracks, valid_tracks, test_tracks = split(all_tracks)
mel_enc = MelEncoder(args.mels, mean=args.mel_mean, std=args.mel_std,
length=args.mel_length, min=args.mel_min, max=args.mel_max)
playlist_train_enc = PlaylistEncoder(train_tracks)
train_set = MPDSubset(train_tracks, all_tracks, playlists, playlist_train_enc, mel_enc,
nb_negatives=args.nb_negatives)
train_dl = DataLoader(train_set, batch_size=args.batch_size, shuffle=True)
playlist_valid_enc = PlaylistEncoder(valid_tracks)
valid_set = MPDSubset(valid_tracks, all_tracks, playlists, playlist_valid_enc, mel_enc,
nb_negatives=args.nb_negatives)
valid_dl = DataLoader(valid_set, batch_size=args.batch_size, shuffle=False)
model = DCUE(len(playlists), cnn_channels=args.mel_bins, emb_size=args.embedding_size)
system = DCUEWrapper(model, train_dl, valid_dl, save=args.save, criterion=args.criterion,
margin=args.margin, lr=args.learning_rate)
system.fit(args.epochs, load=args.load)
@Author : Yixing Lu
@Time : 2021/04/12 16:00:12
@Software : VS Code
'''
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
import cv2
from processing import processing
# read video file
cap = cv2.VideoCapture('ch01.avi')
# write video
fourcc = cv2.VideoWriter_fourcc(*'XVID')
out = cv2.VideoWriter('output.avi', fourcc, 20.0, (640, 480))
while (cap.isOpened()):
ret, frame = cap.read() #捕获一帧图像
if ret == True:
frame = processing(frame) # process the img
out.write(frame) # store the frame
# cv2.imshow('frame',frame) # display it
# if cv2.waitKey(1) & 0xFF == ord('q'):
# break
else:
break
cap.release() #close camera
out.release()
cv2.destroyAllWindows()
"""This program recognize handwritten digits using trained NN model.
"""
import cv2
import numpy as np
from keras.models import load_model
from processing import processing, segmentation
# load data to predict #
path = 'IMG_1702.jpg' # IMG_1702.jpg, IMG_1703.jpg
i_resize, i_dilate = processing(path)
# load model #
vanilla_model = load_model('vanilla_model_classic.h5')
CNN_model = load_model('cnn_model_classic.h5')
def show_prediction(image, prediction, box, model_name):
"""Show the prediction digits above original digits on the image
"""
image = image.copy()
# cv2.imshow('Original', image)
for (x, y, w, h), p in sorted(zip(box, prediction)):
cv2.rectangle(image, (x, y), (x + w, y + h), (0, 0, 255), 1)
cv2.putText(image, str(p), (x + 2, y - 5), cv2.FONT_HERSHEY_SIMPLEX,
1.0, (0, 255, 0), 2)
cv2.imshow('Using ' + model_name, image)
cv2.waitKey(0)
cv2.destroyAllWindows()
# recognize #
# using vanilla model
def details(request):
'''
to display user details and accept the cheque
:param request:
:return: redirects to appropriate page
'''
# get user account details from database
profile = bearerBank.objects.filter(user=request.user, registered=True)
details = None
if profile:
details = dict()
profile = profile[0]
details['accountNumber'] = profile.accountNumber
details['name'] = profile.full_name
details['fatherName'] = profile.fatherName
details['balance'] = profile.balance
details['profilePicture'] = profile.profilePicture.url
details['dateOfBirth'] = profile.dateOfBirth
details['pan'] = profile.pan
if not request.user.is_authenticated: # unauthenticated user
return redirect('ChequeClearingSystem:main')
elif request.POST: # post request
form = chequeUpload(request.POST or None, request.FILES or None)
if form.is_valid():
# extract details and files from form
chequeDetails = form.save(commit=False)
chequeDetails.accountNumber = form.cleaned_data['accountNumber']
chequeDetails.amount = form.cleaned_data['amount']
chequeDetails.cheque = request.FILES['cheque']
file_type_sign = chequeDetails.cheque.url.split('.')[-1]
file_type_sign = file_type_sign.lower()
accountHolder = bearerBank.objects.filter(user=request.user)
accountNumberUser = list()
for accounts in accountHolder:
accountNumberUser.append(accounts.accountNumber)
# cheque if account exists in database and user is registered with that account
if chequeDetails.accountNumber not in accountNumberUser:
context = {
'chequeDetails': chequeDetails,
'form': chequeUpload(),
'msg': 'No account',
'details': details
}
return render(request, 'ChequeClearingSystem/details.html', context)
# Create entry in bearer bank database
accountHolder = bearerBank.objects.filter(accountNumber=chequeDetails.accountNumber)
accountHolder = accountHolder.values()
accountHolder = list(accountHolder)
accountHolder = accountHolder[0]
temp = accountHolder
accountHolder = list()
for x in temp:
accountHolder.append(temp[x])
accountHolder = bearerBank(*accountHolder)
chequeDetails.bearer = accountHolder
# check file type
if file_type_sign not in IMAGE_FILE_TYPES:
context = {
'chequeDetails': chequeDetails,
'form': form,
'msg': 'Image file must be PNG, JPG, or JPEG',
}
return render(request, 'ChequeClearingSystem/details.html', context)
chequeDetails.chequeNumber = -1
# extract details and match them with the database
try:
chequeDetails.save()
acknowledgement = processing(chequeDetails.amount,
BASE_DIR + '/ChequeClearingSystem/files/' + str(chequeDetails.cheque),
accountHolder.full_name)
except:
acknowledgement = 'NAK'
if acknowledgement == 'NAK': # invalid Format
message = 'Transaction Failed'
return render(request, 'ChequeClearingSystem/details.html',
{'chequeDetails': chequeDetails, 'form': chequeUpload(), 'details': details,
'msg': message})
elif acknowledgement[0] == 'NAK': # Details verification failed
try:
message = 'Transaction Failed. Balance in A/C ' + str(acknowledgement[3]) + ": " + str(
acknowledgement[2])
sendMessage(contactNumber=acknowledgement[1], msg=message) # send the message to payee
message = 'Transaction Failed. Balance in A/C ' + str(accountHolder.accountNumber) + ": " + str(
accountHolder.balance)
sendMessage(contactNumber=accountHolder.contactNumber, msg=message) # send the message to bearer
except:
message = "Transaction Failed"
return render(request, 'ChequeClearingSystem/details.html',
{'chequeDetails': chequeDetails, 'form': chequeUpload(), 'details': details,
'msg': message})
else:
try:
# payeeBank object for entry in payee Bank Database
payee = payeeBank.objects.filter(accountNumber=acknowledgement[1])
payee = payee.values()
payee = list(payee)
payee = payee[0]
temp = payee
payee = list()
for x in temp:
payee.append(temp[x])
payee = payeeBank(*payee)
chequeDetails.payee = payee
chequeDetails.chequeNumber = acknowledgement[3]
timeNow = datetime.now()
accountHolder.lastTransaction = timeNow
payee.lastTransaction = timeNow
accountHolder.balance += acknowledgement[2]
payee.balance -= acknowledgement[2]
# payeeBankCheque object for entry in payeeBank cheque database
payeeCheque = payeeBankCheque()
payeeCheque.payee = payee
payeeCheque.bearer = accountHolder
payeeCheque.cheque = chequeDetails.cheque
payeeCheque.chequeNumber = acknowledgement[3]
payeeCheque.timeDeposited = timeNow
payeeCheque.amount = chequeDetails.amount
# save
chequeDetails.save()
accountHolder.save()
payee.save()
payeeCheque.save()
# send message to bearer and payee
sendMessage(acknowledgement[2], accountHolder.balance, accountHolder.contactNumber,
accountHolder.accountNumber)
sendMessage(-acknowledgement[2], payee.balance, payee.contactNumber, payee.accountNumber)
message = 'Transaction Successful'
except:
message = "Transaction Failed"
return render(request, 'ChequeClearingSystem/details.html',
{'chequeDetails': chequeDetails, 'form': chequeUpload(), 'details': details,
'msg': message})
return redirect('ChequeClearingSystem:profile')
else: # for get request
if details is not None: # user has account registered
message = "WELCOME " + details['name']
else:
message = "WELCOME"
return render(request, 'ChequeClearingSystem/details.html',
{'form': chequeUpload(), 'details': details, 'msg': message})
''' DOCUMENTATION ''' # IMPORTS import entry as en import recording as rec import previous as prev import processing as proc # ENTRY POINT en.animate() ; # START RECORDING rec.record() # IMPORT PREVIOUS CALCULATIONS DATA prev.importPrevious() # BEGIN PROCESSING proc.processing() # DISPLAY RESULTS
def newIniProc(self, procID, rt, procType, params):
self.iniProcList.append(
processing.processing(procID, rt, self, procType, params))
processed_dir = "../processed_data/"
isExists = os.path.exists(processed_dir)
if not isExists:
os.makedirs(processed_dir)
origin_dir = "../webpages/"
isExists = os.path.exists(origin_dir)
if not isExists:
os.makedirs(origin_dir)
file_path = "../doc_ku/crawled.txt"
download_html(file_path, origin_dir)
print("Download html finished.")
url_dic = get_url_dic()
file_num = len(url_dic)
processing(origin_dir, processed_dir)
print("Finish processing html files.")
start = time.time()
indexing(url_dic, processed_dir)
end = time.time()
print("Finish indexing.")
print(end - start)
start = time.time()
vs_model_pre(file_num)
end = time.time()
print(end - start)
print("Finish vs model preparation.")
def initprocessing(self):
self.proc = processing.processing()
self.procinitialized = True
