def transform(self):
# this function prepare the dataset to the yolo estructure
aux_path = os.path.join(self.OUTPUT_PATH,
self.DATASET_NAME + "_" + self.model)
shutil.copytree(os.path.join(self.OUTPUT_PATH, self.DATASET_NAME),
aux_path)
f = open(os.path.join(aux_path, "classes.names"), "r")
for line in f:
fn.CLASSES.append(line.split("\n")[0])
train_images = list(
paths.list_files(os.path.join(aux_path, "train"),
validExts=(".jpg")))
test_images = list(
paths.list_files(os.path.join(aux_path, "test"),
validExts=(".jpg")))
traintxt = open(os.path.join(aux_path, "train.txt"), "w")
testtxt = open(os.path.join(aux_path, "test.txt"), "w")
for tr_im in train_images:
traintxt.write(os.path.abspath(tr_im) + "\n")
for te_im in test_images:
testtxt.write(os.path.abspath(te_im) + "\n")
traintxt.close()
testtxt.close()
fn.PascalVOC2YOLO(self.OUTPUT_PATH, self.DATASET_NAME + "_" +
self.model) # , datasetPath + os.sep + "images"
shutil.rmtree(os.path.join(aux_path, "train", "Annotations"))
shutil.rmtree(os.path.join(aux_path, "test", "Annotations"))
def readImagesAndAnnotations(self):
self.imagePaths = list(
paths.list_files(self.imagesPath, validExts=(".jpg", ".jpeg", ".png", ".bmp", ".tiff", ".tif")))
self.labelPaths = list(
paths.list_files(self.labelsPath, validExts=(".jpg", ".jpeg", ".png", ".bmp", ".tiff", ".tif")))
if (len(self.imagePaths) != len(self.labelPaths)):
raise Exception("The number of files is different in the folder of images and in the folder of labels")
def get_HOG_Features(trainingPath, testingPath, cell_size=16, bin_size=8):
from hog import Hog_descriptor
# initialize the local binary patterns descriptor along with the data and label lists
data = []
labels = []
test_data = []
test_labels = []
start_time = time.time()
# loop over the training images
for imagePath in paths.list_files(trainingPath, validExts=(".png",".ppm")):
# open image
img = cv2.imread(imagePath)
gray = np.matrix(cv2.cvtColor(img, cv2.COLOR_BGR2GRAY))
resized_image = cv2.resize(gray, (48, 48))
# get hog features
hog = Hog_descriptor(resized_image, cell_size=cell_size, bin_size=bin_size)
vector = hog.extract()
v = np.array(vector)
# extract the label from the image path, then update the
# label and data lists
labels.append(int(imagePath.split("/")[-2]))
data.append(vector)
# loop over the testing images
for imagePath in paths.list_files(testingPath, validExts=(".png",".ppm")):
# open image
img = cv2.imread(imagePath)
gray = np.matrix(cv2.cvtColor(img, cv2.COLOR_BGR2GRAY))
resized_image = cv2.resize(gray, (48, 48))
# get hog features
hog = Hog_descriptor(resized_image, cell_size=cell_size, bin_size=bin_size)
vector = hog.extract()
# extract the label from the image path, then update the
# label and data lists
test_labels.append(int(imagePath.split("/")[-2]))
test_data.append(vector)
feature_extraction_runtime = (time.time() - start_time)
data = np.array(data)
labels = np.array(labels)
test_data = np.array(test_data)
test_labels = np.array(test_labels)
print "[INFO] HOG Features are ready!"
print "Total image: ", len(data) + len(test_data)
print "Feature extraction runtime: ", feature_extraction_runtime
print "Average for one image:", feature_extraction_runtime / (len(data) + len(test_data))
return (data, labels, test_data, test_labels)
def get_LBP_Features(trainingPath, testingPath, p, r):
from localbinarypatterns import LocalBinaryPatterns
from sklearn.utils import shuffle
# initialize the local binary patterns descriptor along with the data and label lists
desc = LocalBinaryPatterns(p, r)
data = []
labels = []
test_data = []
test_labels = []
start_time = time.time()
# loop over the training images
for imagePath in paths.list_files(trainingPath, validExts=(".png",".ppm")):
# load the image, convert it to grayscale, and describe it
image = cv2.imread(imagePath)
gray = np.matrix(cv2.cvtColor(image, cv2.COLOR_BGR2GRAY))
resized_image = cv2.resize(gray, (32, 32))
hist = desc.describe(resized_image)
hist = hist / max(hist)
# extract the label from the image path, then update the
# label and data lists
labels.append(imagePath.split("/")[-2])
data.append(hist)
# loop over the testing images
for imagePath in paths.list_files(testingPath, validExts=(".png",".ppm")):
# load the image, convert it to grayscale, describe it, and classify it
image = cv2.imread(imagePath)
gray = np.matrix(cv2.cvtColor(image, cv2.COLOR_BGR2GRAY))
resized_image = cv2.resize(gray, (32, 32))
hist = desc.describe(resized_image)
hist = hist / max(hist)
# extract the label from the image path, then update the
# label and data lists
test_labels.append(imagePath.split("/")[-2])
test_data.append(hist)
feature_extraction_runtime = (time.time() - start_time)
data = np.array(data)
labels = np.array(labels)
test_data = np.array(test_data)
test_labels = np.array(test_labels)
data, labels = shuffle(data,labels)
print "[INFO] LBP Features are ready!"
print "Total image:", len(data) + len(test_data)
print "Feature extraction runtime:", feature_extraction_runtime
print "Average for one image:", feature_extraction_runtime / (len(data) + len(test_data))
return (data, labels, test_data, test_labels)
def readImagesAndAnnotations(self):
self.imagePaths = list(
paths.list_files(self.inputPath, validExts=(".jpg", ".jpeg")))
self.labelPaths = list(
paths.list_files(self.inputPath, validExts=(".txt")))
if (len(self.imagePaths) != len(self.labelPaths)):
raise Exception(
"The number of images is different to the number of annotations"
)
def PascalVOC2YOLO(dataset_path, dataset_name):
listaFicheros_train = list(
paths.list_files(os.path.join(dataset_path, dataset_name, "train"), validExts=(".xml")))
listaFicheros_test = list(
paths.list_files(os.path.join(dataset_path, dataset_name, "test"), validExts=(".xml")))
result_path = os.path.join(dataset_path, dataset_name)
if (not (os.path.exists(os.path.join(result_path,"train","labels"))) and not(os.path.exists(os.path.join(result_path,"test","labels")))):
os.makedirs(os.path.join(result_path,"train","labels"))
os.makedirs(os.path.join(result_path,"test","labels"))
for anno in listaFicheros_train:
write_anno(anno,os.path.join(result_path,"train","labels"))
for anno_test in listaFicheros_test:
write_anno(anno_test, os.path.join(result_path,"test","labels"))
def getDataFrame(dataSet = None, pathLoadDataSet = None):
frames = []
if(dataSet == None):
if(pathLoadDataSet == None):
pathLoadDataSet = "dataset"
dsPaths = list(paths.list_files(pathLoadDataSet, "csv"))
dsPaths = np.array(sorted(dsPaths))
frames = [pd.read_csv(dsPath) for dsPath in dsPaths]
else:
dsPaths = np.genfromtxt("dataset/dataset-default.csv", delimiter=',')
frames = [pd.DataFrame(dataSet), pd.DataFrame(dsPaths)]
for frame in frames:
columns = ['tag']
columns += ['col_' + str(col) for col in range(len(frame.columns) -1)]
frame.columns = columns
frame['tag'] = pd.Categorical(frame.tag)
dfPadrao = pd.concat(frames, axis=0)
if(dfPadrao.isnull().values.any()):
dfPadrao = dfPadrao.fillna(0)
return dfPadrao
def load_image(path):
imagePaths = list(paths.list_files(path, validExts=(".jpg")))
random.shuffle(imagePaths)
if imagePaths == []: print("need more gifs at :" + path)
imagePath = imagePaths[0]
return imagePath
def generate_main_pickle_file(encodings_folder, main_pickle_file):
"""The low-level method to generate main encoding file from existing face encodings via merging separated encoding pickle files to one.
Args:
encodings_folder (str): The folder that is keep all faces's encoded data files.
main_pickle_file (str): The file that is keep merged all faces's encoded data.
"""
main_decoding = {"encodings": [], "names": []}
encoding_files = list(paths.list_files(encodings_folder))
for encoding_file in encoding_files:
encoding = open(encoding_file, "rb")
decoding = pickle.load(encoding)
main_decoding["encodings"].extend(decoding["encodings"])
main_decoding["names"].extend(decoding["names"])
encoding.close()
main_encoding = open(main_pickle_file, "wb")
main_encoding.write(pickle.dumps(main_decoding))
main_encoding.close()
def nonMaximumSupression(detections_path):
output_path = detections_path[:detections_path.rfind("/")]
listdirmodels = [
p for p in os.listdir(detections_path) if "detection" in p
]
annotationsFiles = list(
paths.list_files(os.path.join(listdirmodels[0]), validExts=(".xml")))
for an in annotationsFiles:
boxes = []
classesBoxes = []
fileName = an.split("/")[-1]
# boxes += extractBoxes(an)
for dir in listdirmodels:
if os.path.isdir(dir):
ImageBoxes, classesB = extractBoxes(os.path.join(
dir, fileName))
if len(ImageBoxes) != 0:
boxes = boxes + ImageBoxes
classesBoxes = classesBoxes + classesB
# boxes=[extractBoxes(os.path.join(dir,fileName)) for dir in listdirmodels if os.path.isdir(dir)]
boxes = np.array(boxes)
classesBoxes = np.array(classesBoxes)
if (len(boxes) != 0):
boxes, modes = non_max_suppression_fast(boxes, classesBoxes, 0.45)
if not os.path.exists(os.path.join(output_path, "detectionEnsemble")):
os.makedirs(os.path.join(output_path, "detectionEnsemble"))
xml = generateXML(an, boxes, modes, "detectionEnsemble")
file = open(os.path.join(".", "detectionEnsemble", fileName), 'w')
file.write(xml)
def load_data(directory, image_ext):
"""
Load the training data, given a directory and the images file extension.
Returns the data as a numpy a array, the labels and class_weights. class_weights
are necesary to deal with class imabalance problems
:param directory: directory where the data is saved
:param image_ext: extension for the images to train
:param one_hot: default False, whether Y should be converted to one_hot
:return: data, labels, class_weights
"""
image_paths = list(paths.list_files(directory, validExts=(image_ext)))
data = []
labels = []
for (i, image_path) in enumerate(image_paths):
label = image_path.split(os.path.sep)[-1].split('-')[0]
image = cv2.imread(image_path)
image = cv2.resize(image, (32, 32), interpolation=cv2.INTER_AREA)
image = img_to_array(image)
# image = np.expand_dims(image, axis=0)
data.append(image)
labels.append(label)
data = np.array(data).astype('float') / 255.0
labels = np.array(labels)
class_weights = compute_class_weights(labels)
one_hot_labels = convert_to_one_hot(labels)
return data, labels, class_weights, one_hot_labels
def generate_specs():
wav_spectrogram = None
print("[INFO] Generating Spectograms from chunks...")
wavPaths = sorted(list(paths.list_files("wav")))
random.seed(22)
random.shuffle(wavPaths)
#MEL SPECTOGRAMS
for wav in wavPaths:
rate, data = wavfile.read(wav)
data = butter_bandpass_filter(data, lowcut, highcut, rate, order=1)
# Only use a short clip for our demo
if np.shape(data)[0] / float(rate) > 10:
print("?")
data = data[0:rate * 10]
#NORMAL SPECTOGRAM
wav_spectrogram = pretty_spectrogram(data.astype('float64'),
fft_size=fft_size,
step_size=step_size,
log=True,
thresh=spec_thresh)
pltsave.imsave("spectograms/specs/spec_" + wav.split("\\")[1] + ".png",
wav_spectrogram)
#fig, ax = plt.subplots(nrows=1,ncols=1, figsize=(8,5))
#cax = ax.matshow(np.transpose(wav_spectrogram), interpolation='nearest', aspect='auto', cmap=plt.cm.afmhot, origin='lower')
#cax.write_png("specs/spec_"+wav.split("/")[1]+".png")
mel_spec = make_mel(wav_spectrogram,
mel_filter,
shorten_factor=shorten_factor)
pltsave.imsave(
"spectograms/specs_mel/m_spec_" + wav.split("\\")[1] + ".png",
mel_spec)
def pre_compute_features(self, db):
precompute_features = []
DATASET_FOLDER = 'dataset/' + db + '/'
i = 0
for name in os.listdir(DATASET_FOLDER):
userDir = os.path.join(DATASET_FOLDER, name)
for user in os.listdir(userDir):
# Load a sample picture and learn how to recognize it.
data_image = face_recognition.load_image_file(os.path.join(userDir, user))
i = i + 1
print(str(i) + "/" + str(len(list(paths.list_files(DATASET_FOLDER)))))
print(user)
face_locations = np.asarray(face_recognition.face_locations(data_image))
if len(face_locations) > 0:
if self.align_on:
faces = np.asarray(face_alignment(data_image))
if len(faces) > 0:
data_image = faces[0]
else:
data_image, cropped = crop_face(data_image, np.asarray(face_locations[0]), margin=-10, size=self.face_size)
face_locations = np.asarray(face_recognition.face_locations(data_image))
if self.augm_on:
faces_augm = face_augmentation(data_image, 5)
for img in faces_augm:
face_encodings = self.predict_encodings(img, face_locations)
if face_encodings is not None and len(face_encodings) >= 1:
precompute_features.append({"name": user, "features": face_encodings[0]})
face_encodings = self.predict_encodings(data_image, face_locations)
if face_encodings is not None and len(face_encodings) >= 1:
precompute_features.append({"name": user, "features": face_encodings[0]})
pickle_stuff(self.features_files, precompute_features)
def show_local_animation(emotion):
print(emotion)
frame_montage = montage_copy.copy()
print(EMOTIONS_PATH + "/" + emotion)
imagePaths = list(
paths.list_files(EMOTIONS_PATH + "/" + emotion, validExts=(".mp4")))
print(imagePaths)
random.shuffle(imagePaths)
if imagePaths == []: print("need more gifs at :" + emoticon)
imagePath = imagePaths[0]
subject = translate_emotion(emotion)
cv2.putText(frame_montage, subject, (10, frame_montage.shape[0] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 2.45, (255, 255, 255), 2)
frames = grab_frames(imagePath)
for framex in frames:
frame_to_show = imutils.resize(framex, width=880, height=880)
(h, w) = frame_to_show.shape[:2]
h_montage = frame_montage.shape[0]
if h > h_montage:
print("crop image at :" + str(h_montage))
frame_to_show = frame_to_show[0:h_montage, 0:w]
x_montage = insert_animation(frame_montage, frame_to_show, alpha=1.0)
cv2.imshow("Montage", x_montage)
# time.sleep(0.01)
if cv2.waitKey(1) & 0xFF == ord("q"):
break
def move_data_to_datasets():
imgPaths = list(paths.list_files("../do_An_3/dataset/"))
for imgPath in imgPaths:
filename = imgPath.split("/")[-1]
img = cv2.imread(imgPath)
img = cv2.resize(img, (64, 64), cv2.INTER_AREA)
cv2.imwrite("../finall/datasets/" + filename, img)
def mainDataset(imagesPath, outputDataset, conf, pathPesos, fichNames,
fichCfg):
# Give the configuration and weight files for the model and
# load the network using them.
if os.path.exists(outputDataset) == False:
os.mkdir(outputDataset)
dirPath = os.path.dirname(os.path.realpath(__file__))
#fichNames = glob.glob(pathPesos+"/*.names")
#classesFile = fichNames[0]
classes = None
with open(fichNames, 'rt') as f:
classes = f.read().rstrip('\n').split('\n')
#fichCfg = glob.glob(pathPesos + "/*.cfg")
#modelConfiguration = fichCfg[0]
#modelWeights = glob.glob(pathPesos+"/*.weights")
# Invocamos a la funcion con dichos parametros y mostramos el resultado por pantalla
images = list(
paths.list_files(imagesPath,
validExts=(".jpg", ".jpeg", ".png", ".bmp", ".tiff",
".tif")))
#for peso in modelWeights:
net = cv.dnn.readNetFromDarknet(fichCfg, pathPesos) #peso)
net.setPreferableBackend(cv.dnn.DNN_BACKEND_OPENCV)
net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU)
#nomPeso = os.path.basename( pathPesos)#peso)
#nombre = os.path.splitext(nomPeso)[0]
for image in images:
generateXMLFromImage(image, outputDataset, net, classes,
conf) #+'/'+nombre, net)
def load_data(path, label_encoder):
data = {
"train_o": [],
"train_i": [],
"val_o": [],
"val_i": [],
"test_o": [],
"test_i": []
}
for data_type in ["train", "val", "test"]:
csv_files_path = list(
paths.list_files(os.path.join(path, data_type), ".csv"))
for csv_path in tqdm(csv_files_path):
df = pd.read_csv(csv_path)
df = df["close"]
input_data = df.values
input_data = Scaler().fit_transform(input_data.reshape(
-1, 1)).flatten()
output_data = os.path.normpath(csv_path)
output_data = output_data.split(os.path.sep)[-2]
data[f"{data_type}_o"].append(label_encoder[output_data])
data[f"{data_type}_i"].append(input_data)
for key in data:
data[key] = np.asarray(data[key])
return data
def evaluate(self):
aux_path = os.path.join("map", self.dataset_name)
if (not (os.path.exists(aux_path))):
os.makedirs(os.path.join(aux_path))
if( not (os.path.exists(os.path.join(aux_path,"labels")))):
os.makedirs(os.path.join(aux_path,"labels"))
if (not (os.path.exists(os.path.join(aux_path, "detection")))):
os.makedirs(os.path.join(aux_path,"detection"))
shutil.copy(os.path.join(self.dataset_path, "classes.names"), os.path.join(aux_path, "classes.names"))
image_paths = list(paths.list_files(os.path.join(self.dataset_path, "test"), validExts=(".jpg")))
for image in image_paths:
name = os.path.basename(image).split('.')[0]
shutil.copy(image, os.path.join(aux_path,"labels",os.path.basename(image)))
shutil.copy(image, os.path.join(aux_path,"detection",os.path.basename(image)))
anno_splited = os.path.join(self.dataset_path, "test", "Annotations", name + ".xml")
shutil.copy(anno_splited, os.path.join(aux_path,"labels",name + ".xml"))
os.system("python3 map/pascal2yolo_labels.py -d " + os.path.join(os.path.join(aux_path,"labels/") + " -f " + os.path.join(aux_path,"classes.names")))
if not(self.ensemble):
self.predictor.predict(os.path.join(aux_path,"detection/"))
# In this moment we have the images predicted. Now we are going to tranform the predicted annotations to Darknet format
os.system("python3 map/pascal2yolo_detection.py -d " + os.path.join(
os.path.join(aux_path,"detection/") + " -f " + os.path.join(aux_path, "classes.names")))
os.system("find `pwd`/map/" + self.dataset_name+"/labels -name '*.txt' > " + aux_path + "/test.txt")
os.system("./map/darknet detector compare " + os.path.join(aux_path,"test.txt") + " " + os.path.join(aux_path,"classes.names") + " > " + aux_path + "/" + str(self.model_name)+ "results.txt")
# shutil.rmtree(os.path.join(aux_path,"detection"))
shutil.move(os.path.join(aux_path,"detection"),os.path.join(aux_path,"detection"+str(self.model_name)))
shutil.rmtree(os.path.join(aux_path,"labels"))
# Esto es por si se quiere mover el archivo con lo resultados para que esten todos en la misma ubicacion
shutil.copy( aux_path + "/" + str(self.model_name)+ "results.txt", os.path.join( aux_path, self.dataset_name))
os.remove(os.path.join(aux_path,"classes.names"))
def test_train_val_splits(
self,
paths_to_images: str,
test_train_split_ratio: float,
test_val_split_ratio: str,
) -> list:
# grab all image paths then construct the training, validating and testing splits
self.image_paths = list(paths.list_files(paths_to_images))
random.shuffle(self.image_paths)
self.division_i = int(len(self.image_paths) * test_train_split_ratio)
self.trainimage_paths = self.image_paths[:self.division_i]
self.test_val_split_paths = self.image_paths[self.division_i:]
self.division_j = int(
len(self.test_val_split_paths) * test_val_split_ratio)
self.testimage_paths = self.test_val_split_paths[:self.division_j]
self.valimage_paths = self.test_val_split_paths[self.division_j:]
# create the list of datasets to build
self.dataset = [
("train", self.trainimage_paths, self.train_csv),
("test", self.testimage_paths, self.test_csv),
("val", self.valimage_paths, self.val_csv),
]
return self.dataset
def datasetSplit( Nproyecto, darknetPath, pathImages, porcentaje):
listaFicheros = list(paths.list_files(pathImages,validExts=(".jpg")))
train_list,test_list, _ ,_ = train_test_split(listaFicheros, listaFicheros, train_size=porcentaje)
#creamos la estructura de carpetas, la primera contendra las imagenes del entrenamiento
os.makedirs(os.path.join(darknetPath , Nproyecto , 'train', 'JPEGImages'))
#esta carpeta contendra las anotaciones de las imagenes de entrenamiento
os.makedirs(os.path.join(darknetPath , Nproyecto , 'train', 'labels'))
#y esta ultima carpeta va a contener tanto las imagenes como los ficheros de anotaciones del test
os.makedirs(os.path.join(darknetPath , Nproyecto , 'test', 'JPEGImages'))
#para las imagenes de entrenamiento
for file in train_list:
#obtenemos el fichero .txt asociado
ficherolabel = file[0:file.rfind('.')]+'.xml'
#obetenemos el nombre de los ficheros
name = os.path.basename(file).split('.')[0]
#movemos las imagenes a la carpeta JpegImages
shutil.copy(file, os.path.join(darknetPath , Nproyecto , 'train', 'JPEGImages',name+'.jpg'))
#movemos las anotaciones a la carpeta
shutil.copy(ficherolabel,os.path.join(darknetPath , Nproyecto , 'train', 'labels',name+'.xml'))
#para las imagenes de entrenamiento
for file in test_list:
#obtenemos el fichero .txt asociado
ficherolabel = file[0:file.rfind('.')]+'.xml'
#obetenemos el nombre de los ficheros
name = os.path.basename(file).split('.')[0]
#movemos las imagenes a la carpeta JpegImages
shutil.copy(file, os.path.join(darknetPath , Nproyecto , 'test', 'JPEGImages',name+'.jpg'))
#movemos las anotaciones a la carpeta
shutil.copy(ficherolabel, os.path.join(darknetPath , Nproyecto , 'test', 'JPEGImages',name+'.xml'))
def get_hog_features(trainingSetPath, win_stride, padding, mean_shift, scale):
# evaluate the command line arguments (using the eval function like
# this is not good form, but let's tolerate it for the example)
winStride = eval(win_stride)
padding = eval(padding)
meanShift = True if mean_shift > 0 else False
# initialize the HOG descriptor/person detector
hog = cv2.HOGDescriptor()
#hog.setSVMDetector(cv2.HOGDescriptor_getDefaultPeopleDetector()) # CHECK FOR IT!
# initialize the local binary patterns descriptor along with the data and label lists
labels = []
data = []
test_labels = []
test_data = []
# loop over the training images
for imagePath in paths.list_files(trainingSetPath, validExts=(".png",".ppm")):
# open image
img = cv2.imread(imagePath, cv2.IMREAD_GRAYSCALE)
# resized_image = cv2.resize(img, (32, 32)) # RESIZING
rects, weights = hog.detectMultiScale(img, winStride=winStride,
padding=padding, scale=scale, useMeanshiftGrouping=meanShift)
print (rects)
# get hog features
"""STAYED HERE!"""
# extract the label from the image path, then update the
# label and data lists
labels.append(imagePath.split("/")[-2])
data.append(weights)
def tta(model, myTechniques, pathImg, option):
fichs = os.listdir(pathImg)
# 1. Create tmp folder
os.mkdir(pathImg + '/tmp')
# move imgs to tmp
for fich in fichs:
shutil.copy(pathImg + '/' + fich, pathImg + '/tmp')
imgFolder = pathImg
os.mkdir(pathImg + '/../salida')
# 3. Classification
for technique in myTechniques:
function.clasification(imgFolder, technique)
# we get all the folders we have created
listDirOut = []
for filename in os.scandir(pathImg + '/../salida'):
if os.path.isdir(pathImg + '/../salida/' + filename.name) == True:
listDirOut.append(pathImg + '/../salida/' + filename.name)
for dir in listDirOut:
for img in os.scandir(dir + '/tmp'):
img1 = img.name[(img.name).find("_") + 1:]
img2 = img1[img1.find("_") + 1:]
shutil.move(dir + '/tmp/' + img.name, dir + '/' + img2)
os.rmdir(dir + '/tmp')
# 4. Generate xml
for dir in listDirOut:
model.predict(dir, dir)
# 5. Detection
for dir in listDirOut:
tec = dir.split("/")
function.detection(dir, tec[len(tec) - 1])
for dir in listDirOut:
for img in os.listdir(dir):
if os.path.isdir(dir + '/' + img) == False:
os.remove(dir + '/' + img)
for img in os.listdir(dir + '/tmp'):
img1 = img[img.find("_") + 1:]
img2 = img1[img1.find("_") + 1:]
shutil.move(dir + '/tmp/' + img, dir + '/' + img2)
os.rmdir(dir + '/tmp')
# 6. Ensemble
for dirOut in os.listdir(pathImg + '/../salida/'):
for file in list(
paths.list_files(pathImg + '/../salida/' + dirOut,
validExts=(".jpg", ".jpeg", ".png", ".bmp",
".tiff", ".tif"))):
os.remove(file)
ensembleOptions.ensembleOptions(pathImg + '/../salida/', option)
for xml in os.scandir(pathImg + '/../salida/output/'):
shutil.copy(pathImg + '/../salida/output/' + xml.name, pathImg + '/')
if notebook is False:
shutil.rmtree(pathImg + '/../salida/')
shutil.rmtree(pathImg + '/tmp')
def models(listaModels, pathImg, option, combine=False):
if combine == 'False':
# 1. First we create the folder where we will store the resulting images and create as many folders as we have models
if not os.path.exists(pathImg + '/../salida'):
os.mkdir(pathImg + '/../salida')
for model in listaModels:
os.mkdir(pathImg + '/../salida/' +
os.path.splitext(os.path.basename(model.pathPesos))[0])
# 2. We create a list with the folders we have created
listDirOut = []
for filename in os.listdir(pathImg + '/../salida'):
if os.path.isdir(pathImg + '/../salida/' + filename) == True:
listDirOut.append(pathImg + '/../salida/' + filename)
# 3. we copy the images from the initial folder to each of the created folders
for dire in listDirOut:
for fich in os.listdir(pathImg):
shutil.copy(pathImg + '/' + fich, dire + '/')
# 4. Generate xml
for model in listaModels:
#If the model matches the name of the folder, we will predict it is only folder
for dir in os.listdir(pathImg + '/../salida/'):
if (os.path.splitext(
os.path.basename(pathImg + '/../salida/' +
model.pathPesos))[0]) == dir:
#Then we list the files in that folder
images = os.listdir(pathImg + '/../salida/' + dir)
model.predict(pathImg + '/../salida/' + dir,
pathImg + '/../salida/' + dir, 0.5)
list_dir = os.listdir(pathImg + '/../salida')
dest = "/mnt/output"
# shutil.move(pathImg+'/../salida/', '/mnt/output')
for sub_dir in list_dir:
os.makedirs(os.path.join("/mnt/output", sub_dir))
for file in os.listdir(
os.path.join(pathImg + '/../salida', sub_dir)):
dir_to_move = os.path.join(pathImg + '/../salida', sub_dir,
file)
shutil.move(dir_to_move, os.path.join(dest, sub_dir, file))
else:
# 5. We perform the ensemble method
for dirOut in os.listdir(pathImg):
for file in list(
paths.list_files(pathImg + dirOut,
validExts=(".jpg", ".jpeg", ".png",
".bmp", ".tiff", ".tif"))):
os.remove(file)
ensembleOptions.ensembleOptions(pathImg, option)
visualize_images(pathImg)
generate_xml(pathImg)
os.makedirs("/mnt/output/output/")
for sub_dir in os.listdir(pathImg + "output/"):
dir_to_ = os.path.join(pathImg + "output", sub_dir)
shutil.move(dir_to_, os.path.join("/mnt/output", 'output',
sub_dir))
def get_all_tagged_tittle():
with open(f'tagged.txt', 'w', encoding="utf-8") as f:
for file in paths.list_files('.', validExts=('.json',)):
try:
j = json.load(open(file, 'r', encoding='utf-8'))
f.write(j['name'] + '\n')
except Exception:
print(file)
def normalization(path_to_data):
"""Load all wavs file and split 2s/file
ex: path_to_data = './data/'
Args:
path_to_data (string): path to data folder
Returns:
No return
"""
try:
#list all folder
paths = list_files(path_to_data)
folders = set()
for path_ in paths:
folder_name = path_.split('/')[-2]
folders.add(folder_name)
folders = list(folders)
#data normalization, 2s/wav file
for folder in folders:
paths = list_files(path_to_data + folder + '/')
audios = AudioSegment.empty()
#read wav file and remove it
for path_ in paths:
wav = AudioSegment.from_wav(path_)
audios += wav
os.remove(path_)
L = len(audios)
N = int(L / 2000)
for i in range(N):
wav = audios[i * 2000:(i + 1) * 2000]
wav.export(path_to_data + folder + '/{}.wav'.format(i),
format='wav')
#destroy
del audios
del L
del N
except Exception as e:
print('Error: ', e)
def compruebeTXT(pathImages):
listaFicheros = list(paths.list_files(pathImages,validExts=(".jpg")))
for file in listaFicheros:
ficheroAnotaciones = file.split(".jpg")[0] + ".txt"
if(os.path.exists(ficheroAnotaciones)):
print('Todo correcto, continuamos')
else:
print('No existe el fichero de anotaciones')
def main(trainingSetPath, testingSetPath, cell_size=8, bin_size=8):
# initialize the local binary patterns descriptor along with the data and label lists
labels = []
data = []
test_labels = []
test_data = []
# loop over the training images
for imagePath in paths.list_files(trainingSetPath,
validExts=(".png", ".ppm")):
# open image
img = cv2.imread(imagePath, cv2.IMREAD_GRAYSCALE)
# resized_image = cv2.resize(img, (32, 32)) # RESIZING
# get hog features
hog = descriptor(img, cell_size=cell_size, bin_size=bin_size)
vector, image = hog.extract()
# vector = vector / max(vector) # NORMALAZING
print len(vector)
# extract the label from the image path, then update the
# label and data lists
labels.append(imagePath.split("/")[-2])
data.append(vector)
# loop over the testing images
for imagePath in paths.list_files(testingSetPath,
validExts=(".png", ".ppm")):
# open image
img = cv2.imread(imagePath, cv2.IMREAD_GRAYSCALE)
# resized_image = cv2.resize(img, (32, 32)) # RESIZING
# get hog features
hog = descriptor(img, cell_size=cell_size, bin_size=bin_size)
vector, image = hog.extract()
# vector = vector / max(vector) # NORMALAZING
print len(vector)
# extract the label from the image path, then update the
# label and data lists
test_labels.append(imagePath.split("/")[-2])
test_data.append(vector)
def transform(self):
listaFicheros_train = list(
paths.list_files(os.path.join(self.OUTPUT_PATH, self.DATASET_NAME,
"train"),
validExts=(".jpg")))
listaFicheros_test = list(
paths.list_files(os.path.join(self.OUTPUT_PATH, self.DATASET_NAME,
"test"),
validExts=(".jpg")))
outputPath = os.path.join(self.OUTPUT_PATH,
"VOC" + self.DATASET_NAME + "_" + self.model)
# outputPath = os.path.join(self.OUTPUT_PATH, "VOC" + self.DATASET_NAME)
shutil.copytree(
os.path.join(self.OUTPUT_PATH, self.DATASET_NAME, "train",
"JPEGImages"), os.path.join(outputPath, "JPEGImages"))
shutil.copytree(
os.path.join(self.OUTPUT_PATH, self.DATASET_NAME,
"train", "Annotations"),
os.path.join(outputPath, "Annotations"))
if (not (os.path.exists(os.path.join(outputPath, "ImageSets")))):
os.makedirs(os.path.join(outputPath, "ImageSets", "Main"))
shutil.copy(
os.path.join(self.OUTPUT_PATH, self.DATASET_NAME, "classes.names"),
outputPath)
traintxt = open(
os.path.join(outputPath, "ImageSets", "Main", "train.txt"), "w")
testtxt = open(
os.path.join(outputPath, "ImageSets", "Main", "test.txt"), "w")
for f_train in listaFicheros_train:
name = os.path.basename(f_train).split('.')[0]
traintxt.write(name + "\n")
for f_test in listaFicheros_test:
name = os.path.basename(f_test).split('.')[0]
testtxt.write(name + "\n")
shutil.copy(f_test, os.path.join(outputPath, "JPEGImages"))
ficherolabel = f_test[0:f_test.rfind('.')] + '.xml'
ficherolabel = ficherolabel.replace(
"JPEGImages",
"Annotations") # obetenemos el nombre de los ficheros
shutil.copy(ficherolabel, os.path.join(outputPath, "Annotations"))
def mainDataset(imagesPath):
# Leemos el parametro pasado por linea de comandos
#arg1 = sys.argv[1]
# Invocamos a la funcion con dichos parametros y mostramos el resultado por pantalla
images = list(
paths.list_files(imagesPath,
validExts=(".jpg", ".jpeg", ".png", ".bmp", ".tiff",
".tif")))
for image in images:
generateXMLFromImage(image)
def transform(self):
listaFicheros_train = list(paths.list_files(os.path.join(self.OUTPUT_PATH,self.DATASET_NAME,"train"), validExts=(".jpg")))
listaFicheros_test = list(paths.list_files(os.path.join(self.OUTPUT_PATH,self.DATASET_NAME,"test"), validExts=(".jpg")))
outputPath = os.path.join(self.OUTPUT_PATH, "VOC" + self.DATASET_NAME+"_"+str(self.model))
# outputPath = os.path.join(self.OUTPUT_PATH, "VOC" + self.DATASET_NAME)
shutil.copytree(os.path.join(self.OUTPUT_PATH,self.DATASET_NAME,"train","JPEGImages"), os.path.join(outputPath, "JPEGImages"))
shutil.copytree(os.path.join(self.OUTPUT_PATH,self.DATASET_NAME,"train","Annotations"), os.path.join(outputPath, "Annotations"))
if (not (os.path.exists(os.path.join(outputPath, "ImageSets")))):
os.makedirs(os.path.join(outputPath, "ImageSets", "Main"))
shutil.copy(os.path.join(self.OUTPUT_PATH, self.DATASET_NAME, "classes.names"), outputPath)
classescsv = open(os.path.join(outputPath,"classes.csv"), "w")
with open(os.path.join(outputPath,"classes.names")) as f:
classes = f.read()
classes = classes.split('\n')
rows = [",".join([c, str(i)]) for (i, c) in enumerate(classes)]
classescsv.write("\n".join(rows))
classescsv.close()
traintxt = open(os.path.join(outputPath, "ImageSets", "Main", "train.txt"), "w")
testtxt = open(os.path.join(outputPath, "ImageSets", "Main", "test.txt"), "w")
shutil.copy(os.path.join(outputPath,"classes.csv"), os.path.join(self.OUTPUT_PATH, self.DATASET_NAME, self.DATASET_NAME + "_classes.csv"))
"""classescsv = open(os.path.join(self.OUTPUT_PATH, self.DATASET_NAME, self.DATASET_NAME + "_classes.csv"), "w")
classes = [cl for cl in open(os.path.join(self.OUTPUT_PATH, self.DATASET_NAME, "classes.names"))]
rows = [c.replace('\n','')+","+ str(i)+"\n" for (i, c) in enumerate(classes) if i!=len(classes)-1]
for row in rows:
classescsv.write(row)
classescsv.close()"""
for f_train in listaFicheros_train:
name = os.path.basename(f_train).split('.')[0]
traintxt.write(name + "\n")
for f_test in listaFicheros_test:
name = os.path.basename(f_test).split('.')[0]
testtxt.write(name + "\n")
shutil.copy(f_test, os.path.join(outputPath, "JPEGImages"))
ficherolabel = f_test[0:f_test.rfind('.')] + '.xml'
ficherolabel = ficherolabel.replace("JPEGImages", "Annotations") # obetenemos el nombre de los ficheros
shutil.copy(ficherolabel, os.path.join(outputPath, "Annotations"))