def similarity_detection(set_name, cutoff=0.0):
data_path = os.path.join(config.data_path, set_name)
label_path = os.path.join(set_name, set_name + '.csv')
label_path = os.path.join(config.data_path, label_path)
#make new data_path directory if it does not exists
cam_1_path = os.path.join(data_path, '1')
cam_2_path = os.path.join(data_path, '2')
prev_img = None
prev_name = ""
rat_lst = []
res_lst = []
df = pd.read_csv(label_path, names=['cam_1', 'cam_2', 'action'], header=0)
ind = 1
entries = []
sample_length = len(df)
count = 0
for ind in range(sample_length):
util.progress_bar(ind, sample_length)
cam_1 = df['cam_1'].iloc[ind]
cam_2 = df['cam_2'].iloc[ind]
img_paths = [
os.path.join(cam_1_path, cam_1),
os.path.join(cam_2_path, cam_2)
]
action = df['action'].iloc[ind]
img_1 = cv2.imread(img_paths[0], 1)
img_2 = cv2.imread(img_paths[1], 1)
img = np.concatenate((img_1, img_2), axis=1)
img = img / 255.
if type(prev_img) != type(None):
diff = np.fabs(img - prev_img)
avg = (img + prev_img) / 2.
ratio = np.sum(diff) / np.sum(avg)
else:
ratio = 1.
if ratio < cutoff:
os.remove(img_paths[0])
os.remove(img_paths[1])
logger.debug("Image %s is a duplicate of %s" %
(str([cam_1, cam_2]), str(prev_name)))
count += 1
continue
entries.append([cam_1, cam_2, action])
prev_img = img
prev_name = [cam_1, cam_2]
if cutoff:
df = pd.DataFrame(entries, columns=['cam_1', 'cam_2', 'action'])
df.to_csv(label_path, index=False)
logger.info("\nDuplicates detected: %d" % count)
return entries
def download_and_unpack(blob, filename):
'''
Downloads files from the staging folder, unpacks them, then removes
original files from the staging directory. Adds files onto a database
for preprocessing step to work with
'''
logger.info("Downloading %s..." % filename)
local_filename = os.path.join(config.download_path, filename)
blob.download_to_filename(local_filename)
# Completed download
meta_file = ''
with zipfile.ZipFile(local_filename) as zp:
logger.info("-----Unpacking %s-----" % filename)
total = len(zp.namelist())
count = 0
progress_bar(count, total)
for name in zp.namelist():
if '.bmp' in name:
zp.extract(name, config.data_path)
elif '.meta' in name:
zp.extract(name, config.data_path)
meta_file = os.path.join(config.data_path, name)
elif '.csv' in name:
zp.extract(name, config.data_path)
count += 1
progress_bar(count, total)
if meta_file:
# add onto process queue
no_ext = os.path.splitext(filename)[0]
update_process_queue(meta_file, no_ext)
else:
logger.warning("Warning - %s meta file does not exist." % filename)
#After extraction, delete from bucket and machine intermediary files
if blob.exists():
try:
blob.delete()
except google.cloud.exceptions.NotFound as err:
pass
if os.path.exists(local_filename):
try:
os.remove(local_filename)
except OSError as err:
pass
def augment(set_name, equalize=False):
data_path = os.path.join(config.data_path, set_name)
label_path = os.path.join(set_name, set_name + '.csv')
label_path = os.path.join(config.data_path, label_path)
cam_1_path = os.path.join(data_path, '1')
cam_2_path = os.path.join(data_path, '2')
op_todo = [([op_list[0]]), ([op_list[1]]), ([op_list[2]]),
([op_list[0], op_list[2]]), ([op_list[1], op_list[2]])]
with open(label_path, 'r') as in_csv:
for line in in_csv:
if re.search(r"(flip|autocont|equalize|darken|brighten)", line):
util.progress_bar(1, 1)
return
#equalize images
if equalize:
with open(label_path, 'r') as in_csv:
reader = csv.reader(in_csv, delimiter=',')
attribute = next(reader, None)
entries = list(reader)
for entry in entries:
img_1_path = os.path.join(cam_1_path, entry[0])
img_2_path = os.path.join(cam_2_path, entry[1])
cam_1_img = preprocess.equalize(cv2.imread(img_1_path))
cam_2_img = preprocess.equalize(cv2.imread(img_2_path))
cv2.imwrite(filename=img_1_path, img=cam_1_img)
cv2.imwrite(filename=img_2_path, img=cam_2_img)
logger.info("Preprocessing images...")
with open(label_path, 'a+') as io_csv:
io_csv.seek(0)
reader = csv.reader(io_csv, delimiter=',')
attribute = next(reader, None)
entries = list(reader)
cnt_total = len(entries)
cnt_iter = 0
util.progress_bar(cnt_iter, cnt_total)
for entry in entries:
logger.debug("working on %s" % str(entry))
cnt_iter += 1
util.progress_bar(cnt_iter, cnt_total)
new_entries = process_image([cam_1_path, cam_2_path],
[entry[0], entry[1]], int(entry[-1]),
op_todo)
writer = csv.writer(io_csv, delimiter=',')
for new_entry in new_entries:
writer.writerow(new_entry)
time.sleep(0.1)
def augment(set_name, equalize=False):
'''
:param set_name: name of data set
:param equalize: whether or not to improve contrast of images (off by default)
:return: writes the new images and commands to the data set
'''
data_path = os.path.join(
config.data_path, set_name) # data_path = ROOT_DIR/data_sets/set_name
label_path = os.path.join(set_name, set_name +
'.csv') # label_path = set_name/set_name.csv
label_path = os.path.join(
config.data_path,
label_path) # label_path = ROOT_DIR/data_sets/set_name/set_name.csv
cam_1_path = os.path.join(
data_path, '1') # cam1or2 = ROOT_DIR/data_sets/set_name/1or2
cam_2_path = os.path.join(data_path, '2')
op_todo = [ # to do list of op_list
([op_list[0]]), # [0] darken
([op_list[1]]), # [1] brighten
([op_list[2]]), # [2] flip (mirror)
([op_list[0], op_list[2]]), # [3] darken, flip
([op_list[1], op_list[2]]) # [4] brighten, flip
]
with open(label_path, 'r') as in_csv: # open csv file in read
for line in in_csv: # Line by line
if re.search(r"(flip|autocont|equalize|darken|brighten)",
line): # ???
util.progress_bar(1, 1)
return
if equalize: # equalize images (default false)
with open(label_path, 'r') as in_csv: # open csv file in read
reader = csv.reader(in_csv,
delimiter=',') # read in data with ',' dln
attribute = next(reader, None) # ??? needed?
entries = list(reader) # reader broken into entries list
for entry in entries: # for each element in entry
img_1_path = os.path.join(
cam_1_path, entry[0]
) # img_x_path = ROOT_DIR/data_sets/set_name/x/entry[x]
img_2_path = os.path.join(cam_2_path, entry[1])
cam_1_img = preprocess.equalize(cv2.imread(
img_1_path)) # cam_x_img equalized (contrast increase)
cam_2_img = preprocess.equalize(
cv2.imread(img_2_path)) # using cv2 functions
cv2.imwrite(
filename=img_1_path,
img=cam_1_img) # cv2.imwrite to save boosted image to
cv2.imwrite(filename=img_2_path,
img=cam_2_img) # same directory (overwrite?)
logger.info(
"Preprocessing images...") # Begin log for actual preprocessing
with open(label_path,
'a+') as io_csv: # open csv file in append mode as io_csv
io_csv.seek(0) # sets position to 0 (seek.() default is 0?)
reader = csv.reader(
io_csv, delimiter=','
) # iterates over lines of csv file into reader dln = ','
attribute = next(reader, None) # gets next item from reader ???
entries = list(reader) # reader is broken into list
cnt_total = len(entries) # count entries
cnt_iter = 0 # index for progress bar
util.progress_bar(cnt_iter, cnt_total)
for entry in entries: # for all elements in entries
logger.debug("working on %s" % str(entry)) # log particular entry
cnt_iter += 1 # increase index
util.progress_bar(cnt_iter, cnt_total) # update progress bar
new_entries = process_image( # new entries created from proc_img function
[cam_1_path, cam_2_path
], # path = cams, name = entry[1/0], command = int(entry[-1])
[entry[0], entry[1]], # op_todo
int(entry[-1]), # last element in entry
op_todo)
writer = csv.writer(io_csv,
delimiter=',') # writer set with dlm = ','
for new_entry in new_entries: # for each value in entry
writer.writerow(new_entry) # write value to the row
time.sleep(0.1) # wait