def create_style_dataset(output_dir, input_dir, b_dir):
if not os.path.exists(output_dir):
os.makedirs(output_dir)
src_paths = []
dst_paths = []
skipped = 0
for src_path in im.find(input_dir):
name, _ = os.path.splitext(os.path.basename(src_path))
dst_path = os.path.join(output_dir, name + ".png")
if os.path.exists(dst_path):
skipped += 1
else:
src_paths.append(src_path)
dst_paths.append(dst_path)
print("skipping %d files that already exist" % skipped)
global total
total = len(src_paths)
print("processing %d files" % total)
global start
start = time.time()
with tf.compat.v1.Session() as sess:
for src_path, dst_path in zip(src_paths, dst_paths):
src = im.load(src_path)
dst = combine(src, src_path, b_dir)
im.save(dst, dst_path)
complete()
def main():
if not os.path.exists(a.output_dir):
os.makedirs(a.output_dir)
src_paths = []
dst_paths = []
for src_path in im.find(a.input_dir):
name, _ = os.path.splitext(os.path.basename(src_path))
dst_path = os.path.join(a.output_dir, name + ".png")
if not os.path.exists(dst_path):
src_paths.append(src_path)
dst_paths.append(dst_path)
global total
total = len(src_paths)
if a.workers == 1:
with tf.Session() as sess:
for src_path, dst_path in zip(src_paths, dst_paths):
process(src_path, dst_path)
complete()
else:
queue = tf.train.input_producer(zip(src_paths, dst_paths),
shuffle=False,
num_epochs=1)
dequeue_op = queue.dequeue()
def worker(coord):
with sess.as_default():
while not coord.should_stop():
try:
src_path, dst_path = sess.run(dequeue_op)
except tf.errors.OutOfRangeError:
coord.request_stop()
break
process(src_path, dst_path)
complete()
# init epoch counter for the queue
local_init_op = tf.local_variables_initializer()
with tf.Session() as sess:
sess.run(local_init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
for i in range(a.workers):
t = threading.Thread(target=worker, args=(coord, ))
t.start()
threads.append(t)
try:
coord.join(threads)
except KeyboardInterrupt:
coord.request_stop()
coord.join(threads)
def getLabelImages(label_folder):
ret = {}
labelIds = a.labels.split(',')
for lid in labelIds:
for label_path in im.find(label_folder):
if label_path.find('lbl'+lid) > 0: #if found the label
ret[lid] = label_path
break
return ret
def main():
if not os.path.exists(a.output_dir):
os.makedirs(a.output_dir)
src_paths = []
dst_paths = []
for src_path in im.find(a.input_dir):
name, _ = os.path.splitext(os.path.basename(src_path))
dst_path = os.path.join(a.output_dir, name + ".png")
src_paths.append(src_path)
dst_paths.append(dst_path)
with tf.Session() as sess:
for src_path, dst_path in zip(src_paths, dst_paths):
src = im.load(src_path)
dst = combine(src, src_path)
im.save(dst, dst_path)
def generate_font_skeleton_combine_images(label_file, output_dir):
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
# Set the path of hangul-skeleton-combine images in output directory. It will be used later for
# setting up hangul-skeleton-combine images path for hangul-skeleton-combine labels
image_dir = os.path.join(output_dir, 'images')
if not os.path.exists(image_dir):
os.makedirs(os.path.join(image_dir))
src_paths = []
dst_paths = []
# Check if the directory and images already exsist?
# If yes then skip those images else create the paths list
skipped = 0
for src_path in sorted(im.find(args.input_dir), key=os.path.getmtime):
name, _ = os.path.splitext(os.path.basename(src_path))
dst_path = os.path.join(image_dir, name + ".png")
if os.path.exists(dst_path):
skipped += 1
else:
src_paths.append(src_path)
dst_paths.append(dst_path)
print("skipping %d files that already exist" % skipped)
global total
total = len(src_paths)
print("processing %d files" % total)
global start
start = time.time()
if args.workers == 1:
with tf.Session() as sess:
for src_path, dst_path in zip(src_paths, dst_paths):
process(src_path, dst_path, image_dir)
complete()
def main():
if not os.path.exists(a.output_dir):
os.makedirs(a.output_dir)
src_paths = []
dst_paths = []
skipped = 0
for src_path in im.find(a.input_dir):
name, _ = os.path.splitext(os.path.basename(src_path))
dst_path = os.path.join(a.output_dir, name + ".png")
if os.path.exists(dst_path):
skipped += 1
else:
src_paths.append(src_path)
dst_paths.append(dst_path)
print("skipping %d files that already exist" % skipped)
global total
total = len(src_paths)
print("processing %d files" % total)
global start
start = time.time()
if a.operation == "edges":
# use a multiprocessing pool for this operation so it can use multiple CPUs
# create the pool before we launch processing threads
global edge_pool
edge_pool = multiprocessing.Pool(a.workers)
if a.workers == 1:
with tf.Session() as sess:
for src_path, dst_path in zip(src_paths, dst_paths):
if a.operation == "damage_random":
process_damage_random(src_path, dst_path)
else:
process(src_path, dst_path)
complete()
else:
queue = tf.train.input_producer(zip(src_paths, dst_paths),
shuffle=False,
num_epochs=1)
dequeue_op = queue.dequeue()
def worker(coord):
with sess.as_default():
while not coord.should_stop():
try:
src_path, dst_path = sess.run(dequeue_op)
except tf.errors.OutOfRangeError:
coord.request_stop()
break
if a.operation == "damage_random":
process_damage_random(src_path, dst_path)
else:
process(src_path, dst_path)
complete()
# init epoch counter for the queue
local_init_op = tf.local_variables_initializer()
with tf.Session() as sess:
sess.run(local_init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
for i in range(a.workers):
t = threading.Thread(target=worker, args=(coord, ))
t.start()
threads.append(t)
try:
coord.join(threads)
except KeyboardInterrupt:
coord.request_stop()
coord.join(threads)
def generate_hangul_skeleton_combine_images(labels_csv, label_file,
output_dir):
# 출력 디렉토리 유무 확인 후 생성
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
# Open the labels file from image-data of hangul images
# 한글 이미지가 있는 image-data 폴더로부터 레이블 파일을 읽어들임
labels_csv = io.open(labels_csv, 'r', encoding='utf-8')
labels_file = io.open(label_file, 'r',
encoding='utf-8').read().splitlines()
# Map characters to indices.
# 글자 레이블 정보와 인덱스를 맵핑
label_dict = {}
count = 0
for label in labels_file:
label_dict[label] = count
count += 1
# Build the lists.
# 각 이미지에 대한 레이블 정보를 인덱스로 변경한 레이블 리스트를 생성
labels = []
for row in labels_csv:
_, label = row.strip().split(',')
labels.append(label_dict[label])
# Set the path of hangul-skeleton-combine images in output directory. It will be used later for
# setting up hangul-skeleton-combine images path for hangul-skeleton-combine labels
# output 디렉토리 안에 combine 결과 이미지 경로를 설정 (이미지가 저장될 경로)
# 나중에 combine 레이블을 위한 combine 결과 이미지 경로를 설정하기 위해 사용될 것임
image_dir = os.path.join(output_dir, 'images-white')
if not os.path.exists(image_dir):
os.makedirs(os.path.join(image_dir))
# Create the skeleton labels file
# combine 결과 이미지 레이블 파일 생성
labels_csv = io.open(os.path.join(args.output_dir,
'skeleton-labels-map.txt'),
'w',
encoding='utf-8')
src_paths = []
dst_paths = []
# Check if the directory and images already exist?
# If yes then skip those images else create the paths list
# combine 결과 이미지가 이미 존재하는 지 확인
# 만약 존재한다면 그 이미지들을 스킵하고, 그렇지 않으면 경로 리스트를 생성함
skipped = 0
for src_path in sorted(im.find(args.input_dir)):
name, _ = os.path.splitext(os.path.basename(src_path))
dst_path = os.path.join(image_dir, name + ".png")
if os.path.exists(dst_path):
skipped += 1
else:
src_paths.append(src_path)
dst_paths.append(dst_path)
print("skipping %d files that already exist" % skipped)
global total
total = len(src_paths)
print("processing %d files" % total)
global start
start = time.time()
if args.workers == 1:
with tf.Session() as sess:
for src_path, dst_path in zip(src_paths, dst_paths):
process(src_path, dst_path, label_dict, labels, labels_csv,
image_dir)
complete()
import tensorflow as tf
import numpy as np
import tfimage as im
in_dir = '/home/jordi/Desktop/cross-domain-disen/DATA/MNIST-CDCB-copy/train'
out_dir = '/home/jordi/Desktop/cross-domain-disen/DATA/MNIST-CDCB-copy/new_train'
if not os.path.exists(out_dir):
os.makedirs(out_dir)
src_paths1 = []
src_paths2 = []
dst_paths = []
skipped = 0
for src_path in im.find(in_dir):
name, _ = os.path.splitext(os.path.basename(src_path))
dst_path = os.path.join(out_dir, name + ".png")
if os.path.exists(dst_path):
skipped += 1
else:
src_paths1.append(src_path)
src_paths2.append(src_path)
dst_paths.append(dst_path)
src_cutoff = len(src_paths1) / 2
src_paths1 = src_paths1[:, :src_cutoff]
src_paths2 = src_paths1[:, src_cutoff:]
dst_paths = dst_paths[:, :src_cutoff]
def main():
if not os.path.exists(a.output_dir):
os.makedirs(a.output_dir)
src_paths = [] #输入
dst_paths = [] #输出
#二者用于创建 train_pair
skipped = 0
for src_path in im.find(a.input_dir):
name,_ = os.path.splitext(os.path.basename(src_path)) #文件名
dst_path = os.path.join(a.output_dir,name + ".png") #创建输出文件全路径
if os.path.exists(dst_path): #如果输出已经存在，计数
skipped += 1
else:
src_paths.append(src_path)
dst_paths.append(dst_path)
print("skipping %d files that already exist" % skipped)
global total
total = len(src_paths) #input总数
print("processing %d files" % total)
global start
start = time.time()
if a.operation == "edges":
global edge_pool
edge_pool = multiprocessing.Pool(a.workers) #进程池
if a.workers == 1: #如果所用进程为1
with tf.Session() as sess:
for src_path,dst_path in zip(src_paths,dst_paths):
process(src_path,dst_path)
complete() #每进行一次for循环，complete()就会创建thread_lock，保证该进程进行不中断？？？，在进程执行过程中，会记录处理image的数量，及处理时间，剩余时间等内容，并返回
else:
queue = tf.train.input_producer(zip(src_paths,dst_paths),shuffle=False,num_epochs=1) #采用多进程
dequeue_op = queue.dequeue()
def worker(coord):
with sess.as_default():
while not coord.should_stop(): #进程未终止时
try:
src_path,dst_path = sess.run(dequeue_op) #出队
except tf.errors.OutOfRangeError:
coord.request_stop()
break
process(src_path,dst_path) #执行process()
complete()
local_init_op = tf.local_variables_initializer() #进行变量初始化，为什么是局部变量？？？ 什么时候执行全局初始化，什么时候执行local初始化？？？
with tf.Session() as sess:
sess.run(local_init_op)
coord = tf.train.Coordinator() #进行进程管理
threads = tf.train.start_queue_runners(coord=coord)
for i in range(a.workers):
t = threading.Thread(target=worker,args=(coord,)) #创造多个进程a.workers，执行process(src_path,dst_path)
t.start() #开始执行
threads.append(t)
try:
coord.join(threads)
except KeyboardInterrupt:
coord.request_stop()
coord.join(threads)
def main():
if not os.path.exists(a.output_dir):
os.makedirs(a.output_dir)
src_paths = []
dst_paths = []
skipped = 0
for src_path in im.find(a.input_dir):
name, _ = os.path.splitext(os.path.basename(src_path))
dst_path = os.path.join(a.output_dir, name + ".png")
if os.path.exists(dst_path):
skipped += 1
else:
src_paths.append(src_path)
dst_paths.append(dst_path)
print("skipping %d files that already exist" % skipped)
global total
total = len(src_paths)
print("processing %d files" % total)
global start
start = time.time()
if a.operation == "edges":
# use a multiprocessing pool for this operation so it can use multiple CPUs
# create the pool before we launch processing threads
global edge_pool
edge_pool = multiprocessing.Pool(a.workers)
if a.workers == 1:
with tf.Session() as sess:
for src_path, dst_path in zip(src_paths, dst_paths):
process(src_path, dst_path)
complete()
else:
queue = tf.train.input_producer(zip(src_paths, dst_paths), shuffle=False, num_epochs=1)
dequeue_op = queue.dequeue()
def worker(coord):
with sess.as_default():
while not coord.should_stop():
try:
src_path, dst_path = sess.run(dequeue_op)
except tf.errors.OutOfRangeError:
coord.request_stop()
break
process(src_path, dst_path)
complete()
# init epoch counter for the queue
local_init_op = tf.local_variables_initializer()
with tf.Session() as sess:
sess.run(local_init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
for i in range(a.workers):
t = threading.Thread(target=worker, args=(coord,))
t.start()
threads.append(t)
try:
coord.join(threads)
except KeyboardInterrupt:
coord.request_stop()
coord.join(threads)
def main():
if not tf.io.gfile.exists(a.output_dir):
tf.io.gfile.makedirs(a.output_dir)
if a.operation == "edges" and a.crop:
try:
if not tf.io.gfile.exists(a.crop_dir):
tf.io.gfile.makedirs(a.crop_dir)
except Exception as e:
raise Exception("invalid crop_dir: {:s}".format(e))
src_paths = []
dst_paths = []
skipped = 0
for src_path in im.find(a.input_dir):
name, _ = os.path.splitext(os.path.basename(src_path))
dst_path = os.path.join(a.output_dir, name + ".png")
if tf.io.gfile.exists(dst_path):
skipped += 1
else:
src_paths.append(src_path)
dst_paths.append(dst_path)
print("skipping %d files that already exist" % skipped)
global total
total = len(src_paths)
print("processing %d files" % total)
global start
start = time.time()
if a.workers == 1:
with tf.Session() as sess:
for src_path, dst_path in zip(src_paths, dst_paths):
process(src_path, dst_path)
complete()
else:
queue = tf.train.input_producer(zip(src_paths, dst_paths),
shuffle=False,
num_epochs=1)
dequeue_op = queue.dequeue()
def worker(coord):
with sess.as_default():
while not coord.should_stop():
try:
src_path, dst_path = sess.run(dequeue_op)
except tf.errors.OutOfRangeError:
coord.request_stop()
break
process(src_path, dst_path)
complete()
# init epoch counter for the queue
local_init_op = tf.local_variables_initializer()
with tf.Session() as sess:
sess.run(local_init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
for i in range(a.workers):
t = threading.Thread(target=worker, args=(coord, ))
t.start()
threads.append(t)
try:
coord.join(threads)
except KeyboardInterrupt:
coord.request_stop()
coord.join(threads)
def main():
if not os.path.exists(a.output_dir_labels):
os.makedirs(a.output_dir_labels)
if not os.path.exists(output_train_directory_labels):
os.makedirs(output_train_directory_labels)
if not os.path.exists(output_test_directory_labels):
os.makedirs(output_test_directory_labels)
if not os.path.exists(output_val_directory_labels):
os.makedirs(output_val_directory_labels)
processInputImages = a.resize or a.crop
if not os.path.exists(a.output_dir_images) and processInputImages:
os.makedirs(a.output_dir_images)
if not os.path.exists(output_train_directory_images) and processInputImages:
os.makedirs(output_train_directory_images)
if not os.path.exists(output_test_directory_images) and processInputImages:
os.makedirs(output_test_directory_images)
if not os.path.exists(output_val_directory_images) and processInputImages:
os.makedirs(output_val_directory_images)
#cropped images directory
splits = ['train', 'test', 'val']
src_paths = []
dst_paths_labels = []
dst_paths_images = []
skipped = 0
for split in splits:
split_folder = os.path.join(a.input_dir, split)
for src_path in im.find(split_folder):
name, _ = os.path.splitext(os.path.basename(src_path))
dst_path_label = os.path.join(a.output_dir_labels, split)
dst_path_label = os.path.join(dst_path_label, name + ".png")
dst_path_image = os.path.join(a.output_dir_images, split)
dst_path_image = os.path.join(dst_path_image, name + ".png")
if os.path.exists(dst_path_label) or os.path.exists(dst_path_image):
skipped += 1
else:
src_paths.append(src_path)
dst_paths_labels.append(dst_path_label)
dst_paths_images.append(dst_path_image)
print("skipping %d files that already exist" % skipped)
global total
total = len(src_paths)
print("processing %d files" % total)
global start
start = time.time()
if a.workers == 1:
with tf.Session() as sess:
for src_path, dst_path_label, dst_path_image in zip(src_paths, dst_paths_labels, dst_paths_images):
name, _ = os.path.splitext(os.path.basename(src_path))
print 'Name: ' + name
label_folder = os.path.join(a.label_images_dir, name)
label_image_paths = getLabelImages(label_folder)
print label_image_paths
color_dict = getLabelToColorDictionary()
label_img = getLabelImage(label_image_paths, color_dict)
if processInputImages:
processedImage = im.load(src_path)
if a.crop:
crop_reference = getCropReference(label_image_paths)
processedImage = crop(processedImage, crop_reference)
label_img = crop(label_img, crop_reference)
if a.resize:
processedImage = resize(processedImage)
label_img = resize(label_img)
im.save(processedImage, dst_path_image)
im.save(label_img, dst_path_label)
complete()
