def gen_celeba(): logger.info('loading CelebA') train_data, val_data = load_celeba() logger.info('total images, train: %d, val: %d', len(train_data), len(val_data)) def gen(data, db_name): remove_if_exists(db_name) logger.info('fill queues') q_in = [multiprocessing.Queue() for i in range(cfg.WORKER_N)] q_out = multiprocessing.Queue(1024) fill_queues(data, q_in) readers = [multiprocessing.Process(target=celeba_reader_func, args=(q_in[i], q_out)) \ for i in range(cfg.WORKER_N)] print("\n\nstarting write to DB\n\n") for p in readers: p.start() writer = multiprocessing.Process(target=celeba_writer_func, args=(q_out, db_name)) writer.start() for p in readers: p.join() #print(p) q_out.put(('finish', [])) writer.join() logger.info('writing train data, %d images', len(train_data)) gen(train_data, 'data/%snet_landmark_train' % cfg.NET_TYPE) logger.info('writing val data, %d images', len(val_data)) gen(val_data, 'data/%snet_landmark_val' % cfg.NET_TYPE)
def prepare(args): '''prepare data for lnet ''' logger.info('loading CelebA') train_data, val_data = load_celeba() def gen(data, is_train): txt = 'data/lnet_train.txt' if is_train else 'data/lnet_val.txt' remove(txt) q_in = [mp.Queue() for i in range(cfg.WORKER_N)] q_out = mp.Queue(1024) fill_queues(data, q_in) readers = [mp.Process(target=lnet_reader_func, args=(q_in[i], q_out)) \ for i in range(cfg.WORKER_N)] for p in readers: p.start() writer = mp.Process(target=lnet_writer_func, args=(q_out, txt)) writer.start() for p in readers: p.join() q_out.put(('finish', [])) writer.join() logger.info('writing train data') gen(train_data, True) logger.info('writing val data') gen(val_data, False)