def __get_data(self, data_file): # Data for fitting generator data = [img for _, img, _ in read_csv(data_file).to_records()] size = len(data) data = np.zeros((size, ) + self.img_shape, dtype=K.floatx()) for i in tqdm(range(size)): I = Image.open(expand_path(data[i])) I = I.convert('L') # convert image to grayscale I = I.resize(self.img_shape[1:]) I = img_to_array(I) data[i] = I
def load(self, path): """ Parse a configuration file and generate all needed object. Raise a ValueError if there are at least one error raised when reading the configuration file. """ def prepare_value(value): value.update(self.force) return value if path in self.already_loaded: logger.warning('file %s has already been loaded, skip -- ' 'you might want to check for cycles', path) return self.already_loaded.append(path) logger.debug('open configuration %s', path) errors = [] with open(path, 'r') as file: data = yaml.load(file) # imports imports = data.get('imports') if imports: base_dir = os.path.dirname(path) for i_path in utils.as_list_of(str, imports): i_path = utils.expand_path(base_dir, i_path, expand_vars=True) self.load(i_path) # syncs syncs = data.get('syncs') if syncs: syncs = [ (name, Sync(self, name, **prepare_value(value))) for name, value in syncs.items() ] self.syncs.update(syncs) if errors: raise ValueError('configuration contains {} errors: abort' .format(len(errors)))
def main(_): pp.pprint(flags.FLAGS.__flags) # expand user name and environment variables FLAGS.data_dir = expand_path(FLAGS.data_dir) FLAGS.out_dir = expand_path(FLAGS.out_dir) FLAGS.out_name = expand_path(FLAGS.out_name) FLAGS.checkpoint_dir = expand_path(FLAGS.checkpoint_dir) FLAGS.sample_dir = expand_path(FLAGS.sample_dir) if FLAGS.output_height is None: FLAGS.output_height = FLAGS.input_height if FLAGS.input_width is None: FLAGS.input_width = FLAGS.input_height if FLAGS.output_width is None: FLAGS.output_width = FLAGS.output_height # output folders if FLAGS.out_name == "": # FLAGS.out_name = '{} - {} - {}'.format(timestamp(), FLAGS.data_dir.split('/')[-1], FLAGS.dataset) # penultimate folder of path FLAGS.out_name = '{} - {}'.format( FLAGS.data_dir.split('/')[-1], FLAGS.dataset) if FLAGS.train: FLAGS.out_name += ' - x{}.z{}.{}.y{}.b{}'.format( FLAGS.input_width, FLAGS.z_dim, FLAGS.z_dist, FLAGS.output_width, FLAGS.batch_size) FLAGS.out_dir = os.path.join(FLAGS.out_dir, FLAGS.out_name) FLAGS.checkpoint_dir = os.path.join(FLAGS.out_dir, FLAGS.checkpoint_dir) FLAGS.sample_dir = os.path.join(FLAGS.out_dir, FLAGS.sample_dir) if not os.path.exists(FLAGS.checkpoint_dir): os.makedirs(FLAGS.checkpoint_dir) if not os.path.exists(FLAGS.sample_dir): os.makedirs(FLAGS.sample_dir) with open(os.path.join(FLAGS.out_dir, 'FLAGS.json'), 'w') as f: flags_dict = {k: flags.FLAGS.__flags[k] for k in flags.FLAGS.__flags} json.dump(flags_dict, f, indent=4, sort_keys=True, ensure_ascii=False) #gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333) run_config = tf.ConfigProto() run_config.gpu_options.allow_growth = True with tf.Session(config=run_config) as sess: if FLAGS.dataset == 'mnist': dcgan = DCGAN(sess, input_width=FLAGS.input_width, input_height=FLAGS.input_height, output_width=FLAGS.output_width, output_height=FLAGS.output_height, batch_size=FLAGS.batch_size, sample_num=FLAGS.batch_size, y_dim=10, z_dim=FLAGS.z_dim, dataset_name=FLAGS.dataset, input_fname_pattern=FLAGS.input_fname_pattern, crop=FLAGS.crop, checkpoint_dir=FLAGS.checkpoint_dir, sample_dir=FLAGS.sample_dir, data_dir=FLAGS.data_dir, out_dir=FLAGS.out_dir, max_to_keep=FLAGS.max_to_keep) else: dcgan = DCGAN(sess, input_width=FLAGS.input_width, input_height=FLAGS.input_height, output_width=FLAGS.output_width, output_height=FLAGS.output_height, batch_size=FLAGS.batch_size, sample_num=FLAGS.batch_size, z_dim=FLAGS.z_dim, dataset_name=FLAGS.dataset, input_fname_pattern=FLAGS.input_fname_pattern, crop=FLAGS.crop, checkpoint_dir=FLAGS.checkpoint_dir, sample_dir=FLAGS.sample_dir, data_dir=FLAGS.data_dir, out_dir=FLAGS.out_dir, max_to_keep=FLAGS.max_to_keep) show_all_variables() if FLAGS.train: dcgan.train(FLAGS) else: load_success, load_counter = dcgan.load(FLAGS.checkpoint_dir) if not load_success: raise Exception("Checkpoint not found in " + FLAGS.checkpoint_dir) # to_json("./web/js/layers.js", [dcgan.h0_w, dcgan.h0_b, dcgan.g_bn0], # [dcgan.h1_w, dcgan.h1_b, dcgan.g_bn1], # [dcgan.h2_w, dcgan.h2_b, dcgan.g_bn2], # [dcgan.h3_w, dcgan.h3_b, dcgan.g_bn3], # [dcgan.h4_w, dcgan.h4_b, None]) # Below is codes for visualization if FLAGS.export: export_dir = os.path.join(FLAGS.checkpoint_dir, 'export_b' + str(FLAGS.batch_size)) dcgan.save(export_dir, load_counter, ckpt=True, frozen=False) if FLAGS.freeze: export_dir = os.path.join(FLAGS.checkpoint_dir, 'frozen_b' + str(FLAGS.batch_size)) dcgan.save(export_dir, load_counter, ckpt=False, frozen=True) if True: OPTION = 1 visualize(sess, dcgan, FLAGS, OPTION, FLAGS.sample_dir)
def txt_file_to_str(path): """ Given a path to a textfile, read its contents """ return open(expand_path(path)).read()
def main(_): pp.pprint(flags.FLAGS.__flags) FLAGS.train = True alpha_max_str = str(FLAGS.alpha_max) # if FLAGS.steer: # print('Training with steerable G -> loading model_argminGW2_{} ...'.format(FLAGS.transform_type)) # DCGAN = getattr(importlib.import_module('model_argminGW2_{}'.format(FLAGS.transform_type)), 'DCGAN') # else: # print('Training vanilla G -> loading model_vanilla_{} ...'.format(FLAGS.transform_type)) # DCGAN = getattr(importlib.import_module('model_vanilla_{}'.format(FLAGS.transform_type)), 'DCGAN') print('Training with steerable G for {} transformation ...'.format(FLAGS.transform_type)) if FLAGS.transform_type == 'zoom': if FLAGS.steer: from model_argminGW2_zoom import DCGAN else: from model_vanilla_zoom import DCGAN if FLAGS.transform_type == 'shiftx': alpha_max_str = str(np.uint8(FLAGS.alpha_max)) if FLAGS.steer: from model_argminGW2_shiftx import DCGAN else: from model_vanilla_shiftx import DCGAN if FLAGS.transform_type == 'shifty': alpha_max_str = str(np.uint8(FLAGS.alpha_max)) if FLAGS.steer: from model_argminGW2_shifty import DCGAN else: from model_vanilla_shifty import DCGAN if FLAGS.transform_type == 'rot2d': alpha_max_str = str(np.uint8(FLAGS.alpha_max)) if FLAGS.steer: from model_argminGW2_rot2d import DCGAN else: from model_vanilla_rot2d import DCGAN augment_flag_str = 'NoAug' if FLAGS.aug: augment_flag_str = 'aug' steer_flag_str = 'vanilla' if FLAGS.steer: steer_flag_str = 'argminGW' else: if FLAGS.aug: steer_flag_str = 'argminW' if FLAGS.out_name: FLAGS.out_name = expand_path(FLAGS.out_name) else: FLAGS.out_name = FLAGS.transform_type+'_'+augment_flag_str+'_'+steer_flag_str+\ '_alphamax'+alpha_max_str+'_lr'+ str(FLAGS.learning_rate) print('Results will be saved in {}'.format(FLAGS.out_name)) # expand user name and environment variables FLAGS.data_dir = expand_path(FLAGS.data_dir) FLAGS.out_dir = expand_path(FLAGS.out_dir) # FLAGS.out_name = expand_path(FLAGS.out_name) FLAGS.checkpoint_dir = expand_path(FLAGS.checkpoint_dir) FLAGS.sample_dir = expand_path(FLAGS.sample_dir) if FLAGS.output_height is None: FLAGS.output_height = FLAGS.input_height if FLAGS.input_width is None: FLAGS.input_width = FLAGS.input_height if FLAGS.output_width is None: FLAGS.output_width = FLAGS.output_height # output folders if FLAGS.out_name == "": FLAGS.out_name = '{} - {} - {}'.format(timestamp(), FLAGS.data_dir.split('/')[-1], FLAGS.dataset) # penultimate folder of path if FLAGS.train: FLAGS.out_name += ' - x{}.z{}.{}.y{}.b{}'.format(FLAGS.input_width, FLAGS.z_dim, FLAGS.z_dist, FLAGS.output_width, FLAGS.batch_size) FLAGS.out_dir = os.path.join(FLAGS.out_dir, FLAGS.out_name) FLAGS.checkpoint_dir = os.path.join(FLAGS.out_dir, FLAGS.checkpoint_dir) FLAGS.sample_dir = os.path.join(FLAGS.out_dir, FLAGS.sample_dir) if not os.path.exists(FLAGS.checkpoint_dir): os.makedirs(FLAGS.checkpoint_dir) if not os.path.exists(FLAGS.sample_dir): os.makedirs(FLAGS.sample_dir) with open(os.path.join(FLAGS.out_dir, 'FLAGS.json'), 'w') as f: flags_dict = {k:FLAGS[k].value for k in FLAGS} json.dump(flags_dict, f, indent=4, sort_keys=True, ensure_ascii=False) #gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333) run_config = tf.ConfigProto() run_config.gpu_options.allow_growth=True with tf.Session(config=run_config) as sess: if FLAGS.dataset == 'mnist': dcgan = DCGAN( sess, input_width=FLAGS.input_width, input_height=FLAGS.input_height, output_width=FLAGS.output_width, output_height=FLAGS.output_height, batch_size=FLAGS.batch_size, sample_num=FLAGS.batch_size, y_dim=10, z_dim=FLAGS.z_dim, dataset_name=FLAGS.dataset, aug=FLAGS.aug, alpha_max=FLAGS.alpha_max, input_fname_pattern=FLAGS.input_fname_pattern, crop=FLAGS.crop, checkpoint_dir=FLAGS.checkpoint_dir, sample_dir=FLAGS.sample_dir, data_dir=FLAGS.data_dir, out_dir=FLAGS.out_dir, max_to_keep=FLAGS.max_to_keep) else: dcgan = DCGAN( sess, input_width=FLAGS.input_width, input_height=FLAGS.input_height, output_width=FLAGS.output_width, output_height=FLAGS.output_height, batch_size=FLAGS.batch_size, sample_num=FLAGS.batch_size, z_dim=FLAGS.z_dim, dataset_name=FLAGS.dataset, aug=FLAGS.aug, input_fname_pattern=FLAGS.input_fname_pattern, crop=FLAGS.crop, checkpoint_dir=FLAGS.checkpoint_dir, sample_dir=FLAGS.sample_dir, data_dir=FLAGS.data_dir, out_dir=FLAGS.out_dir, max_to_keep=FLAGS.max_to_keep) show_all_variables() if FLAGS.train: print('>>>---Traning mode is set to {}---<<<'.format(FLAGS.train)) time.sleep(10) dcgan.train(FLAGS) else: print('<<<---Testing mode--->>>') time.sleep(10) load_success, load_counter = dcgan.load(FLAGS.checkpoint_dir) if not load_success: raise Exception("Checkpoint not found in " + FLAGS.checkpoint_dir) # to_json("./web/js/layers.js", [dcgan.h0_w, dcgan.h0_b, dcgan.g_bn0], # [dcgan.h1_w, dcgan.h1_b, dcgan.g_bn1], # [dcgan.h2_w, dcgan.h2_b, dcgan.g_bn2], # [dcgan.h3_w, dcgan.h3_b, dcgan.g_bn3], # [dcgan.h4_w, dcgan.h4_b, None]) # Below is codes for visualization if FLAGS.export: export_dir = os.path.join(FLAGS.checkpoint_dir, 'export_b'+str(FLAGS.batch_size)) dcgan.save(export_dir, load_counter, ckpt=True, frozen=False) if FLAGS.freeze: export_dir = os.path.join(FLAGS.checkpoint_dir, 'frozen_b'+str(FLAGS.batch_size)) dcgan.save(export_dir, load_counter, ckpt=False, frozen=True) if FLAGS.visualize: OPTION = 1 visualize(sess, dcgan, FLAGS, OPTION, FLAGS.sample_dir)
def get_files_from_dir(path): """ Given a path to a directory, list all of its files """ return os.listdir(expand_path(path))
def main(_): pp.pprint(flags.FLAGS.__flags) FLAGS.data_dir = expand_path(FLAGS.data_dir) FLAGS.out_dir = expand_path(FLAGS.out_dir) FLAGS.out_name = expand_path(FLAGS.out_name) FLAGS.checkpoint_dir = expand_path(FLAGS.checkpoint_dir) FLAGS.sample_dir = expand_path(FLAGS.sample_dir) if FLAGS.output_height is None: FLAGS.output_height = FLAGS.input_height if FLAGS.input_width is None: FLAGS.input_width = FLAGS.input_height if FLAGS.output_width is None: FLAGS.output_width = FLAGS.output_height if FLAGS.out_name == "": FLAGS.out_name = '{} - {} - {}'.format(timestamp(), FLAGS.data_dir.split('/')[-1], FLAGS.dataset) if FLAGS.train: FLAGS.out_name += ' - x{}.z{}.{}.y{}.b{}'.format( FLAGS.input_width, FLAGS.z_dim, FLAGS.z_dist, FLAGS.output_width, FLAGS.batch_size) #FLAGS.out_dir = os.path.join(FLAGS.out_dir, FLAGS.out_name) #FLAGS.checkpoint_dir = os.path.join(FLAGS.out_dir, FLAGS.checkpoint_dir) #FLAGS.sample_dir = os.path.join(FLAGS.out_dir, FLAGS.sample_dir) if not os.path.exists(FLAGS.checkpoint_dir): os.makedirs(FLAGS.checkpoint_dir) if not os.path.exists(FLAGS.sample_dir): os.makedirs(FLAGS.sample_dir) with open(os.path.join(FLAGS.out_dir, 'FLAGS.json'), 'w') as f: flags_dict = {k: FLAGS[k].value for k in FLAGS} json.dump(flags_dict, f, indent=4, sort_keys=True, ensure_ascii=False) run_config = tf.ConfigProto() run_config.gpu_options.allow_growth = True with tf.Session(config=run_config) as sess: dcgan = DCGAN(sess, input_width=FLAGS.input_width, input_height=FLAGS.input_height, output_width=FLAGS.output_width, output_height=FLAGS.output_height, batch_size=FLAGS.batch_size, sample_num=FLAGS.batch_size, z_dim=FLAGS.z_dim, dataset_name=FLAGS.dataset, input_fname_pattern=FLAGS.input_fname_pattern, crop=FLAGS.crop, checkpoint_dir=FLAGS.checkpoint_dir, sample_dir=FLAGS.sample_dir, data_dir=FLAGS.data_dir, out_dir=FLAGS.out_dir, max_to_keep=FLAGS.max_to_keep) show_all_variables() if FLAGS.train: dcgan.train(FLAGS) else: load_success, load_counter = dcgan.load(FLAGS.checkpoint_dir) if not load_success: raise Exception("checkpoint not found in " + FLAGS.checkpoint_dir) if FLAGS.export: export_dir = os.path.join(FLAGS.checkpoint_dir, 'export_b' + str(FLAGS.batch_size)) dcgan.save(export_dir, load_counter, ckpt=True, frozen=False) if FLAGS.freeze: export_dir = os.path.join(FLAGS.checkpoint_dir, 'frozen_b' + str(FLAGS.batch_size)) dcgan.save(export_dir, load_counter, ckpt=False, frozen=True) if FLAGS.visualize: OPTION = 1 visualize(sess, dcgan, FLAGS, OPTION, FLAGS.sample_dir) sess.close()
def read_raw_image(p): img = Image.open(expand_path(p)) return img