def predict_batch(): patch_shape = (33,33) nkerns = [64, 32] param_snapshot_path = 'try1/param_50.npy' plot_save_dir = 'output_not_scale' # prepare the data print ".preparing data" datasets = np.load(utils.complement_path('/share/blur_images/all_in_one/testing_full_images.npy')) predict_result = [] for one in datasets: predict_result.extend(predict_single( param_snapshot_path = param_snapshot_path, # one = utils.shared_dataset(one), one = one.astype(theano.config.floatX), # shape = one.shape, patch_shape = patch_shape, nkerns = nkerns )) # assert len(predict_result) == len(datasets) # save plot image plot_save_dir = utils.complement_path(plot_save_dir) if not os.path.exists(plot_save_dir): os.mkdir(plot_save_dir) print '.saving plot image' for i in xrange(len(datasets)): origin_img = datasets[i] # origin_img = origin_img + origin_img.min() # origin_img = origin_img / origin_img.max() * 255. reconstruct_img = predict_result[i][0] # reconstruct_img = reconstruct_img + reconstruct_img.min() # reconstruct_img = reconstruct_img / reconstruct_img.max() * 255. # assert origin_img.shape == reconstruct_img.shape # join = np.vstack([origin_img, reconstruct_img]) img1 = Image.fromarray(origin_img.astype(np.uint8)) img2 = Image.fromarray(reconstruct_img.astype(np.uint8)) img_path = os.path.join(plot_save_dir, str(i) + '_o.jpg') print 'saving to %s' % img_path img1.save(img_path) img_path = os.path.join(plot_save_dir, str(i) + '_r.jpg') print 'saving to %s' % img_path img2.save(img_path)
def save_param(layers, param_path): import utils import numpy as np param_path = utils.complement_path(param_path) params = [] for layer in layers: params.extend(layer.params) print "..saving to %s (params length: %i)" % (param_path, len(params)) np.save(param_path, params)
def synthesize_predicting_images( data_paths, output_dir = "/share/blur_images/all_in_one2/" ): images = [] for data_path in data_paths: for root, dirs, files in os.walk(utils.complement_path(data_path)): for file in files: path = os.path.join(root, file) print path img = utils.PIL2array(path) images.append(img) print "total images size: %i" % len(images) output_dir = utils.complement_path(output_dir) if not os.path.exists(output_dir): os.mkdir(output_dir) np.save(os.path.join(output_dir,'testing_full_images.npy'), images)
def init(repo_dir=None, log_file=None): INIFILE = os.path.join(repo_dir, INIFILE_NAME) log.register_options(CONF) if os.path.exists(INIFILE): CONF([], default_config_files=[INIFILE]) else: CONF([]) CONF._inifile = INIFILE CONF._repo_dir = repo_dir CONF._fabfile_dir = os.path.join(repo_dir, 'fabfile') CONF._conf_dir = os.path.join(repo_dir, 'conf') CONF._sqlalchemy_dir = os.path.join(CONF._fabfile_dir, 'core', 'db', 'impl_sqlalchemy') CONF._pdns_sqlalchemy_dir = os.path.join(CONF._fabfile_dir, 'core', 'pdns', 'impl_sqlalchemy') CONF._webapp_dir = os.path.join(CONF._fabfile_dir, 'core', 'webapp') CONF._storage_dir = complement_path(CONF.storage_dir) CONF._webapp_storage_dir = os.path.join(CONF._storage_dir, 'webapp') CONF._handler_dir = complement_path(CONF.handler_dir) CONF._databag_dir = complement_path(CONF.databag_dir) CONF._filebag_dir = complement_path(CONF.filebag_dir) CONF._tmp_dir = os.path.join(CONF._storage_dir, 'tmp') CONF._log_dir = os.path.join(CONF._storage_dir, 'log') CONF._node_dir = complement_path(CONF.node_dir) CONF._tmp_node_dir = complement_path(CONF.node_dir + '.tmp') CONF._job_yml = os.path.join(CONF._conf_dir, 'job.yml') CONF._fabscript_module_dir = complement_path(CONF.fabscript_module) CONF._fablib_module_dir = complement_path(CONF.fablib_module) CONF._node_meta_pickle = os.path.join(CONF._node_dir, 'meta.pickle') CONF._all_log_file_name = ALL_LOG_FILE_NAME CONF._error_log_file_name = ERROR_LOG_FILE_NAME CONF._fabscript_yaml = FABSCRIPT_YAML CONF._yaml_extension = YAML_EXTENSION CONF._cluster_yaml = CLUSTER_YAML CONF._cluster_pickle = CLUSTER_PICKLE CONF._datamap_dir = DATAMAP_DIR CONF._logger_formatter = logging.Formatter(fmt=CONF.logger.format) CONF._logger_console_formatter = logging.Formatter( fmt=CONF.logger.console_format) CONF._check_agent_interval = CONF.agent.agent_downtime // 2 if log_file is None: log.setup(CONF, 'fabkit') else: CONF.log_file = os.path.join(CONF.log_dir, log_file) log.setup(CONF, 'fabkit')
def init(repo_dir=None, log_file=None): INIFILE = os.path.join(repo_dir, INIFILE_NAME) log.register_options(CONF) if os.path.exists(INIFILE): CONF([], default_config_files=[INIFILE]) else: CONF([]) CONF._inifile = INIFILE CONF._repo_dir = repo_dir CONF._fabfile_dir = os.path.join(repo_dir, 'fabfile') CONF._conf_dir = os.path.join(repo_dir, 'conf') CONF._sqlalchemy_dir = os.path.join(CONF._fabfile_dir, 'core', 'db', 'impl_sqlalchemy') CONF._pdns_sqlalchemy_dir = os.path.join(CONF._fabfile_dir, 'core', 'pdns', 'impl_sqlalchemy') CONF._webapp_dir = os.path.join(CONF._fabfile_dir, 'core', 'webapp') CONF._storage_dir = complement_path(CONF.storage_dir) CONF._webapp_storage_dir = os.path.join(CONF._storage_dir, 'webapp') CONF._handler_dir = complement_path(CONF.handler_dir) CONF._databag_dir = complement_path(CONF.databag_dir) CONF._filebag_dir = complement_path(CONF.filebag_dir) CONF._tmp_dir = os.path.join(CONF._storage_dir, 'tmp') CONF._log_dir = os.path.join(CONF._storage_dir, 'log') CONF._node_dir = complement_path(CONF.node_dir) CONF._tmp_node_dir = complement_path(CONF.node_dir + '.tmp') CONF._job_yml = os.path.join(CONF._conf_dir, 'job.yml') CONF._fabscript_module_dir = complement_path(CONF.fabscript_module) CONF._fablib_module_dir = complement_path(CONF.fablib_module) CONF._node_meta_pickle = os.path.join(CONF._node_dir, 'meta.pickle') CONF._all_log_file_name = ALL_LOG_FILE_NAME CONF._error_log_file_name = ERROR_LOG_FILE_NAME CONF._fabscript_yaml = FABSCRIPT_YAML CONF._yaml_extension = YAML_EXTENSION CONF._cluster_yaml = CLUSTER_YAML CONF._cluster_pickle = CLUSTER_PICKLE CONF._datamap_dir = DATAMAP_DIR CONF._logger_formatter = logging.Formatter(fmt=CONF.logger.format) CONF._logger_console_formatter = logging.Formatter(fmt=CONF.logger.console_format) CONF._check_agent_interval = CONF.agent.agent_downtime // 2 if log_file is None: log.setup(CONF, 'fabkit') else: CONF.log_file = os.path.join(CONF.log_dir, log_file) log.setup(CONF, 'fabkit')
def init(): if CONF.user is not None: env.user = CONF.user if CONF.password is not None: env.password = CONF.password elif CONF.password_file is not None: password_file = complement_path(CONF.password_file) if os.path.exists(password_file): with open(password_file, 'r') as f: env.password = f.read() CONF._remote_dir = os.path.join('/home', env.user, CONF.remote_dir) CONF._remote_storage_dir = os.path.join(CONF._remote_dir, 'storage') CONF._remote_tmp_dir = os.path.join(CONF._remote_storage_dir, 'tmp')
def init(): if CONF.user is not None: env.user = CONF.user if CONF.password is not None: env.password = CONF.password elif CONF.password_file is not None: password_file = complement_path(CONF.password_file) if os.path.exists(password_file): with open(password_file, 'r') as f: env.password = f.read() CONF._remote_dir = os.path.join('/home', env.user, CONF.remote_dir) CONF._remote_storage_dir = os.path.join(CONF._remote_dir, 'storage') CONF._remote_tmp_dir = os.path.join(CONF._remote_storage_dir, 'tmp') CONF._remote_stats_dir = os.path.join(CONF._remote_storage_dir, 'stats')
def load_param(layers, param_path): import utils import numpy as np param_path = utils.complement_path(param_path) params = np.load(param_path) print "..loading from %s (params length: %i)" % (param_path, len(params)) assert len(params) == 2 * len(layers) for i in range(len(layers)): layers[i].params[0].data.set_value(params[2*i].data.get_value()) layers[i].params[0].name = params[2*i].name layers[i].params[0].eps = params[2*i].eps layers[i].params[0].wc = params[2*i].wc layers[i].params[1].data.set_value(params[2*i+1].data.get_value()) layers[i].params[1].name = params[2*i+1].name layers[i].params[1].eps = params[2*i+1].eps layers[i].params[1].wc = params[2*i+1].wc
def synthesize_training_image( dir_paths, output_dir = '/share/blur_images/all_in_one2/', patch_shape = (48, 48), stride_shape = (48, 48), scale_ratio = 6, is_shuffle = True, is_scale = False, is_noise = False, ): # read and split images, type is list print "patch_shape: {}".format(patch_shape) print "stride_shape: {}".format(stride_shape) print "scale ratio: {}".format(scale_ratio) images = [] for dir_path in dir_paths: dir_path = utils.complement_path(dir_path) images.extend( utils.sub_images( dir_path = dir_path, patch_shape = patch_shape, stride_shape = stride_shape, is_scale = is_scale, is_noise = is_noise)[0]) images = np.asarray(images, dtype=theano.config.floatX) print images.shape mean = np.mean(images, axis = 0) # operation on images, type is list of ndarray original_images = [] corrupted_images = [] for each_image in images: original_images.append(each_image - mean) ## gaussian noise noise_each_image = each_image - mean + np.random.normal(size=each_image.shape) im = Image.fromarray(noise_each_image.reshape(patch_shape)) ## downsample im_downsampling = im.resize((patch_shape[0] // scale_ratio, patch_shape[1] // scale_ratio), Image.BICUBIC) ## upsampling im_upsampling = im_downsampling.resize(patch_shape, Image.BICUBIC) assert im_upsampling.size == im.size new_image = np.array(im_upsampling.getdata()) corrupted_images.append(new_image) original_images = np.asarray(original_images, dtype=theano.config.floatX) corrupted_images = np.asarray(corrupted_images, dtype=theano.config.floatX) assert images.shape == corrupted_images.shape # shuffle if is_shuffle: perms = np.random.permutation(len(images)) original_images = original_images[perms] corrupted_images = corrupted_images[perms] # save images and corrupted_images if not os.path.exists(output_dir): os.mkdir(output_dir) train_nums = len(images) // 7 * 5 valid_nums = len(images) // 7 train_o = original_images[:train_nums] train_b = corrupted_images[:train_nums] valid_o = original_images[train_nums:train_nums+valid_nums] valid_b = corrupted_images[train_nums:train_nums+valid_nums] test_o = original_images[train_nums+valid_nums:] test_b = corrupted_images[train_nums+valid_nums:] np.save(os.path.join(output_dir, 'train_o_set.npy'), train_o) np.save(os.path.join(output_dir, 'train_b_set.npy'), train_b) np.save(os.path.join(output_dir, 'valid_o_set.npy'), valid_o) np.save(os.path.join(output_dir, 'valid_b_set.npy'), valid_b) np.save(os.path.join(output_dir, 'test_o_set.npy'), test_o) np.save(os.path.join(output_dir, 'test_b_set.npy'), test_b)
import context from layers import ConvLayer import utils import numpy as np patch_shape = (33,33) nkerns = [64, 32] batch_size = 5000 learning_rate = 3e-7 weight_decay = 2 # prepare the data print ".preparing data" dataset_paths = [ utils.complement_path('/share/blur_images/all_in_one/train_o_set.npy'), utils.complement_path('/share/blur_images/all_in_one/train_b_set.npy'), utils.complement_path('/share/blur_images/all_in_one/valid_o_set.npy'), utils.complement_path('/share/blur_images/all_in_one/valid_b_set.npy'), utils.complement_path('/share/blur_images/all_in_one/test_o_set.npy'), utils.complement_path('/share/blur_images/all_in_one/test_b_set.npy')] datasets = [ utils.shared_dataset(np.load(dataset_path)) for dataset_path in dataset_paths] # build the network print ".building network" normal = T.fmatrix('normal') corrupt = T.fmatrix('corrupt')