def predict_batch():
patch_shape = (33,33)
nkerns = [64, 32]
param_snapshot_path = 'try1/param_50.npy'
plot_save_dir = 'output_not_scale'
# prepare the data
print ".preparing data"
datasets = np.load(utils.complement_path('/share/blur_images/all_in_one/testing_full_images.npy'))
predict_result = []
for one in datasets:
predict_result.extend(predict_single(
param_snapshot_path = param_snapshot_path,
# one = utils.shared_dataset(one),
one = one.astype(theano.config.floatX),
# shape = one.shape,
patch_shape = patch_shape,
nkerns = nkerns
))
# assert len(predict_result) == len(datasets)
# save plot image
plot_save_dir = utils.complement_path(plot_save_dir)
if not os.path.exists(plot_save_dir):
os.mkdir(plot_save_dir)
print '.saving plot image'
for i in xrange(len(datasets)):
origin_img = datasets[i]
# origin_img = origin_img + origin_img.min()
# origin_img = origin_img / origin_img.max() * 255.
reconstruct_img = predict_result[i][0]
# reconstruct_img = reconstruct_img + reconstruct_img.min()
# reconstruct_img = reconstruct_img / reconstruct_img.max() * 255.
# assert origin_img.shape == reconstruct_img.shape
# join = np.vstack([origin_img, reconstruct_img])
img1 = Image.fromarray(origin_img.astype(np.uint8))
img2 = Image.fromarray(reconstruct_img.astype(np.uint8))
img_path = os.path.join(plot_save_dir, str(i) + '_o.jpg')
print 'saving to %s' % img_path
img1.save(img_path)
img_path = os.path.join(plot_save_dir, str(i) + '_r.jpg')
print 'saving to %s' % img_path
img2.save(img_path)
def save_param(layers, param_path):
import utils
import numpy as np
param_path = utils.complement_path(param_path)
params = []
for layer in layers:
params.extend(layer.params)
print "..saving to %s (params length: %i)" % (param_path, len(params))
np.save(param_path, params)
def synthesize_predicting_images(
data_paths,
output_dir = "/share/blur_images/all_in_one2/"
):
images = []
for data_path in data_paths:
for root, dirs, files in os.walk(utils.complement_path(data_path)):
for file in files:
path = os.path.join(root, file)
print path
img = utils.PIL2array(path)
images.append(img)
print "total images size: %i" % len(images)
output_dir = utils.complement_path(output_dir)
if not os.path.exists(output_dir):
os.mkdir(output_dir)
np.save(os.path.join(output_dir,'testing_full_images.npy'), images)
def init(repo_dir=None, log_file=None):
INIFILE = os.path.join(repo_dir, INIFILE_NAME)
log.register_options(CONF)
if os.path.exists(INIFILE):
CONF([], default_config_files=[INIFILE])
else:
CONF([])
CONF._inifile = INIFILE
CONF._repo_dir = repo_dir
CONF._fabfile_dir = os.path.join(repo_dir, 'fabfile')
CONF._conf_dir = os.path.join(repo_dir, 'conf')
CONF._sqlalchemy_dir = os.path.join(CONF._fabfile_dir, 'core', 'db',
'impl_sqlalchemy')
CONF._pdns_sqlalchemy_dir = os.path.join(CONF._fabfile_dir, 'core', 'pdns',
'impl_sqlalchemy')
CONF._webapp_dir = os.path.join(CONF._fabfile_dir, 'core', 'webapp')
CONF._storage_dir = complement_path(CONF.storage_dir)
CONF._webapp_storage_dir = os.path.join(CONF._storage_dir, 'webapp')
CONF._handler_dir = complement_path(CONF.handler_dir)
CONF._databag_dir = complement_path(CONF.databag_dir)
CONF._filebag_dir = complement_path(CONF.filebag_dir)
CONF._tmp_dir = os.path.join(CONF._storage_dir, 'tmp')
CONF._log_dir = os.path.join(CONF._storage_dir, 'log')
CONF._node_dir = complement_path(CONF.node_dir)
CONF._tmp_node_dir = complement_path(CONF.node_dir + '.tmp')
CONF._job_yml = os.path.join(CONF._conf_dir, 'job.yml')
CONF._fabscript_module_dir = complement_path(CONF.fabscript_module)
CONF._fablib_module_dir = complement_path(CONF.fablib_module)
CONF._node_meta_pickle = os.path.join(CONF._node_dir, 'meta.pickle')
CONF._all_log_file_name = ALL_LOG_FILE_NAME
CONF._error_log_file_name = ERROR_LOG_FILE_NAME
CONF._fabscript_yaml = FABSCRIPT_YAML
CONF._yaml_extension = YAML_EXTENSION
CONF._cluster_yaml = CLUSTER_YAML
CONF._cluster_pickle = CLUSTER_PICKLE
CONF._datamap_dir = DATAMAP_DIR
CONF._logger_formatter = logging.Formatter(fmt=CONF.logger.format)
CONF._logger_console_formatter = logging.Formatter(
fmt=CONF.logger.console_format)
CONF._check_agent_interval = CONF.agent.agent_downtime // 2
if log_file is None:
log.setup(CONF, 'fabkit')
else:
CONF.log_file = os.path.join(CONF.log_dir, log_file)
log.setup(CONF, 'fabkit')
def init(repo_dir=None, log_file=None):
INIFILE = os.path.join(repo_dir, INIFILE_NAME)
log.register_options(CONF)
if os.path.exists(INIFILE):
CONF([], default_config_files=[INIFILE])
else:
CONF([])
CONF._inifile = INIFILE
CONF._repo_dir = repo_dir
CONF._fabfile_dir = os.path.join(repo_dir, 'fabfile')
CONF._conf_dir = os.path.join(repo_dir, 'conf')
CONF._sqlalchemy_dir = os.path.join(CONF._fabfile_dir, 'core', 'db', 'impl_sqlalchemy')
CONF._pdns_sqlalchemy_dir = os.path.join(CONF._fabfile_dir, 'core', 'pdns', 'impl_sqlalchemy')
CONF._webapp_dir = os.path.join(CONF._fabfile_dir, 'core', 'webapp')
CONF._storage_dir = complement_path(CONF.storage_dir)
CONF._webapp_storage_dir = os.path.join(CONF._storage_dir, 'webapp')
CONF._handler_dir = complement_path(CONF.handler_dir)
CONF._databag_dir = complement_path(CONF.databag_dir)
CONF._filebag_dir = complement_path(CONF.filebag_dir)
CONF._tmp_dir = os.path.join(CONF._storage_dir, 'tmp')
CONF._log_dir = os.path.join(CONF._storage_dir, 'log')
CONF._node_dir = complement_path(CONF.node_dir)
CONF._tmp_node_dir = complement_path(CONF.node_dir + '.tmp')
CONF._job_yml = os.path.join(CONF._conf_dir, 'job.yml')
CONF._fabscript_module_dir = complement_path(CONF.fabscript_module)
CONF._fablib_module_dir = complement_path(CONF.fablib_module)
CONF._node_meta_pickle = os.path.join(CONF._node_dir, 'meta.pickle')
CONF._all_log_file_name = ALL_LOG_FILE_NAME
CONF._error_log_file_name = ERROR_LOG_FILE_NAME
CONF._fabscript_yaml = FABSCRIPT_YAML
CONF._yaml_extension = YAML_EXTENSION
CONF._cluster_yaml = CLUSTER_YAML
CONF._cluster_pickle = CLUSTER_PICKLE
CONF._datamap_dir = DATAMAP_DIR
CONF._logger_formatter = logging.Formatter(fmt=CONF.logger.format)
CONF._logger_console_formatter = logging.Formatter(fmt=CONF.logger.console_format)
CONF._check_agent_interval = CONF.agent.agent_downtime // 2
if log_file is None:
log.setup(CONF, 'fabkit')
else:
CONF.log_file = os.path.join(CONF.log_dir, log_file)
log.setup(CONF, 'fabkit')
def init():
if CONF.user is not None:
env.user = CONF.user
if CONF.password is not None:
env.password = CONF.password
elif CONF.password_file is not None:
password_file = complement_path(CONF.password_file)
if os.path.exists(password_file):
with open(password_file, 'r') as f:
env.password = f.read()
CONF._remote_dir = os.path.join('/home', env.user, CONF.remote_dir)
CONF._remote_storage_dir = os.path.join(CONF._remote_dir, 'storage')
CONF._remote_tmp_dir = os.path.join(CONF._remote_storage_dir, 'tmp')
def init():
if CONF.user is not None:
env.user = CONF.user
if CONF.password is not None:
env.password = CONF.password
elif CONF.password_file is not None:
password_file = complement_path(CONF.password_file)
if os.path.exists(password_file):
with open(password_file, 'r') as f:
env.password = f.read()
CONF._remote_dir = os.path.join('/home', env.user, CONF.remote_dir)
CONF._remote_storage_dir = os.path.join(CONF._remote_dir, 'storage')
CONF._remote_tmp_dir = os.path.join(CONF._remote_storage_dir, 'tmp')
CONF._remote_stats_dir = os.path.join(CONF._remote_storage_dir, 'stats')
def load_param(layers, param_path):
import utils
import numpy as np
param_path = utils.complement_path(param_path)
params = np.load(param_path)
print "..loading from %s (params length: %i)" % (param_path, len(params))
assert len(params) == 2 * len(layers)
for i in range(len(layers)):
layers[i].params[0].data.set_value(params[2*i].data.get_value())
layers[i].params[0].name = params[2*i].name
layers[i].params[0].eps = params[2*i].eps
layers[i].params[0].wc = params[2*i].wc
layers[i].params[1].data.set_value(params[2*i+1].data.get_value())
layers[i].params[1].name = params[2*i+1].name
layers[i].params[1].eps = params[2*i+1].eps
layers[i].params[1].wc = params[2*i+1].wc
def synthesize_training_image(
dir_paths,
output_dir = '/share/blur_images/all_in_one2/',
patch_shape = (48, 48),
stride_shape = (48, 48),
scale_ratio = 6,
is_shuffle = True,
is_scale = False,
is_noise = False,
):
# read and split images, type is list
print "patch_shape: {}".format(patch_shape)
print "stride_shape: {}".format(stride_shape)
print "scale ratio: {}".format(scale_ratio)
images = []
for dir_path in dir_paths:
dir_path = utils.complement_path(dir_path)
images.extend(
utils.sub_images(
dir_path = dir_path,
patch_shape = patch_shape,
stride_shape = stride_shape,
is_scale = is_scale,
is_noise = is_noise)[0])
images = np.asarray(images, dtype=theano.config.floatX)
print images.shape
mean = np.mean(images, axis = 0)
# operation on images, type is list of ndarray
original_images = []
corrupted_images = []
for each_image in images:
original_images.append(each_image - mean)
## gaussian noise
noise_each_image = each_image - mean + np.random.normal(size=each_image.shape)
im = Image.fromarray(noise_each_image.reshape(patch_shape))
## downsample
im_downsampling = im.resize((patch_shape[0] // scale_ratio, patch_shape[1] // scale_ratio), Image.BICUBIC)
## upsampling
im_upsampling = im_downsampling.resize(patch_shape, Image.BICUBIC)
assert im_upsampling.size == im.size
new_image = np.array(im_upsampling.getdata())
corrupted_images.append(new_image)
original_images = np.asarray(original_images, dtype=theano.config.floatX)
corrupted_images = np.asarray(corrupted_images, dtype=theano.config.floatX)
assert images.shape == corrupted_images.shape
# shuffle
if is_shuffle:
perms = np.random.permutation(len(images))
original_images = original_images[perms]
corrupted_images = corrupted_images[perms]
# save images and corrupted_images
if not os.path.exists(output_dir):
os.mkdir(output_dir)
train_nums = len(images) // 7 * 5
valid_nums = len(images) // 7
train_o = original_images[:train_nums]
train_b = corrupted_images[:train_nums]
valid_o = original_images[train_nums:train_nums+valid_nums]
valid_b = corrupted_images[train_nums:train_nums+valid_nums]
test_o = original_images[train_nums+valid_nums:]
test_b = corrupted_images[train_nums+valid_nums:]
np.save(os.path.join(output_dir, 'train_o_set.npy'), train_o)
np.save(os.path.join(output_dir, 'train_b_set.npy'), train_b)
np.save(os.path.join(output_dir, 'valid_o_set.npy'), valid_o)
np.save(os.path.join(output_dir, 'valid_b_set.npy'), valid_b)
np.save(os.path.join(output_dir, 'test_o_set.npy'), test_o)
np.save(os.path.join(output_dir, 'test_b_set.npy'), test_b)
import context
from layers import ConvLayer
import utils
import numpy as np
patch_shape = (33,33)
nkerns = [64, 32]
batch_size = 5000
learning_rate = 3e-7
weight_decay = 2
# prepare the data
print ".preparing data"
dataset_paths = [
utils.complement_path('/share/blur_images/all_in_one/train_o_set.npy'),
utils.complement_path('/share/blur_images/all_in_one/train_b_set.npy'),
utils.complement_path('/share/blur_images/all_in_one/valid_o_set.npy'),
utils.complement_path('/share/blur_images/all_in_one/valid_b_set.npy'),
utils.complement_path('/share/blur_images/all_in_one/test_o_set.npy'),
utils.complement_path('/share/blur_images/all_in_one/test_b_set.npy')]
datasets = [
utils.shared_dataset(np.load(dataset_path)) for dataset_path in dataset_paths]
# build the network
print ".building network"
normal = T.fmatrix('normal')
corrupt = T.fmatrix('corrupt')
