def __call__(self):
"""returns the total count based on the query"""
awstat_pattern = api.portal.get_registry_record(
'awstats_hitcounter.awstats_url_pattern')
# we depend on a properly configured url pattern
# which is in the docs
site = api.portal.get()
path = self._relative_path()
downloads = None
self.request.response.setHeader("Content-type", "application/json")
if self.context.portal_type == "File":
downloads = counter("{0}/at_download/file".format(path),
awstat_pattern)
views = counter(path, awstat_pattern)
hits = counter(path, awstat_pattern, hits=True)
if views <= 0:
json_data = json.dumps({'success':'false'})
else:
content_type = self.context.Type()
creation_date = self.context.CreationDate()
creation_date = site.toLocalizedTime(creation_date)
modification_date = self.context.modified()
modification_date = site.toLocalizedTime(modification_date)
data = dict(success = "true",
creation_date = creation_date,
modification_date = modification_date,
page_views = views,
hits = hits,
content_type = content_type)
if downloads:
data['downloads'] = downloads
json_data = json.dumps(data)
return json_data
def choose_word(self):
if self.rand_order:
self.word = self.words.sample(n=1)
inx = self.word.index[0]
self.w_hanzi = self.word.sim[inx]
self.w_pinyin = self.word.pyt[inx]
self.w_pinyin_num = self.word.pyn[inx]
self.w_english = self.word.en[inx]
elif not self.rand_order:
from utils import counter
inx = counter()
print(inx)
# if inx > len(self.words)-1:
# counter(reset=True)
# inx = counter()
self.w_hanzi = self.word.sim[inx]
self.w_pinyin = self.word.pyt[inx]
self.w_pinyin_num = self.word.pyn[inx]
self.w_english = self.word.en[inx]
def plot_emoticons(self, amount=10): data = utils.counter(self.get_all_emoticons()) return utils.plot.hist( data, amount=max(amount, len(data)), sort="counter" )
def _create_friends_list(self):
# PATTERN_WITH = "(?<=with )(.*?)(?=\\s[@\\(]|$)"
# a name starts with a capital letter
# I write full names without spaces
_cappital_word = "[A-Z][A-Za-z]*"
# names list is
# Name Name Name and Name
# which is - at least one
# where the last one (if there is more than one) follows "and"
# and any name which is not the first nor the last has space before and after itself
_names_list = "((%s)( %s)*( and %s)*)" % (tuple([_cappital_word]) * 3)
PATTERN_WITH = "(?<=with )" + _names_list
PATTERN_FOR = "(?<=for )" + _names_list
descriptions = '\n'.join(i.description for i in self.data)
all_friends_raw = [i[0] for i in re.findall(PATTERN_WITH, descriptions)] \
+ [i[0] for i in re.findall(PATTERN_FOR , descriptions)]
self._friends_unsorted = '\n'.join(all_friends_raw).replace(
"and", "").split()
self.friends_histogram = utils.counter(self._friends_unsorted)
# counter object is a list of tuples (name, amount)
self.friends_histogram.sort(key=lambda x: x[1], reverse=True)
self.friends = [i[0] for i in self.friends_histogram]
def create_all_words(self):
self.words = sum([
P("WORDS").findall(i[MI("MESSAGE")])
for i in self.lines
if i[MI("TYPE")] == MT(0)
], [])
self.words_histogram = utils.counter(self.words)
return self.words
def TTS(word):
voices = engine.getProperty('voices')
engine.setProperty('voice', voices[17].id)
# setVolumn(0.25)
if counter() == 0:
say(word)
else:
engine.endLoop()
engine.stop()
say(word)
return "ok"
def plot_emojis(self, amount=5): # a = wp.d.get_all_emojis() # b = Counter(a) # c = list(b.items()) # d = list(map(lambda x:[wp.utils.emoji.emoji_to_hex(x[0]), x[1]], c)) data = list(map( lambda x: [ utils.emoji.emoji_to_hex(x[0]), x[1] ], utils.counter(self.get_all_emojis()) )) return utils.plot.emoji_bar( data, amount=amount, sort=lambda x: x[1], # sort by the value of the word and not the amount )
def fit(self, pts, polys, recache=False):
self._n = len(pts)
self._surface = Surface(pts, polys)
# check to see if the full K is cached
dataset_hash = str(hash(pts.tostring())) + str(hash(
polys.tostring())) + str(hash(self._m))
cache_path = os.path.join(self.CACHE_DIR,
dataset_hash + '_geodesic_K_cache.npz')
if not os.path.exists(cache_path) or recache:
print('Cache not found, computing K..')
K = np.vstack([
self._surface.geodesic_distance([i], m=self._m)
for i in counter(range(self._n))
])
self._K = (K + K.T) / 2.0
np.savez(cache_path, geodesic_K=self._K)
else:
print('Loading K from cache..')
self._K = np.load(cache_path)['geodesic_K']
def count(self, counter_value):
return counter(counter_value)
def train():
epoch = 200
batch_size = 1
lr = 0.0002
crop_size = 4
tar_db_a = "cameron_images.tgz"
tar_db_b = "teresa_images.tgz"
db_a_i = importer_tar.Importer(tar_db_a)
db_b_i = importer_tar.Importer(tar_db_b)
image_a_names = db_a_i.get_sorted_image_name()
image_b_names = db_b_i.get_sorted_image_name()
train_a_size = int(len(image_a_names) * 0.8)
train_b_size = int(len(image_b_names) * 0.8)
image_a_train_names = image_a_names[0:train_a_size]
image_b_train_names = image_b_names[0:train_b_size]
image_a_test_names = image_a_names[train_a_size:]
image_b_test_names = image_b_names[train_b_size:]
print("A train size:{},test size:{}".format(len(image_a_train_names),
len(image_a_test_names)))
print("B train size:{},test size:{}".format(len(image_b_train_names),
len(image_b_test_names)))
""" graph """
# models
generator_a2b = partial(models.generator, scope='a2b')
generator_b2a = partial(models.generator, scope='b2a')
discriminator_a = partial(models.discriminator, scope='a')
discriminator_b = partial(models.discriminator, scope='b')
#print("Discriminator shape:{}".format())
# operations
a_real = tf.placeholder(tf.float32, shape=[None, crop_size, crop_size, 3])
b_real = tf.placeholder(tf.float32, shape=[None, crop_size, crop_size, 3])
a2b_sample = tf.placeholder(tf.float32,
shape=[None, crop_size, crop_size, 3])
b2a_sample = tf.placeholder(tf.float32,
shape=[None, crop_size, crop_size, 3])
a2b = generator_a2b(a_real)
b2a = generator_b2a(b_real)
b2a2b = generator_a2b(b2a)
a2b2a = generator_b2a(a2b)
a_logit = discriminator_a(a_real)
b2a_logit = discriminator_a(b2a)
b2a_sample_logit = discriminator_a(b2a_sample)
b_logit = discriminator_b(b_real)
a2b_logit = discriminator_b(a2b)
a2b_sample_logit = discriminator_b(a2b_sample)
# losses
#g_loss_a2b = tf.losses.mean_squared_error(a2b_logit, tf.ones_like(a2b_logit))
#g_loss_b2a = tf.losses.mean_squared_error(b2a_logit, tf.ones_like(b2a_logit))
g_loss_a2b = -tf.reduce_mean(a2b_logit)
g_loss_b2a = -tf.reduce_mean(b22_logit)
cyc_loss_a = tf.losses.absolute_difference(a_real, a2b2a)
cyc_loss_b = tf.losses.absolute_difference(b_real, b2a2b)
g_loss = g_loss_a2b + g_loss_b2a + cyc_loss_a * 10.0 + cyc_loss_b * 10.0
# d_loss_a_real = tf.losses.mean_squared_error(a_logit, tf.ones_like(a_logit))
# d_loss_b2a_sample = tf.losses.mean_squared_error(b2a_sample_logit, tf.zeros_like(b2a_sample_logit))
# d_loss_a = d_loss_a_real + d_loss_b2a_sample
#
wd_a = tf.reduce_mean(a_logit) - tf.reduce_mean(b2a_logit)
wd_b = tf.reduce_mean(b_logit) - tf.reduce_mean(a2b_logit)
gp_a = gradient_penalty(a_real, b2a, discriminator_a)
gp_b = gradient_penalty(b_real, a2b, discriminator_b)
d_loss_a = -wd_a + 10.0 * gp_a
d_loss_b = -wd_b + 10.0 * gp_b
# d_loss_b_real = tf.losses.mean_squared_error(b_logit, tf.ones_like(b_logit))
# d_loss_a2b_sample = tf.losses.mean_squared_error(a2b_sample_logit, tf.zeros_like(a2b_sample_logit))
# d_loss_b = d_loss_b_real + d_loss_a2b_sample
# summaries
g_summary = utils.summary({
g_loss_a2b: 'g_loss_a2b',
g_loss_b2a: 'g_loss_b2a',
cyc_loss_a: 'cyc_loss_a',
cyc_loss_b: 'cyc_loss_b'
})
d_summary_a = utils.summary({d_loss_a: 'd_loss_a'})
d_summary_b = utils.summary({d_loss_b: 'd_loss_b'})
# optim
t_var = tf.trainable_variables()
d_a_var = [var for var in t_var if 'a_discriminator' in var.name]
d_b_var = [var for var in t_var if 'b_discriminator' in var.name]
g_var = [
var for var in t_var
if 'a2b_generator' in var.name or 'b2a_generator' in var.name
]
d_a_train_op = tf.train.AdamOptimizer(lr,
beta1=0.5).minimize(d_loss_a,
var_list=d_a_var)
d_b_train_op = tf.train.AdamOptimizer(lr,
beta1=0.5).minimize(d_loss_b,
var_list=d_b_var)
g_train_op = tf.train.AdamOptimizer(lr, beta1=0.5).minimize(g_loss,
var_list=g_var)
""" train """
''' init '''
# session
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
# counter
it_cnt, update_cnt = utils.counter()
''' summary '''
summary_writer = tf.summary.FileWriter('./outputs/summaries/', sess.graph)
''' saver '''
saver = tf.train.Saver(max_to_keep=5)
''' restore '''
ckpt_dir = './outputs/checkpoints/'
utils.mkdir(ckpt_dir)
try:
utils.load_checkpoint(ckpt_dir, sess)
except:
sess.run(tf.global_variables_initializer())
'''train'''
try:
batch_epoch = min(train_a_size, train_b_size) // batch_size
max_it = epoch * batch_epoch
for it in range(sess.run(it_cnt), max_it):
sess.run(update_cnt)
epoch = it // batch_epoch
it_epoch = it % batch_epoch + 1
# prepare data
a_real_ipt = db_a_i.get_image(image_a_train_names[it_epoch])
a_real_ipt = cv2.resize(a_real_ipt, (crop_size, crop_size),
interpolation=cv2.INTER_CUBIC)
b_real_ipt = db_b_i.get_image(image_b_train_names[it_epoch])
b_real_ipt = cv2.resize(b_real_ipt, (crop_size, crop_size),
interpolation=cv2.INTER_CUBIC)
a_real_ipt = [a_real_ipt]
b_real_ipt = [b_real_ipt]
a2b_opt, b2a_opt = sess.run([a2b, b2a],
feed_dict={
a_real: a_real_ipt,
b_real: b_real_ipt
})
a2b_sample_ipt = a2b_opt
b2a_sample_ipt = b2a_opt
# train G
g_summary_opt, _ = sess.run([g_summary, g_train_op],
feed_dict={
a_real: a_real_ipt,
b_real: b_real_ipt
})
summary_writer.add_summary(g_summary_opt, it)
# train D_b
d_summary_b_opt, _ = sess.run([d_summary_b, d_b_train_op],
feed_dict={
b_real: b_real_ipt,
a2b_sample: a2b_sample_ipt
})
summary_writer.add_summary(d_summary_b_opt, it)
# train D_a
d_summary_a_opt, _ = sess.run([d_summary_a, d_a_train_op],
feed_dict={
a_real: a_real_ipt,
b2a_sample: b2a_sample_ipt
})
summary_writer.add_summary(d_summary_a_opt, it)
# display
if it % 1 == 0:
print("Epoch: (%3d) (%5d/%5d)" %
(epoch, it_epoch, batch_epoch))
# save
if (it + 1) % 1000 == 0:
save_path = saver.save(
sess, '%s/Epoch_(%d)_(%dof%d).ckpt' %
(ckpt_dir, epoch, it_epoch, batch_epoch))
print('Model saved in file: % s' % save_path)
# sample
# if (it + 1) % 100 == 0:
# a_real_ipt = a_test_pool.batch()
# b_real_ipt = b_test_pool.batch()
# [a2b_opt, a2b2a_opt, b2a_opt, b2a2b_opt] = sess.run([a2b, a2b2a, b2a, b2a2b], feed_dict={a_real: a_real_ipt, b_real: b_real_ipt})
# sample_opt = np.concatenate((a_real_ipt, a2b_opt, a2b2a_opt, b_real_ipt, b2a_opt, b2a2b_opt), axis=0)
#
# save_dir = './outputs/sample_images_while_training/' + dataset
# utils.mkdir(save_dir)
# im.imwrite(im.immerge(sample_opt, 2, 3), '%s/Epoch_(%d)_(%dof%d).jpg' % (save_dir, epoch, it_epoch, batch_epoch))
except:
save_path = saver.save(
sess, '%s/Epoch_(%d)_(%dof%d).ckpt' %
(ckpt_dir, epoch, it_epoch, batch_epoch))
print('Model saved in file: % s' % save_path)
sess.close()
raise
for var in d_var))
g_step = tf.train.RMSPropOptimizer(learning_rate=lr).minimize(
g_loss, var_list=g_var)
# summaries
d_summary = utils.summary({wd: 'wd'})
g_summary = utils.summary({g_loss: 'g_loss'})
# sample
f_sample = generator(z, training=False)
""" train """
''' init '''
# session
sess = utils.session()
# iteration counter
it_cnt, update_cnt = utils.counter()
# saver
saver = tf.train.Saver(max_to_keep=5)
# summary writer
summary_writer = tf.summary.FileWriter('./summaries/celeba_wgan', sess.graph)
''' initialization '''
ckpt_dir = './checkpoints/celeba_wgan'
utils.mkdir(ckpt_dir + '/')
if not utils.load_checkpoint(ckpt_dir, sess):
sess.run(tf.global_variables_initializer())
''' train '''
try:
z_ipt_sample = np.random.normal(size=[100, z_dim])
batch_epoch = len(data_pool) // (batch_size * n_critic)
max_it = epoch * batch_epoch
def bootstrap_ridge(Rstim, Rresp, Pstim, Presp, alphas, nboots, chunklen, nchunks, dtype=np.single,
corrmin=0.2, joined=None, singcutoff=1e-10, normalpha=False, single_alpha=False,
use_corr=True, logger=logging.getLogger("ridge_corr")):
"""Uses ridge regression with a bootstrapped held-out set to get optimal alpha values for each response.
[nchunks] random chunks of length [chunklen] will be taken from [Rstim] and [Rresp] for each regression
run. [nboots] total regression runs will be performed. The best alpha value for each response will be
averaged across the bootstraps to estimate the best alpha for that response.
If [joined] is given, it should be a list of lists where the STRFs for all the voxels in each sublist
will be given the same regularization parameter (the one that is the best on average).
Parameters
----------
Rstim : array_like, shape (TR, N)
Training stimuli with TR time points and N features. Each feature should be Z-scored across time.
Rresp : array_like, shape (TR, M)
Training responses with TR time points and M different responses (voxels, neurons, what-have-you).
Each response should be Z-scored across time.
Pstim : array_like, shape (TP, N)
Test stimuli with TP time points and N features. Each feature should be Z-scored across time.
Presp : array_like, shape (TP, M)
Test responses with TP time points and M different responses. Each response should be Z-scored across
time.
alphas : list or array_like, shape (A,)
Ridge parameters that will be tested. Should probably be log-spaced. np.logspace(0, 3, 20) works well.
nboots : int
The number of bootstrap samples to run. 15 to 30 works well.
chunklen : int
On each sample, the training data is broken into chunks of this length. This should be a few times
longer than your delay/STRF. e.g. for a STRF with 3 delays, I use chunks of length 10.
nchunks : int
The number of training chunks held out to test ridge parameters for each bootstrap sample. The product
of nchunks and chunklen is the total number of training samples held out for each sample, and this
product should be about 20 percent of the total length of the training data.
dtype : np.dtype
All data will be cast as this dtype for computation. np.single is used by default for memory efficiency,
as using np.double will thrash most machines on a big problem. If you want to do regression on
complex variables, this should be changed to np.complex128.
corrmin : float in [0..1]
Purely for display purposes. After each alpha is tested for each bootstrap sample, the number of
responses with correlation greater than this value will be printed. For long-running regressions this
can give a rough sense of how well the model works before it's done.
joined : None or list of array_like indices
If you want the STRFs for two (or more) responses to be directly comparable, you need to ensure that
the regularization parameter that they use is the same. To do that, supply a list of the response sets
that should use the same ridge parameter here. For example, if you have four responses, joined could
be [np.array([0,1]), np.array([2,3])], in which case responses 0 and 1 will use the same ridge parameter
(which will be parameter that is best on average for those two), and likewise for responses 2 and 3.
singcutoff : float
The first step in ridge regression is computing the singular value decomposition (SVD) of the
stimulus Rstim. If Rstim is not full rank, some singular values will be approximately equal
to zero and the corresponding singular vectors will be noise. These singular values/vectors
should be removed both for speed (the fewer multiplications the better!) and accuracy. Any
singular values less than singcutoff will be removed.
normalpha : boolean
Whether ridge parameters (alphas) should be normalized by the Frobenius norm of Rstim. Good for rigorously
comparing models with different numbers of parameters.
single_alpha : boolean
Whether to use a single alpha for all responses. Good for identification/decoding.
use_corr : boolean
If True, this function will use correlation as its metric of model fit. If False, this function
will instead use variance explained (R-squared) as its metric of model fit. For ridge regression
this can make a big difference -- highly regularized solutions will have very small norms and
will thus explain very little variance while still leading to high correlations, as correlation
is scale-free while R**2 is not.
Returns
-------
wt : array_like, shape (N, M)
Regression weights for N features and M responses.
corrs : array_like, shape (M,)
Validation set correlations. Predicted responses for the validation set are obtained using the regression
weights: pred = np.dot(Pstim, wt), and then the correlation between each predicted response and each
column in Presp is found.
alphas : array_like, shape (M,)
The regularization coefficient (alpha) selected for each voxel using bootstrap cross-validation.
bootstrap_corrs : array_like, shape (A, M, B)
Correlation between predicted and actual responses on randomly held out portions of the training set,
for each of A alphas, M voxels, and B bootstrap samples.
valinds : array_like, shape (TH, B)
The indices of the training data that were used as "validation" for each bootstrap sample.
"""
nresp, nvox = Rresp.shape
bestalphas = np.zeros((nboots, nvox)) ## Will hold the best alphas for each voxel
valinds = [] ## Will hold the indices into the validation data for each bootstrap
Rcmats = []
for bi in counter(range(nboots), countevery=1, total=nboots):
logger.info("Selecting held-out test set..")
allinds = range(nresp)
indchunks = zip(*[iter(allinds)]*chunklen)
random.shuffle(indchunks)
heldinds = list(itools.chain(*indchunks[:nchunks]))
notheldinds = list(set(allinds)-set(heldinds))
valinds.append(heldinds)
RRstim = Rstim[notheldinds,:]
PRstim = Rstim[heldinds,:]
RRresp = Rresp[notheldinds,:]
PRresp = Rresp[heldinds,:]
## Run ridge regression using this test set
Rcmat = ridge_corr(RRstim, PRstim, RRresp, PRresp, alphas,
dtype=dtype, corrmin=corrmin, singcutoff=singcutoff,
normalpha=normalpha, use_corr=use_corr)
Rcmats.append(Rcmat)
## Find weights for each voxel
try:
U,S,Vh = np.linalg.svd(Rstim, full_matrices=False)
except np.linalg.LinAlgError, e:
logger.info("NORMAL SVD FAILED, trying more robust dgesvd..")
from text.regression.svd_dgesvd import svd_dgesvd
U,S,Vh = svd_dgesvd(Rstim, full_matrices=False)
def bootstrap_ridge(Rstim, Rresp, Pstim, Presp, alphas, nboots, chunklen, nchunks,
corrmin=0.2, joined=None, singcutoff=1e-10, normalpha=False, single_alpha=False,
use_corr=True, logger=ridge_logger):
"""Uses ridge regression with a bootstrapped held-out set to get optimal alpha values for each response.
[nchunks] random chunks of length [chunklen] will be taken from [Rstim] and [Rresp] for each regression
run. [nboots] total regression runs will be performed. The best alpha value for each response will be
averaged across the bootstraps to estimate the best alpha for that response.
If [joined] is given, it should be a list of lists where the STRFs for all the voxels in each sublist
will be given the same regularization parameter (the one that is the best on average).
Parameters
----------
Rstim : array_like, shape (TR, N)
Training stimuli with TR time points and N features. Each feature should be Z-scored across time.
Rresp : array_like, shape (TR, M)
Training responses with TR time points and M different responses (voxels, neurons, what-have-you).
Each response should be Z-scored across time.
Pstim : array_like, shape (TP, N)
Test stimuli with TP time points and N features. Each feature should be Z-scored across time.
Presp : array_like, shape (TP, M)
Test responses with TP time points and M different responses. Each response should be Z-scored across
time.
alphas : list or array_like, shape (A,)
Ridge parameters that will be tested. Should probably be log-spaced. np.logspace(0, 3, 20) works well.
nboots : int
The number of bootstrap samples to run. 15 to 30 works well.
chunklen : int
On each sample, the training data is broken into chunks of this length. This should be a few times
longer than your delay/STRF. e.g. for a STRF with 3 delays, I use chunks of length 10.
nchunks : int
The number of training chunks held out to test ridge parameters for each bootstrap sample. The product
of nchunks and chunklen is the total number of training samples held out for each sample, and this
product should be about 20 percent of the total length of the training data.
corrmin : float in [0..1]
Purely for display purposes. After each alpha is tested for each bootstrap sample, the number of
responses with correlation greater than this value will be printed. For long-running regressions this
can give a rough sense of how well the model works before it's done.
joined : None or list of array_like indices
If you want the STRFs for two (or more) responses to be directly comparable, you need to ensure that
the regularization parameter that they use is the same. To do that, supply a list of the response sets
that should use the same ridge parameter here. For example, if you have four responses, joined could
be [np.array([0,1]), np.array([2,3])], in which case responses 0 and 1 will use the same ridge parameter
(which will be parameter that is best on average for those two), and likewise for responses 2 and 3.
singcutoff : float
The first step in ridge regression is computing the singular value decomposition (SVD) of the
stimulus Rstim. If Rstim is not full rank, some singular values will be approximately equal
to zero and the corresponding singular vectors will be noise. These singular values/vectors
should be removed both for speed (the fewer multiplications the better!) and accuracy. Any
singular values less than singcutoff will be removed.
normalpha : boolean
Whether ridge parameters (alphas) should be normalized by the largest singular value (LSV)
norm of Rstim. Good for rigorously comparing models with different numbers of parameters.
single_alpha : boolean
Whether to use a single alpha for all responses. Good for identification/decoding.
use_corr : boolean
If True, this function will use correlation as its metric of model fit. If False, this function
will instead use variance explained (R-squared) as its metric of model fit. For ridge regression
this can make a big difference -- highly regularized solutions will have very small norms and
will thus explain very little variance while still leading to high correlations, as correlation
is scale-free while R**2 is not.
Returns
-------
wt : array_like, shape (N, M)
Regression weights for N features and M responses.
corrs : array_like, shape (M,)
Validation set correlations. Predicted responses for the validation set are obtained using the regression
weights: pred = np.dot(Pstim, wt), and then the correlation between each predicted response and each
column in Presp is found.
alphas : array_like, shape (M,)
The regularization coefficient (alpha) selected for each voxel using bootstrap cross-validation.
bootstrap_corrs : array_like, shape (A, M, B)
Correlation between predicted and actual responses on randomly held out portions of the training set,
for each of A alphas, M voxels, and B bootstrap samples.
valinds : array_like, shape (TH, B)
The indices of the training data that were used as "validation" for each bootstrap sample.
"""
nresp, nvox = Rresp.shape
valinds = [] # Will hold the indices into the validation data for each bootstrap
Rcmats = []
for bi in counter(range(nboots), countevery=1, total=nboots):
logger.info("Selecting held-out test set..")
allinds = range(nresp)
indchunks = zip(*[iter(allinds)]*chunklen)
random.shuffle(indchunks)
heldinds = list(itools.chain(*indchunks[:nchunks]))
notheldinds = list(set(allinds)-set(heldinds))
valinds.append(heldinds)
RRstim = Rstim[notheldinds,:]
PRstim = Rstim[heldinds,:]
RRresp = Rresp[notheldinds,:]
PRresp = Rresp[heldinds,:]
# Run ridge regression using this test set
Rcmat = ridge_corr(RRstim, PRstim, RRresp, PRresp, alphas,
corrmin=corrmin, singcutoff=singcutoff,
normalpha=normalpha, use_corr=use_corr,
logger=logger)
Rcmats.append(Rcmat)
# Find best alphas
if nboots>0:
allRcorrs = np.dstack(Rcmats)
else:
allRcorrs = None
if not single_alpha:
if nboots==0:
raise ValueError("You must run at least one cross-validation step to assign "
"different alphas to each response.")
logger.info("Finding best alpha for each voxel..")
if joined is None:
# Find best alpha for each voxel
meanbootcorrs = allRcorrs.mean(2)
bestalphainds = np.argmax(meanbootcorrs, 0)
valphas = alphas[bestalphainds]
else:
# Find best alpha for each group of voxels
valphas = np.zeros((nvox,))
for jl in joined:
# Mean across voxels in the set, then mean across bootstraps
jcorrs = allRcorrs[:,jl,:].mean(1).mean(1)
bestalpha = np.argmax(jcorrs)
valphas[jl] = alphas[bestalpha]
else:
logger.info("Finding single best alpha..")
if nboots==0:
if len(alphas)==1:
bestalphaind = 0
bestalpha = alphas[0]
else:
raise ValueError("You must run at least one cross-validation step "
"to choose best overall alpha, or only supply one"
"possible alpha value.")
else:
meanbootcorr = allRcorrs.mean(2).mean(1)
bestalphaind = np.argmax(meanbootcorr)
bestalpha = alphas[bestalphaind]
valphas = np.array([bestalpha]*nvox)
logger.info("Best alpha = %0.3f"%bestalpha)
# Find weights
logger.info("Computing weights for each response using entire training set..")
wt = ridge(Rstim, Rresp, valphas, singcutoff=singcutoff, normalpha=normalpha)
# Predict responses on prediction set
logger.info("Predicting responses for predictions set..")
pred = np.dot(Pstim, wt)
# Find prediction correlations
nnpred = np.nan_to_num(pred)
if use_corr:
corrs = np.nan_to_num(np.array([np.corrcoef(Presp[:,ii], nnpred[:,ii].ravel())[0,1]
for ii in range(Presp.shape[1])]))
else:
resvar = (Presp-pred).var(0)
Rsqs = 1 - (resvar / Presp.var(0))
corrs = np.sqrt(np.abs(Rsqs)) * np.sign(Rsqs)
return wt, corrs, valphas, allRcorrs, valinds
def get_features_ratio(self):
ratio = {}
ratio_description = {}
# message ratio
ratio_description["message"] = "[amount of messages from] user0 / user1"
ratio["message"] = numpy.divide(*self.user_message_amount)
# media ratio
ratio_description["media"] = "[amount of media from] user0 / user1"
ratio["media"] = numpy.divide(*self.user_media_amount)
# words ratio
ratio_description["words"] = "[amount of words sent by] user0 / user1"
ratio["words"] = numpy.divide(*self.user_words_amount)
# long_messages ratio
ratio_description["long messages"] = "[amount of messages with more than 10 words from] user0 / user1"
_long_messages = self.get_messages(lambda x:len(x[MI('M')].split())>=10)
_long_messages_amount_per_user = list(map(
# iterate all the users
# and count how many long messages each of them has
lambda u: len(list(filter(
# iterate all the long messages
# and filter out only messages by the user
lambda m: m[MI('U')] == u,
_long_messages
))),
self.users
))
ratio["long messages"] = numpy.divide(*_long_messages_amount_per_user)
del _long_messages
del _long_messages_amount_per_user
# messages with h
ratio_description["messages with h"] = "[amount of messages with h by] user0 / user1"
_messages_with_h = self.get_messages(P("H"))
_messages_with_h_per_user = list(map(
lambda u: len(list(filter(
lambda m: m[MI('U')] == u,
_messages_with_h
))),
self.users
))
ratio["messages with h"] = numpy.divide(*_messages_with_h_per_user)
del _messages_with_h
del _messages_with_h_per_user
ratio_description["conversations"] = "[amount of conversations started by] user0 / user1"
_conversations_started = utils.counter([
c[0][MI("USER")]
for c in self.chats
])
ratio["conversations"] = numpy.divide(*[
# get the conversation started by the user
next(filter(
lambda x: x[0] == u,
_conversations_started
# get the amount of conversations
))[1]
for u in
# keep the order of the users
self.users
])
return ratio, ratio_description
def preprocess2d(mode='train'):
"""
Split up all 3d volumes into separate 2d slice images and store as separate files.
Volumes are fetched from directory specified in config.
"""
## Generator to generate indices for slices.
numgen = counter()
dirpaths = [
config[f"dst2d_labels_liver_path"],
config[f"dst2d_labels_lesion_path"],
config[f"dst2d_slices_path"],
]
for p in dirpaths:
if not os.path.exists(p):
os.makedirs(p)
## Processing
start = time()
for f in tqdm(os.listdir(config[f"{config['mode']}_volumes_path"]), desc=f"{config['mode']} 2d processing"):
## Get Volume matrices
ct = sitk.ReadImage(os.path.join(config[f"{config['mode']}_volumes_path"], f), sitk.sitkInt16)
ct_array = sitk.GetArrayFromImage(ct)
seg = sitk.ReadImage(os.path.join(config[f"{config['mode']}_labels_path"], f.replace('volume', 'segmentation')), sitk.sitkUInt8)
seg_array = sitk.GetArrayFromImage(seg)
## TODO: Change if looking for tumors
## Make all cancer labels to liverlabels (for liver segmentation, not tumor segmentation)
liver_array = np.zeros_like(seg_array)
lesion_array = np.zeros_like(seg_array)
liver_array[seg_array >= 1] = 1
lesion_array[seg_array > 1] = 1
## Clip upper and lower values of CT images
ct_array[ct_array > config["upper"]] = config["upper"]
ct_array[ct_array < config["lower"]] = config["lower"]
## Pick out relevant slices
z = np.any(liver_array, axis=(1, 2))
start_slice, end_slice = np.where(z)[0][[0, -1]]
start_slice = max(0, start_slice - config["expand_slice"])
end_slice = min(liver_array.shape[0] - 1, end_slice + config["expand_slice"])
ct_array = ct_array[start_slice:end_slice, :, :]
liver_array = liver_array[start_slice:end_slice, :, :]
lesion_array = lesion_array[start_slice:end_slice, :, :]
if ct_array.shape[0] != liver_array.shape[0]:
breakpoint()
## Store each slice as .mat numpy array
for i in range(ct_array.shape[0]):
idx = next(numgen)
slice_filename = "slice_{:05}".format(idx)
segmentation_filename = "segmentation_{:05}".format(idx)
np.save(os.path.join(config[f"dst2d_slices_path"], slice_filename),
ct_array[i, ...])
np.save(os.path.join(config[f"dst2d_labels_liver_path"], segmentation_filename),
liver_array[i, ...])
np.save(os.path.join(config[f"dst2d_labels_lesion_path"], segmentation_filename),
lesion_array[i, ...])
print("Finished {} preprocessing in: {:02}s".format(config['mode'], time() - start))
def bootstrap_ridge(Rstim,
Rresp,
Pstim,
Presp,
alphas,
nboots,
chunklen,
nchunks,
corrmin=0.2,
joined=None,
singcutoff=1e-10,
normalpha=False,
single_alpha=False,
use_corr=True,
logger=ridge_logger):
"""Uses ridge regression with a bootstrapped held-out set to get optimal alpha values for each response.
[nchunks] random chunks of length [chunklen] will be taken from [Rstim] and [Rresp] for each regression
run. [nboots] total regression runs will be performed. The best alpha value for each response will be
averaged across the bootstraps to estimate the best alpha for that response.
If [joined] is given, it should be a list of lists where the STRFs for all the voxels in each sublist
will be given the same regularization parameter (the one that is the best on average).
Parameters
----------
Rstim : array_like, shape (TR, N)
Training stimuli with TR time points and N features. Each feature should be Z-scored across time.
Rresp : array_like, shape (TR, M)
Training responses with TR time points and M different responses (voxels, neurons, what-have-you).
Each response should be Z-scored across time.
Pstim : array_like, shape (TP, N)
Test stimuli with TP time points and N features. Each feature should be Z-scored across time.
Presp : array_like, shape (TP, M)
Test responses with TP time points and M different responses. Each response should be Z-scored across
time.
alphas : list or array_like, shape (A,)
Ridge parameters that will be tested. Should probably be log-spaced. np.logspace(0, 3, 20) works well.
nboots : int
The number of bootstrap samples to run. 15 to 30 works well.
chunklen : int
On each sample, the training data is broken into chunks of this length. This should be a few times
longer than your delay/STRF. e.g. for a STRF with 3 delays, I use chunks of length 10.
nchunks : int
The number of training chunks held out to test ridge parameters for each bootstrap sample. The product
of nchunks and chunklen is the total number of training samples held out for each sample, and this
product should be about 20 percent of the total length of the training data.
corrmin : float in [0..1]
Purely for display purposes. After each alpha is tested for each bootstrap sample, the number of
responses with correlation greater than this value will be printed. For long-running regressions this
can give a rough sense of how well the model works before it's done.
joined : None or list of array_like indices
If you want the STRFs for two (or more) responses to be directly comparable, you need to ensure that
the regularization parameter that they use is the same. To do that, supply a list of the response sets
that should use the same ridge parameter here. For example, if you have four responses, joined could
be [np.array([0,1]), np.array([2,3])], in which case responses 0 and 1 will use the same ridge parameter
(which will be parameter that is best on average for those two), and likewise for responses 2 and 3.
singcutoff : float
The first step in ridge regression is computing the singular value decomposition (SVD) of the
stimulus Rstim. If Rstim is not full rank, some singular values will be approximately equal
to zero and the corresponding singular vectors will be noise. These singular values/vectors
should be removed both for speed (the fewer multiplications the better!) and accuracy. Any
singular values less than singcutoff will be removed.
normalpha : boolean
Whether ridge parameters (alphas) should be normalized by the largest singular value (LSV)
norm of Rstim. Good for rigorously comparing models with different numbers of parameters.
single_alpha : boolean
Whether to use a single alpha for all responses. Good for identification/decoding.
use_corr : boolean
If True, this function will use correlation as its metric of model fit. If False, this function
will instead use variance explained (R-squared) as its metric of model fit. For ridge regression
this can make a big difference -- highly regularized solutions will have very small norms and
will thus explain very little variance while still leading to high correlations, as correlation
is scale-free while R**2 is not.
Returns
-------
wt : array_like, shape (N, M)
Regression weights for N features and M responses.
corrs : array_like, shape (M,)
Validation set correlations. Predicted responses for the validation set are obtained using the regression
weights: pred = np.dot(Pstim, wt), and then the correlation between each predicted response and each
column in Presp is found.
alphas : array_like, shape (M,)
The regularization coefficient (alpha) selected for each voxel using bootstrap cross-validation.
bootstrap_corrs : array_like, shape (A, M, B)
Correlation between predicted and actual responses on randomly held out portions of the training set,
for each of A alphas, M voxels, and B bootstrap samples.
valinds : array_like, shape (TH, B)
The indices of the training data that were used as "validation" for each bootstrap sample.
"""
nresp, nvox = Rresp.shape
valinds = [
] # Will hold the indices into the validation data for each bootstrap
Rcmats = []
for bi in counter(range(nboots), countevery=1, total=nboots):
logger.info("Selecting held-out test set..")
allinds = range(nresp)
indchunks = zip(*[iter(allinds)] * chunklen)
random.shuffle(indchunks)
heldinds = list(itools.chain(*indchunks[:nchunks]))
notheldinds = list(set(allinds) - set(heldinds))
valinds.append(heldinds)
RRstim = Rstim[notheldinds, :]
PRstim = Rstim[heldinds, :]
RRresp = Rresp[notheldinds, :]
PRresp = Rresp[heldinds, :]
# Run ridge regression using this test set
Rcmat = ridge_corr(RRstim,
PRstim,
RRresp,
PRresp,
alphas,
corrmin=corrmin,
singcutoff=singcutoff,
normalpha=normalpha,
use_corr=use_corr,
logger=logger)
Rcmats.append(Rcmat)
# Find best alphas
if nboots > 0:
allRcorrs = np.dstack(Rcmats)
else:
allRcorrs = None
if not single_alpha:
if nboots == 0:
raise ValueError(
"You must run at least one cross-validation step to assign "
"different alphas to each response.")
logger.info("Finding best alpha for each voxel..")
if joined is None:
# Find best alpha for each voxel
meanbootcorrs = allRcorrs.mean(2)
bestalphainds = np.argmax(meanbootcorrs, 0)
valphas = alphas[bestalphainds]
else:
# Find best alpha for each group of voxels
valphas = np.zeros((nvox, ))
for jl in joined:
# Mean across voxels in the set, then mean across bootstraps
jcorrs = allRcorrs[:, jl, :].mean(1).mean(1)
bestalpha = np.argmax(jcorrs)
valphas[jl] = alphas[bestalpha]
else:
logger.info("Finding single best alpha..")
if nboots == 0:
if len(alphas) == 1:
bestalphaind = 0
bestalpha = alphas[0]
else:
raise ValueError(
"You must run at least one cross-validation step "
"to choose best overall alpha, or only supply one"
"possible alpha value.")
else:
meanbootcorr = allRcorrs.mean(2).mean(1)
bestalphaind = np.argmax(meanbootcorr)
bestalpha = alphas[bestalphaind]
valphas = np.array([bestalpha] * nvox)
logger.info("Best alpha = %0.3f" % bestalpha)
# Find weights
logger.info(
"Computing weights for each response using entire training set..")
wt = ridge(Rstim,
Rresp,
valphas,
singcutoff=singcutoff,
normalpha=normalpha)
# Predict responses on prediction set
logger.info("Predicting responses for predictions set..")
pred = np.dot(Pstim, wt)
# Find prediction correlations
nnpred = np.nan_to_num(pred)
if use_corr:
corrs = np.nan_to_num(
np.array([
np.corrcoef(Presp[:, ii], nnpred[:, ii].ravel())[0, 1]
for ii in range(Presp.shape[1])
]))
else:
resvar = (Presp - pred).var(0)
Rsqs = 1 - (resvar / Presp.var(0))
corrs = np.sqrt(np.abs(Rsqs)) * np.sign(Rsqs)
return wt, corrs, valphas, allRcorrs, valinds
def main(epoch, batch_size, lr, z_dim, bottle_dim, i_c, alpha, n_critic,
gpu_id, data_pool):
with tf.device('/gpu:%d' % gpu_id): #Placing the ops under devices
generator = models.generator #Generator Object
discriminator = models.discriminator_wgan_gp #Discriminator Object
# inputs Placeholders
real = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
z = tf.placeholder(tf.float32, shape=[None, z_dim])
# generate fake data with the generator
fake = generator(z, reuse=False)
# Obtaining scores , means and stds for real and fake data from the discriminator
r_logit, r_mus, r_sigmas = discriminator(real,
reuse=False,
gen_train=False,
bottleneck_dim=bottle_dim)
f_logit, f_mus, f_sigmas = discriminator(fake,
gen_train=False,
bottleneck_dim=bottle_dim)
#Obtaining wasserstein loss and gradient penalty losses to train the discriminator
wasserstein_d = losses.wgan_loss(r_logit, f_logit)
gp = losses.gradient_penalty(real, fake, discriminator)
#We obtain the bottleneck loss in the discriminator
#Inputs to this function are bottleneck layer mus and stds for both real and fake data. i_c is the
#the information constriant or upperbound. This is an important paramters
bottleneck_loss=losses._bottleneck_loss(real_mus=r_mus, fake_mus=f_mus,\
real_sigmas=r_sigmas,fake_sigmas=f_sigmas,i_c=i_c)
#This used in lagrangian multiplier optimization. This is paramters also get updated adaptivly.
#To read more about duel gradient desenet in deep learning please read - https://medium.com/@jonathan_hui/rl-dual-gradient-descent-fac524c1f049
#Initialize with the zero
beta = tf.Variable(tf.zeros([]), name="beta")
#Combined both losses (10 is the default hyper paramters given by the paper
# - https://arxiv.org/pdf/1704.00028.pdf )
d_loss = -wasserstein_d + gp * 10.0 + beta * bottleneck_loss
#We said b also should adaptively get updated. Here we maximize the beta paramters with follwoing function
#Please refer to the VDB paper's equation (9) understand more about the update
beta_new = tf.maximum(0.0, beta + alpha * bottleneck_loss)
#This is the main difference from the pytoch implementation. In tensorlfow we have a static graph. S
# to update the beta with above menitoned function we have to use tf.assign()
assign_op = tf.assign(beta, beta_new) #beta.assign(beta_new)
#This is the generator loss
#As described in the paper we have a simple loss to the generator which uses mean scores from
#the generated samples
f_logit_gen, f_mus_gen, f_sigmas_gen = discriminator(
fake, gen_train=True, bottleneck_dim=bottle_dim)
g_loss = -tf.reduce_mean(f_logit_gen)
#Assigning two optimizers to train both Generator and the Discriminator
d_var = tf.trainable_variables('discriminator')
g_var = tf.trainable_variables('generator')
d_step = tf.train.RMSPropOptimizer(learning_rate=lr).minimize(
d_loss, var_list=d_var)
g_step = tf.train.RMSPropOptimizer(learning_rate=lr).minimize(
g_loss, var_list=g_var)
# Tensorbored summaries for plot losses
wd = wasserstein_d
d_summary = utils.summary({wd: 'wd', gp: 'gp'})
g_summary = utils.summary({g_loss: 'g_loss'})
beta_summary = utils.b_summary(beta)
#beta_summary = utils.summary({beta: 'beta'})
#sess= tf.Session()
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
with tf.Session(config=config) as sess:
# iteration counter
it_cnt, update_cnt = utils.counter()
# saver
saver = tf.train.Saver(
max_to_keep=5
) #Use to save both generator and discriminator paramters
# summary writer
summary_writer = tf.summary.FileWriter('./summaries/mnist_wgan_gp',
sess.graph)
''' Checking for previuosly trained checkpints'''
ckpt_dir = './checkpoints/mnist_wgan_gp'
utils.mkdir(ckpt_dir + '/')
if not utils.load_checkpoint(ckpt_dir, sess):
sess.run(tf.global_variables_initializer())
#Starting the training loop
batch_epoch = len(data_pool) // (batch_size * n_critic)
max_it = epoch * batch_epoch
for it in range(sess.run(it_cnt), max_it):
sess.run(update_cnt)
# which epoch
epoch = it // batch_epoch
it_epoch = it % batch_epoch + 1
# train D
for i in range(
n_critic
): #Fist we train the discriminator for few iterations (Here I used only 1)
# batch data
real_ipt = data_pool.batch('img') #Read data batch
z_ipt = np.random.normal(size=[batch_size,
z_dim]) #Sample nosice input
d_summary_opt, _ = sess.run([d_summary, d_step],
feed_dict={
real: real_ipt,
z: z_ipt
}) #Discriminator Gradient Update
beta_summary_opt = sess.run(beta_summary)
#_ = sess.run([d_step], feed_dict={real: real_ipt, z: z_ipt})
sess.run([assign_op], feed_dict={
real: real_ipt,
z: z_ipt
}) #Adpatively update the beta parameter
summary_writer.add_summary(d_summary_opt, it)
summary_writer.add_summary(beta_summary_opt, it)
# train the geenrator (Here we have a simple generator as in normal Wgan)
z_ipt = np.random.normal(size=[batch_size, z_dim])
g_summary_opt, _ = sess.run([g_summary, g_step],
feed_dict={z: z_ipt})
#_ = sess.run([g_step], feed_dict={z: z_ipt})
summary_writer.add_summary(g_summary_opt, it)
# display training progress
if it % 100 == 0:
print("Epoch: (%3d) (%5d/%5d)" %
(epoch, it_epoch, batch_epoch))
# saving the checpoints after every 1000 interation
if (it + 1) % 1000 == 0:
save_path = saver.save(
sess, '%s/Epoch_(%d)_(%dof%d).ckpt' %
(ckpt_dir, epoch, it_epoch, batch_epoch))
print('Model saved in file: % s' % save_path)
#This is to save the image generation during the trainign as tiles
if (it + 1) % 100 == 0:
z_input_sample = np.random.normal(size=[100,
z_dim]) #Noise samples
f_sample = generator(z)
f_sample_opt = sess.run(f_sample,
feed_dict={z: z_input_sample})
save_dir = './sample_images_while_training/mnist_wgan_gp'
utils.mkdir(save_dir + '/')
utils.imwrite(
utils.immerge(f_sample_opt, 10,
10), '%s/Epoch_(%d)_(%dof%d).jpg' %
(save_dir, epoch, it_epoch, batch_epoch))
return decision, (person, persons[1])
else:
print(
f"{person.name} perdeu a vez por passar uma decisão inválida!")
return decision, (person, )
@persons.event
def on_decision(decision):
print("Decisão escolhida:", decision.name, end="\n\n")
sleep(1)
@persons.event
def on_dead(person):
print(f"{person.name} morreu! Fim de jogo!")
while persons.alive:
system("cls || clear") # Clear terminal in Windows/Linux
person = persons[counter()]
print(f"Vez de {person.name} - {person.stats}!", end="\n\n")
decision, arguments = person.make_decision(decisions)
decision.action(*arguments)
sleep(3)
def train():
epoch = 200
batch_size = 1
lr = 0.0002
crop_size = 128
load_size = 128
tar_db_a = "cameron_images.tgz"
tar_db_b = "teresa_images.tgz"
db_a_i = importer_tar.Importer(tar_db_a)
db_b_i = importer_tar.Importer(tar_db_b)
image_a_names = db_a_i.get_sorted_image_name()
image_b_names = db_b_i.get_sorted_image_name()
train_a_size = int(len(image_a_names) * 0.8)
train_b_size = int(len(image_b_names) * 0.8)
image_a_train_names = image_a_names[0:train_a_size]
image_b_train_names = image_b_names[0:train_b_size]
image_a_test_names = image_a_names[train_a_size:]
image_b_test_names = image_b_names[train_b_size:]
print("A train size:{},test size:{}".format(len(image_a_train_names),
len(image_a_test_names)))
print("B train size:{},test size:{}".format(len(image_b_train_names),
len(image_b_test_names)))
""" graph """
# models
generator_a2b = partial(models.generator, scope='a2b')
generator_b2a = partial(models.generator, scope='b2a')
discriminator_a = partial(models.discriminator, scope='a')
discriminator_b = partial(models.discriminator, scope='b')
# operations
a_real_in = tf.placeholder(tf.float32,
shape=[None, load_size, load_size, 3],
name="a_real")
b_real_in = tf.placeholder(tf.float32,
shape=[None, load_size, load_size, 3],
name="b_real")
a_real = utils.preprocess_image(a_real_in, crop_size=crop_size)
b_real = utils.preprocess_image(b_real_in, crop_size=crop_size)
a2b = generator_a2b(a_real)
b2a = generator_b2a(b_real)
b2a2b = generator_a2b(b2a)
a2b2a = generator_b2a(a2b)
a_logit = discriminator_a(a_real)
b2a_logit = discriminator_a(b2a)
b_logit = discriminator_b(b_real)
a2b_logit = discriminator_b(a2b)
# losses
g_loss_a2b = -tf.reduce_mean(a2b_logit)
g_loss_b2a = -tf.reduce_mean(b2a_logit)
cyc_loss_a = tf.losses.absolute_difference(a_real, a2b2a)
cyc_loss_b = tf.losses.absolute_difference(b_real, b2a2b)
g_loss = g_loss_a2b + g_loss_b2a + cyc_loss_a * 10.0 + cyc_loss_b * 10.0
wd_a = tf.reduce_mean(a_logit) - tf.reduce_mean(b2a_logit)
wd_b = tf.reduce_mean(b_logit) - tf.reduce_mean(a2b_logit)
gp_a = gradient_penalty(a_real, b2a, discriminator_a)
gp_b = gradient_penalty(b_real, a2b, discriminator_b)
d_loss_a = -wd_a + 10.0 * gp_a
d_loss_b = -wd_b + 10.0 * gp_b
# summaries
utils.summary({
g_loss_a2b: 'g_loss_a2b',
g_loss_b2a: 'g_loss_b2a',
cyc_loss_a: 'cyc_loss_a',
cyc_loss_b: 'cyc_loss_b'
})
utils.summary({d_loss_a: 'd_loss_a'})
utils.summary({d_loss_b: 'd_loss_b'})
for variable in slim.get_model_variables():
tf.summary.histogram(variable.op.name, variable)
merged = tf.summary.merge_all()
im1_op = tf.summary.image("real_a", a_real_in)
im2_op = tf.summary.image("a2b", a2b)
im3_op = tf.summary.image("b2a2b", b2a2b)
im4_op = tf.summary.image("real_b", b_real_in)
im5_op = tf.summary.image("b2a", b2a)
im6_op = tf.summary.image("b2a2b", b2a2b)
# optim
t_var = tf.trainable_variables()
d_a_var = [var for var in t_var if 'a_discriminator' in var.name]
d_b_var = [var for var in t_var if 'b_discriminator' in var.name]
g_var = [
var for var in t_var
if 'a2b_generator' in var.name or 'b2a_generator' in var.name
]
d_a_train_op = tf.train.AdamOptimizer(lr,
beta1=0.5).minimize(d_loss_a,
var_list=d_a_var)
d_b_train_op = tf.train.AdamOptimizer(lr,
beta1=0.5).minimize(d_loss_b,
var_list=d_b_var)
g_train_op = tf.train.AdamOptimizer(lr, beta1=0.5).minimize(g_loss,
var_list=g_var)
""" train """
''' init '''
# session
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
# counter
it_cnt, update_cnt = utils.counter()
''' summary '''
summary_writer = tf.summary.FileWriter('./outputs/summaries/', sess.graph)
''' saver '''
saver = tf.train.Saver(max_to_keep=5)
''' restore '''
ckpt_dir = './outputs/checkpoints/'
utils.mkdir(ckpt_dir)
try:
utils.load_checkpoint(ckpt_dir, sess)
except:
sess.run(tf.global_variables_initializer())
'''train'''
try:
batch_epoch = min(train_a_size, train_b_size) // batch_size
max_it = epoch * batch_epoch
for it in range(sess.run(it_cnt), max_it):
sess.run(update_cnt)
epoch = it // batch_epoch
it_epoch = it % batch_epoch + 1
# read data
a_real_np = cv2.resize(
db_a_i.get_image(image_a_train_names[it_epoch % batch_epoch]),
(load_size, load_size))
b_real_np = cv2.resize(
db_b_i.get_image(image_b_train_names[it_epoch % batch_epoch]),
(load_size, load_size))
# train G
sess.run(g_train_op,
feed_dict={
a_real_in: [a_real_np],
b_real_in: [b_real_np]
})
# train discriminator
sess.run([d_a_train_op, d_b_train_op],
feed_dict={
a_real_in: [a_real_np],
b_real_in: [b_real_np]
})
# display
if it % 100 == 0:
# make summary
summary = sess.run(merged,
feed_dict={
a_real_in: [a_real_np],
b_real_in: [b_real_np]
})
summary_writer.add_summary(summary, it)
print("Epoch: (%3d) (%5d/%5d)" %
(epoch, it_epoch, batch_epoch))
# save
if (it + 1) % 1000 == 0:
save_path = saver.save(
sess, '{}/epoch_{}_{}.ckpt'.format(ckpt_dir, epoch,
it_epoch))
print('###Model saved in file: {}'.format(save_path))
# sample
if (it + 1) % 1000 == 0:
a_test_index = int(
np.random.uniform(high=len(image_a_test_names)))
b_test_index = int(
np.random.uniform(high=len(image_b_test_names)))
a_real_np = cv2.resize(
db_a_i.get_image(image_a_test_names[a_test_index]),
(load_size, load_size))
b_real_np = cv2.resize(
db_b_i.get_image(image_b_test_names[b_test_index]),
(load_size, load_size))
[a_opt, a2b_opt, a2b2a_opt, b_opt, b2a_opt, b2a2b_opt
] = sess.run([a_real, a2b, a2b2a, b_real, b2a, b2a2b],
feed_dict={
a_real_in: [a_real_np],
b_real_in: [b_real_np]
})
sample_opt = np.concatenate(
(a_opt, a2b_opt, a2b2a_opt, b_opt, b2a_opt, b2a2b_opt),
axis=0)
[im1_sum,im2_sum,im3_sum,im4_sum,im5_sum,im6_sum] = \
sess.run([im1_op,im2_op,im3_op,im4_op,im5_op,im6_op],
feed_dict={a_real_in: [a_real_np], b_real_in: [b_real_np]})
summary_writer.add_summary(im1_sum, it)
summary_writer.add_summary(im2_sum, it)
summary_writer.add_summary(im3_sum, it)
summary_writer.add_summary(im4_sum, it)
summary_writer.add_summary(im5_sum, it)
summary_writer.add_summary(im6_sum, it)
save_dir = './outputs/sample_images_while_training/'
utils.mkdir(save_dir)
im.imwrite(
im.immerge(sample_opt, 2, 3),
'{}/epoch_{}_it_{}.jpg'.format(save_dir, epoch, it_epoch))
except:
raise
finally:
save_path = saver.save(
sess, '{}/epoch_{}_{}.ckpt'.format(ckpt_dir, epoch, it_epoch))
print('###Model saved in file: {}'.format(save_path))
sess.close()
def compute_P(self):
Winds = self._Winds
nonWinds = self._nonWinds
n = self._M.shape[0]
# pull out part of M for unselected points
M_aa = self._M[nonWinds,:][:,nonWinds].tocsc()
# pull out part of M that crosses selected and unselected points
M_ab = self._M[Winds,:][:,nonWinds]
if self._nnz_row is not None:
if self._sparse_direction == 'cols':
self._threshold = self._nnz_row * (n-self._l) // self._l
else:
self._threshold = self._nnz_row
try:
from sksparse.cholmod import cholesky
solve_method = 'cholmod'
except ImportError:
solve_method = 'spsolve'
# compute Pprime, part of the dense interpolation matrix
if self._threshold is None:
if solve_method == 'spsolve':
Pprime = sparse.linalg.spsolve(M_aa, -M_ab.T)
elif solve_method == 'cholmod':
Pprime = cholesky(M_aa).solve_A(-M_ab.T)
# compute P, the full dense interpolation matrix
P = np.zeros((n, self._l))
P[nonWinds,:] = Pprime.todense()
P[Winds,:] = np.eye(self._l)
Pnnz = n * self._l
if self._soft_p is not None:
# don't force P to be exactly identity for known points,
# allow it to fudge a little
print("Softening P..")
M_bb = self._M[Winds,:][:,Winds]
soft_eye = sparse.eye(self._l) * self._soft_p
to_invert = (M_bb + soft_eye + M_ab.dot(Pprime)).todense()
soft_factor = np.linalg.inv(to_invert) * self._soft_p
P = P.dot(soft_factor).A
else:
# Compute the sparse bha
if solve_method == 'cholmod':
chol_M_aa = cholesky(M_aa)
if self._sparse_direction == 'rows':
thresh = THR_ROWS(k=self._threshold)
Prows = np.empty(self._threshold*(n-self._l)+self._l, dtype=int)
Pcols = np.empty(self._threshold*(n-self._l)+self._l, dtype=int)
Pvals = np.empty(self._threshold*(n-self._l)+self._l)
else:
thresh = THR(k=self._threshold)
Prows = np.empty(self._threshold*self._l+self._l, dtype=int)
Pcols = np.empty(self._threshold*self._l+self._l, dtype=int)
Pvals = np.empty(self._threshold*self._l+self._l)
chunk_size = 64 # min(self._l // self._njobs, 64)
chunks = self._l // chunk_size + ((self._l % chunk_size) > 0)
for chunk in counter(range(chunks)):
start = chunk*chunk_size
end = min(((chunk+1)*chunk_size, self._l))
if solve_method == 'spsolve':
sol = sparse.linalg.spsolve(M_aa, -M_ab.T[:, start:end].toarray())
elif solve_method == 'cholmod':
sol = chol_M_aa.solve_A(-M_ab.T[:, start:end].toarray())
if self._sparse_direction == 'rows':
thresh.fit(sol)
else:
if self._fit_group:
l_i = 0
for l in range(start,end):
thresh.fit_partition(sol[:, l_i])
Prows[l*self._threshold:(l+1)*self._threshold] = nonWinds[thresh._idxs]
Pvals[l*self._threshold:(l+1)*self._threshold] = thresh._vals
l_i += 1
else:
l_i = 0
for l in range(start,end):
thresh.fit(sol[:, l_i])
Prows[l*self._threshold:(l+1)*self._threshold] = nonWinds[thresh._idxs]
Pvals[l*self._threshold:(l+1)*self._threshold] = thresh._vals
l_i += 1
if self._sparse_direction == 'rows':
cols, vals = thresh.get_best_k()
Prows[:(n-self._l)*self._threshold] = np.repeat(nonWinds[np.arange(n-self._l)],self._threshold)
Pcols[:(n-self._l)*self._threshold] = cols
Pvals[:(n-self._l)*self._threshold] = vals
lastnonWindElement = (n-self._l)*self._threshold
else:
Pcols[:self._l*self._threshold] = np.repeat(np.arange(self._l),self._threshold)
lastnonWindElement = self._l*self._threshold
# add the identity for indices in W
Prows[lastnonWindElement:] = Winds
Pcols[lastnonWindElement:] = np.arange(self._l)
Pvals[lastnonWindElement:] = 1.0
P = sparse.csr_matrix((Pvals,(Prows, Pcols)), shape=(n,self._l))
P.eliminate_zeros()
Pnnz = P.nnz
# save values
self._nnz = Pnnz
self._P = P
