def __call__(self): """returns the total count based on the query""" awstat_pattern = api.portal.get_registry_record( 'awstats_hitcounter.awstats_url_pattern') # we depend on a properly configured url pattern # which is in the docs site = api.portal.get() path = self._relative_path() downloads = None self.request.response.setHeader("Content-type", "application/json") if self.context.portal_type == "File": downloads = counter("{0}/at_download/file".format(path), awstat_pattern) views = counter(path, awstat_pattern) hits = counter(path, awstat_pattern, hits=True) if views <= 0: json_data = json.dumps({'success':'false'}) else: content_type = self.context.Type() creation_date = self.context.CreationDate() creation_date = site.toLocalizedTime(creation_date) modification_date = self.context.modified() modification_date = site.toLocalizedTime(modification_date) data = dict(success = "true", creation_date = creation_date, modification_date = modification_date, page_views = views, hits = hits, content_type = content_type) if downloads: data['downloads'] = downloads json_data = json.dumps(data) return json_data
def choose_word(self): if self.rand_order: self.word = self.words.sample(n=1) inx = self.word.index[0] self.w_hanzi = self.word.sim[inx] self.w_pinyin = self.word.pyt[inx] self.w_pinyin_num = self.word.pyn[inx] self.w_english = self.word.en[inx] elif not self.rand_order: from utils import counter inx = counter() print(inx) # if inx > len(self.words)-1: # counter(reset=True) # inx = counter() self.w_hanzi = self.word.sim[inx] self.w_pinyin = self.word.pyt[inx] self.w_pinyin_num = self.word.pyn[inx] self.w_english = self.word.en[inx]
def plot_emoticons(self, amount=10): data = utils.counter(self.get_all_emoticons()) return utils.plot.hist( data, amount=max(amount, len(data)), sort="counter" )
def _create_friends_list(self): # PATTERN_WITH = "(?<=with )(.*?)(?=\\s[@\\(]|$)" # a name starts with a capital letter # I write full names without spaces _cappital_word = "[A-Z][A-Za-z]*" # names list is # Name Name Name and Name # which is - at least one # where the last one (if there is more than one) follows "and" # and any name which is not the first nor the last has space before and after itself _names_list = "((%s)( %s)*( and %s)*)" % (tuple([_cappital_word]) * 3) PATTERN_WITH = "(?<=with )" + _names_list PATTERN_FOR = "(?<=for )" + _names_list descriptions = '\n'.join(i.description for i in self.data) all_friends_raw = [i[0] for i in re.findall(PATTERN_WITH, descriptions)] \ + [i[0] for i in re.findall(PATTERN_FOR , descriptions)] self._friends_unsorted = '\n'.join(all_friends_raw).replace( "and", "").split() self.friends_histogram = utils.counter(self._friends_unsorted) # counter object is a list of tuples (name, amount) self.friends_histogram.sort(key=lambda x: x[1], reverse=True) self.friends = [i[0] for i in self.friends_histogram]
def create_all_words(self): self.words = sum([ P("WORDS").findall(i[MI("MESSAGE")]) for i in self.lines if i[MI("TYPE")] == MT(0) ], []) self.words_histogram = utils.counter(self.words) return self.words
def TTS(word): voices = engine.getProperty('voices') engine.setProperty('voice', voices[17].id) # setVolumn(0.25) if counter() == 0: say(word) else: engine.endLoop() engine.stop() say(word) return "ok"
def plot_emojis(self, amount=5): # a = wp.d.get_all_emojis() # b = Counter(a) # c = list(b.items()) # d = list(map(lambda x:[wp.utils.emoji.emoji_to_hex(x[0]), x[1]], c)) data = list(map( lambda x: [ utils.emoji.emoji_to_hex(x[0]), x[1] ], utils.counter(self.get_all_emojis()) )) return utils.plot.emoji_bar( data, amount=amount, sort=lambda x: x[1], # sort by the value of the word and not the amount )
def fit(self, pts, polys, recache=False): self._n = len(pts) self._surface = Surface(pts, polys) # check to see if the full K is cached dataset_hash = str(hash(pts.tostring())) + str(hash( polys.tostring())) + str(hash(self._m)) cache_path = os.path.join(self.CACHE_DIR, dataset_hash + '_geodesic_K_cache.npz') if not os.path.exists(cache_path) or recache: print('Cache not found, computing K..') K = np.vstack([ self._surface.geodesic_distance([i], m=self._m) for i in counter(range(self._n)) ]) self._K = (K + K.T) / 2.0 np.savez(cache_path, geodesic_K=self._K) else: print('Loading K from cache..') self._K = np.load(cache_path)['geodesic_K']
def count(self, counter_value): return counter(counter_value)
def train(): epoch = 200 batch_size = 1 lr = 0.0002 crop_size = 4 tar_db_a = "cameron_images.tgz" tar_db_b = "teresa_images.tgz" db_a_i = importer_tar.Importer(tar_db_a) db_b_i = importer_tar.Importer(tar_db_b) image_a_names = db_a_i.get_sorted_image_name() image_b_names = db_b_i.get_sorted_image_name() train_a_size = int(len(image_a_names) * 0.8) train_b_size = int(len(image_b_names) * 0.8) image_a_train_names = image_a_names[0:train_a_size] image_b_train_names = image_b_names[0:train_b_size] image_a_test_names = image_a_names[train_a_size:] image_b_test_names = image_b_names[train_b_size:] print("A train size:{},test size:{}".format(len(image_a_train_names), len(image_a_test_names))) print("B train size:{},test size:{}".format(len(image_b_train_names), len(image_b_test_names))) """ graph """ # models generator_a2b = partial(models.generator, scope='a2b') generator_b2a = partial(models.generator, scope='b2a') discriminator_a = partial(models.discriminator, scope='a') discriminator_b = partial(models.discriminator, scope='b') #print("Discriminator shape:{}".format()) # operations a_real = tf.placeholder(tf.float32, shape=[None, crop_size, crop_size, 3]) b_real = tf.placeholder(tf.float32, shape=[None, crop_size, crop_size, 3]) a2b_sample = tf.placeholder(tf.float32, shape=[None, crop_size, crop_size, 3]) b2a_sample = tf.placeholder(tf.float32, shape=[None, crop_size, crop_size, 3]) a2b = generator_a2b(a_real) b2a = generator_b2a(b_real) b2a2b = generator_a2b(b2a) a2b2a = generator_b2a(a2b) a_logit = discriminator_a(a_real) b2a_logit = discriminator_a(b2a) b2a_sample_logit = discriminator_a(b2a_sample) b_logit = discriminator_b(b_real) a2b_logit = discriminator_b(a2b) a2b_sample_logit = discriminator_b(a2b_sample) # losses #g_loss_a2b = tf.losses.mean_squared_error(a2b_logit, tf.ones_like(a2b_logit)) #g_loss_b2a = tf.losses.mean_squared_error(b2a_logit, tf.ones_like(b2a_logit)) g_loss_a2b = -tf.reduce_mean(a2b_logit) g_loss_b2a = -tf.reduce_mean(b22_logit) cyc_loss_a = tf.losses.absolute_difference(a_real, a2b2a) cyc_loss_b = tf.losses.absolute_difference(b_real, b2a2b) g_loss = g_loss_a2b + g_loss_b2a + cyc_loss_a * 10.0 + cyc_loss_b * 10.0 # d_loss_a_real = tf.losses.mean_squared_error(a_logit, tf.ones_like(a_logit)) # d_loss_b2a_sample = tf.losses.mean_squared_error(b2a_sample_logit, tf.zeros_like(b2a_sample_logit)) # d_loss_a = d_loss_a_real + d_loss_b2a_sample # wd_a = tf.reduce_mean(a_logit) - tf.reduce_mean(b2a_logit) wd_b = tf.reduce_mean(b_logit) - tf.reduce_mean(a2b_logit) gp_a = gradient_penalty(a_real, b2a, discriminator_a) gp_b = gradient_penalty(b_real, a2b, discriminator_b) d_loss_a = -wd_a + 10.0 * gp_a d_loss_b = -wd_b + 10.0 * gp_b # d_loss_b_real = tf.losses.mean_squared_error(b_logit, tf.ones_like(b_logit)) # d_loss_a2b_sample = tf.losses.mean_squared_error(a2b_sample_logit, tf.zeros_like(a2b_sample_logit)) # d_loss_b = d_loss_b_real + d_loss_a2b_sample # summaries g_summary = utils.summary({ g_loss_a2b: 'g_loss_a2b', g_loss_b2a: 'g_loss_b2a', cyc_loss_a: 'cyc_loss_a', cyc_loss_b: 'cyc_loss_b' }) d_summary_a = utils.summary({d_loss_a: 'd_loss_a'}) d_summary_b = utils.summary({d_loss_b: 'd_loss_b'}) # optim t_var = tf.trainable_variables() d_a_var = [var for var in t_var if 'a_discriminator' in var.name] d_b_var = [var for var in t_var if 'b_discriminator' in var.name] g_var = [ var for var in t_var if 'a2b_generator' in var.name or 'b2a_generator' in var.name ] d_a_train_op = tf.train.AdamOptimizer(lr, beta1=0.5).minimize(d_loss_a, var_list=d_a_var) d_b_train_op = tf.train.AdamOptimizer(lr, beta1=0.5).minimize(d_loss_b, var_list=d_b_var) g_train_op = tf.train.AdamOptimizer(lr, beta1=0.5).minimize(g_loss, var_list=g_var) """ train """ ''' init ''' # session config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.allow_growth = True sess = tf.Session(config=config) # counter it_cnt, update_cnt = utils.counter() ''' summary ''' summary_writer = tf.summary.FileWriter('./outputs/summaries/', sess.graph) ''' saver ''' saver = tf.train.Saver(max_to_keep=5) ''' restore ''' ckpt_dir = './outputs/checkpoints/' utils.mkdir(ckpt_dir) try: utils.load_checkpoint(ckpt_dir, sess) except: sess.run(tf.global_variables_initializer()) '''train''' try: batch_epoch = min(train_a_size, train_b_size) // batch_size max_it = epoch * batch_epoch for it in range(sess.run(it_cnt), max_it): sess.run(update_cnt) epoch = it // batch_epoch it_epoch = it % batch_epoch + 1 # prepare data a_real_ipt = db_a_i.get_image(image_a_train_names[it_epoch]) a_real_ipt = cv2.resize(a_real_ipt, (crop_size, crop_size), interpolation=cv2.INTER_CUBIC) b_real_ipt = db_b_i.get_image(image_b_train_names[it_epoch]) b_real_ipt = cv2.resize(b_real_ipt, (crop_size, crop_size), interpolation=cv2.INTER_CUBIC) a_real_ipt = [a_real_ipt] b_real_ipt = [b_real_ipt] a2b_opt, b2a_opt = sess.run([a2b, b2a], feed_dict={ a_real: a_real_ipt, b_real: b_real_ipt }) a2b_sample_ipt = a2b_opt b2a_sample_ipt = b2a_opt # train G g_summary_opt, _ = sess.run([g_summary, g_train_op], feed_dict={ a_real: a_real_ipt, b_real: b_real_ipt }) summary_writer.add_summary(g_summary_opt, it) # train D_b d_summary_b_opt, _ = sess.run([d_summary_b, d_b_train_op], feed_dict={ b_real: b_real_ipt, a2b_sample: a2b_sample_ipt }) summary_writer.add_summary(d_summary_b_opt, it) # train D_a d_summary_a_opt, _ = sess.run([d_summary_a, d_a_train_op], feed_dict={ a_real: a_real_ipt, b2a_sample: b2a_sample_ipt }) summary_writer.add_summary(d_summary_a_opt, it) # display if it % 1 == 0: print("Epoch: (%3d) (%5d/%5d)" % (epoch, it_epoch, batch_epoch)) # save if (it + 1) % 1000 == 0: save_path = saver.save( sess, '%s/Epoch_(%d)_(%dof%d).ckpt' % (ckpt_dir, epoch, it_epoch, batch_epoch)) print('Model saved in file: % s' % save_path) # sample # if (it + 1) % 100 == 0: # a_real_ipt = a_test_pool.batch() # b_real_ipt = b_test_pool.batch() # [a2b_opt, a2b2a_opt, b2a_opt, b2a2b_opt] = sess.run([a2b, a2b2a, b2a, b2a2b], feed_dict={a_real: a_real_ipt, b_real: b_real_ipt}) # sample_opt = np.concatenate((a_real_ipt, a2b_opt, a2b2a_opt, b_real_ipt, b2a_opt, b2a2b_opt), axis=0) # # save_dir = './outputs/sample_images_while_training/' + dataset # utils.mkdir(save_dir) # im.imwrite(im.immerge(sample_opt, 2, 3), '%s/Epoch_(%d)_(%dof%d).jpg' % (save_dir, epoch, it_epoch, batch_epoch)) except: save_path = saver.save( sess, '%s/Epoch_(%d)_(%dof%d).ckpt' % (ckpt_dir, epoch, it_epoch, batch_epoch)) print('Model saved in file: % s' % save_path) sess.close() raise
for var in d_var)) g_step = tf.train.RMSPropOptimizer(learning_rate=lr).minimize( g_loss, var_list=g_var) # summaries d_summary = utils.summary({wd: 'wd'}) g_summary = utils.summary({g_loss: 'g_loss'}) # sample f_sample = generator(z, training=False) """ train """ ''' init ''' # session sess = utils.session() # iteration counter it_cnt, update_cnt = utils.counter() # saver saver = tf.train.Saver(max_to_keep=5) # summary writer summary_writer = tf.summary.FileWriter('./summaries/celeba_wgan', sess.graph) ''' initialization ''' ckpt_dir = './checkpoints/celeba_wgan' utils.mkdir(ckpt_dir + '/') if not utils.load_checkpoint(ckpt_dir, sess): sess.run(tf.global_variables_initializer()) ''' train ''' try: z_ipt_sample = np.random.normal(size=[100, z_dim]) batch_epoch = len(data_pool) // (batch_size * n_critic) max_it = epoch * batch_epoch
def bootstrap_ridge(Rstim, Rresp, Pstim, Presp, alphas, nboots, chunklen, nchunks, dtype=np.single, corrmin=0.2, joined=None, singcutoff=1e-10, normalpha=False, single_alpha=False, use_corr=True, logger=logging.getLogger("ridge_corr")): """Uses ridge regression with a bootstrapped held-out set to get optimal alpha values for each response. [nchunks] random chunks of length [chunklen] will be taken from [Rstim] and [Rresp] for each regression run. [nboots] total regression runs will be performed. The best alpha value for each response will be averaged across the bootstraps to estimate the best alpha for that response. If [joined] is given, it should be a list of lists where the STRFs for all the voxels in each sublist will be given the same regularization parameter (the one that is the best on average). Parameters ---------- Rstim : array_like, shape (TR, N) Training stimuli with TR time points and N features. Each feature should be Z-scored across time. Rresp : array_like, shape (TR, M) Training responses with TR time points and M different responses (voxels, neurons, what-have-you). Each response should be Z-scored across time. Pstim : array_like, shape (TP, N) Test stimuli with TP time points and N features. Each feature should be Z-scored across time. Presp : array_like, shape (TP, M) Test responses with TP time points and M different responses. Each response should be Z-scored across time. alphas : list or array_like, shape (A,) Ridge parameters that will be tested. Should probably be log-spaced. np.logspace(0, 3, 20) works well. nboots : int The number of bootstrap samples to run. 15 to 30 works well. chunklen : int On each sample, the training data is broken into chunks of this length. This should be a few times longer than your delay/STRF. e.g. for a STRF with 3 delays, I use chunks of length 10. nchunks : int The number of training chunks held out to test ridge parameters for each bootstrap sample. The product of nchunks and chunklen is the total number of training samples held out for each sample, and this product should be about 20 percent of the total length of the training data. dtype : np.dtype All data will be cast as this dtype for computation. np.single is used by default for memory efficiency, as using np.double will thrash most machines on a big problem. If you want to do regression on complex variables, this should be changed to np.complex128. corrmin : float in [0..1] Purely for display purposes. After each alpha is tested for each bootstrap sample, the number of responses with correlation greater than this value will be printed. For long-running regressions this can give a rough sense of how well the model works before it's done. joined : None or list of array_like indices If you want the STRFs for two (or more) responses to be directly comparable, you need to ensure that the regularization parameter that they use is the same. To do that, supply a list of the response sets that should use the same ridge parameter here. For example, if you have four responses, joined could be [np.array([0,1]), np.array([2,3])], in which case responses 0 and 1 will use the same ridge parameter (which will be parameter that is best on average for those two), and likewise for responses 2 and 3. singcutoff : float The first step in ridge regression is computing the singular value decomposition (SVD) of the stimulus Rstim. If Rstim is not full rank, some singular values will be approximately equal to zero and the corresponding singular vectors will be noise. These singular values/vectors should be removed both for speed (the fewer multiplications the better!) and accuracy. Any singular values less than singcutoff will be removed. normalpha : boolean Whether ridge parameters (alphas) should be normalized by the Frobenius norm of Rstim. Good for rigorously comparing models with different numbers of parameters. single_alpha : boolean Whether to use a single alpha for all responses. Good for identification/decoding. use_corr : boolean If True, this function will use correlation as its metric of model fit. If False, this function will instead use variance explained (R-squared) as its metric of model fit. For ridge regression this can make a big difference -- highly regularized solutions will have very small norms and will thus explain very little variance while still leading to high correlations, as correlation is scale-free while R**2 is not. Returns ------- wt : array_like, shape (N, M) Regression weights for N features and M responses. corrs : array_like, shape (M,) Validation set correlations. Predicted responses for the validation set are obtained using the regression weights: pred = np.dot(Pstim, wt), and then the correlation between each predicted response and each column in Presp is found. alphas : array_like, shape (M,) The regularization coefficient (alpha) selected for each voxel using bootstrap cross-validation. bootstrap_corrs : array_like, shape (A, M, B) Correlation between predicted and actual responses on randomly held out portions of the training set, for each of A alphas, M voxels, and B bootstrap samples. valinds : array_like, shape (TH, B) The indices of the training data that were used as "validation" for each bootstrap sample. """ nresp, nvox = Rresp.shape bestalphas = np.zeros((nboots, nvox)) ## Will hold the best alphas for each voxel valinds = [] ## Will hold the indices into the validation data for each bootstrap Rcmats = [] for bi in counter(range(nboots), countevery=1, total=nboots): logger.info("Selecting held-out test set..") allinds = range(nresp) indchunks = zip(*[iter(allinds)]*chunklen) random.shuffle(indchunks) heldinds = list(itools.chain(*indchunks[:nchunks])) notheldinds = list(set(allinds)-set(heldinds)) valinds.append(heldinds) RRstim = Rstim[notheldinds,:] PRstim = Rstim[heldinds,:] RRresp = Rresp[notheldinds,:] PRresp = Rresp[heldinds,:] ## Run ridge regression using this test set Rcmat = ridge_corr(RRstim, PRstim, RRresp, PRresp, alphas, dtype=dtype, corrmin=corrmin, singcutoff=singcutoff, normalpha=normalpha, use_corr=use_corr) Rcmats.append(Rcmat) ## Find weights for each voxel try: U,S,Vh = np.linalg.svd(Rstim, full_matrices=False) except np.linalg.LinAlgError, e: logger.info("NORMAL SVD FAILED, trying more robust dgesvd..") from text.regression.svd_dgesvd import svd_dgesvd U,S,Vh = svd_dgesvd(Rstim, full_matrices=False)
def bootstrap_ridge(Rstim, Rresp, Pstim, Presp, alphas, nboots, chunklen, nchunks, corrmin=0.2, joined=None, singcutoff=1e-10, normalpha=False, single_alpha=False, use_corr=True, logger=ridge_logger): """Uses ridge regression with a bootstrapped held-out set to get optimal alpha values for each response. [nchunks] random chunks of length [chunklen] will be taken from [Rstim] and [Rresp] for each regression run. [nboots] total regression runs will be performed. The best alpha value for each response will be averaged across the bootstraps to estimate the best alpha for that response. If [joined] is given, it should be a list of lists where the STRFs for all the voxels in each sublist will be given the same regularization parameter (the one that is the best on average). Parameters ---------- Rstim : array_like, shape (TR, N) Training stimuli with TR time points and N features. Each feature should be Z-scored across time. Rresp : array_like, shape (TR, M) Training responses with TR time points and M different responses (voxels, neurons, what-have-you). Each response should be Z-scored across time. Pstim : array_like, shape (TP, N) Test stimuli with TP time points and N features. Each feature should be Z-scored across time. Presp : array_like, shape (TP, M) Test responses with TP time points and M different responses. Each response should be Z-scored across time. alphas : list or array_like, shape (A,) Ridge parameters that will be tested. Should probably be log-spaced. np.logspace(0, 3, 20) works well. nboots : int The number of bootstrap samples to run. 15 to 30 works well. chunklen : int On each sample, the training data is broken into chunks of this length. This should be a few times longer than your delay/STRF. e.g. for a STRF with 3 delays, I use chunks of length 10. nchunks : int The number of training chunks held out to test ridge parameters for each bootstrap sample. The product of nchunks and chunklen is the total number of training samples held out for each sample, and this product should be about 20 percent of the total length of the training data. corrmin : float in [0..1] Purely for display purposes. After each alpha is tested for each bootstrap sample, the number of responses with correlation greater than this value will be printed. For long-running regressions this can give a rough sense of how well the model works before it's done. joined : None or list of array_like indices If you want the STRFs for two (or more) responses to be directly comparable, you need to ensure that the regularization parameter that they use is the same. To do that, supply a list of the response sets that should use the same ridge parameter here. For example, if you have four responses, joined could be [np.array([0,1]), np.array([2,3])], in which case responses 0 and 1 will use the same ridge parameter (which will be parameter that is best on average for those two), and likewise for responses 2 and 3. singcutoff : float The first step in ridge regression is computing the singular value decomposition (SVD) of the stimulus Rstim. If Rstim is not full rank, some singular values will be approximately equal to zero and the corresponding singular vectors will be noise. These singular values/vectors should be removed both for speed (the fewer multiplications the better!) and accuracy. Any singular values less than singcutoff will be removed. normalpha : boolean Whether ridge parameters (alphas) should be normalized by the largest singular value (LSV) norm of Rstim. Good for rigorously comparing models with different numbers of parameters. single_alpha : boolean Whether to use a single alpha for all responses. Good for identification/decoding. use_corr : boolean If True, this function will use correlation as its metric of model fit. If False, this function will instead use variance explained (R-squared) as its metric of model fit. For ridge regression this can make a big difference -- highly regularized solutions will have very small norms and will thus explain very little variance while still leading to high correlations, as correlation is scale-free while R**2 is not. Returns ------- wt : array_like, shape (N, M) Regression weights for N features and M responses. corrs : array_like, shape (M,) Validation set correlations. Predicted responses for the validation set are obtained using the regression weights: pred = np.dot(Pstim, wt), and then the correlation between each predicted response and each column in Presp is found. alphas : array_like, shape (M,) The regularization coefficient (alpha) selected for each voxel using bootstrap cross-validation. bootstrap_corrs : array_like, shape (A, M, B) Correlation between predicted and actual responses on randomly held out portions of the training set, for each of A alphas, M voxels, and B bootstrap samples. valinds : array_like, shape (TH, B) The indices of the training data that were used as "validation" for each bootstrap sample. """ nresp, nvox = Rresp.shape valinds = [] # Will hold the indices into the validation data for each bootstrap Rcmats = [] for bi in counter(range(nboots), countevery=1, total=nboots): logger.info("Selecting held-out test set..") allinds = range(nresp) indchunks = zip(*[iter(allinds)]*chunklen) random.shuffle(indchunks) heldinds = list(itools.chain(*indchunks[:nchunks])) notheldinds = list(set(allinds)-set(heldinds)) valinds.append(heldinds) RRstim = Rstim[notheldinds,:] PRstim = Rstim[heldinds,:] RRresp = Rresp[notheldinds,:] PRresp = Rresp[heldinds,:] # Run ridge regression using this test set Rcmat = ridge_corr(RRstim, PRstim, RRresp, PRresp, alphas, corrmin=corrmin, singcutoff=singcutoff, normalpha=normalpha, use_corr=use_corr, logger=logger) Rcmats.append(Rcmat) # Find best alphas if nboots>0: allRcorrs = np.dstack(Rcmats) else: allRcorrs = None if not single_alpha: if nboots==0: raise ValueError("You must run at least one cross-validation step to assign " "different alphas to each response.") logger.info("Finding best alpha for each voxel..") if joined is None: # Find best alpha for each voxel meanbootcorrs = allRcorrs.mean(2) bestalphainds = np.argmax(meanbootcorrs, 0) valphas = alphas[bestalphainds] else: # Find best alpha for each group of voxels valphas = np.zeros((nvox,)) for jl in joined: # Mean across voxels in the set, then mean across bootstraps jcorrs = allRcorrs[:,jl,:].mean(1).mean(1) bestalpha = np.argmax(jcorrs) valphas[jl] = alphas[bestalpha] else: logger.info("Finding single best alpha..") if nboots==0: if len(alphas)==1: bestalphaind = 0 bestalpha = alphas[0] else: raise ValueError("You must run at least one cross-validation step " "to choose best overall alpha, or only supply one" "possible alpha value.") else: meanbootcorr = allRcorrs.mean(2).mean(1) bestalphaind = np.argmax(meanbootcorr) bestalpha = alphas[bestalphaind] valphas = np.array([bestalpha]*nvox) logger.info("Best alpha = %0.3f"%bestalpha) # Find weights logger.info("Computing weights for each response using entire training set..") wt = ridge(Rstim, Rresp, valphas, singcutoff=singcutoff, normalpha=normalpha) # Predict responses on prediction set logger.info("Predicting responses for predictions set..") pred = np.dot(Pstim, wt) # Find prediction correlations nnpred = np.nan_to_num(pred) if use_corr: corrs = np.nan_to_num(np.array([np.corrcoef(Presp[:,ii], nnpred[:,ii].ravel())[0,1] for ii in range(Presp.shape[1])])) else: resvar = (Presp-pred).var(0) Rsqs = 1 - (resvar / Presp.var(0)) corrs = np.sqrt(np.abs(Rsqs)) * np.sign(Rsqs) return wt, corrs, valphas, allRcorrs, valinds
def get_features_ratio(self): ratio = {} ratio_description = {} # message ratio ratio_description["message"] = "[amount of messages from] user0 / user1" ratio["message"] = numpy.divide(*self.user_message_amount) # media ratio ratio_description["media"] = "[amount of media from] user0 / user1" ratio["media"] = numpy.divide(*self.user_media_amount) # words ratio ratio_description["words"] = "[amount of words sent by] user0 / user1" ratio["words"] = numpy.divide(*self.user_words_amount) # long_messages ratio ratio_description["long messages"] = "[amount of messages with more than 10 words from] user0 / user1" _long_messages = self.get_messages(lambda x:len(x[MI('M')].split())>=10) _long_messages_amount_per_user = list(map( # iterate all the users # and count how many long messages each of them has lambda u: len(list(filter( # iterate all the long messages # and filter out only messages by the user lambda m: m[MI('U')] == u, _long_messages ))), self.users )) ratio["long messages"] = numpy.divide(*_long_messages_amount_per_user) del _long_messages del _long_messages_amount_per_user # messages with h ratio_description["messages with h"] = "[amount of messages with h by] user0 / user1" _messages_with_h = self.get_messages(P("H")) _messages_with_h_per_user = list(map( lambda u: len(list(filter( lambda m: m[MI('U')] == u, _messages_with_h ))), self.users )) ratio["messages with h"] = numpy.divide(*_messages_with_h_per_user) del _messages_with_h del _messages_with_h_per_user ratio_description["conversations"] = "[amount of conversations started by] user0 / user1" _conversations_started = utils.counter([ c[0][MI("USER")] for c in self.chats ]) ratio["conversations"] = numpy.divide(*[ # get the conversation started by the user next(filter( lambda x: x[0] == u, _conversations_started # get the amount of conversations ))[1] for u in # keep the order of the users self.users ]) return ratio, ratio_description
def preprocess2d(mode='train'): """ Split up all 3d volumes into separate 2d slice images and store as separate files. Volumes are fetched from directory specified in config. """ ## Generator to generate indices for slices. numgen = counter() dirpaths = [ config[f"dst2d_labels_liver_path"], config[f"dst2d_labels_lesion_path"], config[f"dst2d_slices_path"], ] for p in dirpaths: if not os.path.exists(p): os.makedirs(p) ## Processing start = time() for f in tqdm(os.listdir(config[f"{config['mode']}_volumes_path"]), desc=f"{config['mode']} 2d processing"): ## Get Volume matrices ct = sitk.ReadImage(os.path.join(config[f"{config['mode']}_volumes_path"], f), sitk.sitkInt16) ct_array = sitk.GetArrayFromImage(ct) seg = sitk.ReadImage(os.path.join(config[f"{config['mode']}_labels_path"], f.replace('volume', 'segmentation')), sitk.sitkUInt8) seg_array = sitk.GetArrayFromImage(seg) ## TODO: Change if looking for tumors ## Make all cancer labels to liverlabels (for liver segmentation, not tumor segmentation) liver_array = np.zeros_like(seg_array) lesion_array = np.zeros_like(seg_array) liver_array[seg_array >= 1] = 1 lesion_array[seg_array > 1] = 1 ## Clip upper and lower values of CT images ct_array[ct_array > config["upper"]] = config["upper"] ct_array[ct_array < config["lower"]] = config["lower"] ## Pick out relevant slices z = np.any(liver_array, axis=(1, 2)) start_slice, end_slice = np.where(z)[0][[0, -1]] start_slice = max(0, start_slice - config["expand_slice"]) end_slice = min(liver_array.shape[0] - 1, end_slice + config["expand_slice"]) ct_array = ct_array[start_slice:end_slice, :, :] liver_array = liver_array[start_slice:end_slice, :, :] lesion_array = lesion_array[start_slice:end_slice, :, :] if ct_array.shape[0] != liver_array.shape[0]: breakpoint() ## Store each slice as .mat numpy array for i in range(ct_array.shape[0]): idx = next(numgen) slice_filename = "slice_{:05}".format(idx) segmentation_filename = "segmentation_{:05}".format(idx) np.save(os.path.join(config[f"dst2d_slices_path"], slice_filename), ct_array[i, ...]) np.save(os.path.join(config[f"dst2d_labels_liver_path"], segmentation_filename), liver_array[i, ...]) np.save(os.path.join(config[f"dst2d_labels_lesion_path"], segmentation_filename), lesion_array[i, ...]) print("Finished {} preprocessing in: {:02}s".format(config['mode'], time() - start))
def bootstrap_ridge(Rstim, Rresp, Pstim, Presp, alphas, nboots, chunklen, nchunks, corrmin=0.2, joined=None, singcutoff=1e-10, normalpha=False, single_alpha=False, use_corr=True, logger=ridge_logger): """Uses ridge regression with a bootstrapped held-out set to get optimal alpha values for each response. [nchunks] random chunks of length [chunklen] will be taken from [Rstim] and [Rresp] for each regression run. [nboots] total regression runs will be performed. The best alpha value for each response will be averaged across the bootstraps to estimate the best alpha for that response. If [joined] is given, it should be a list of lists where the STRFs for all the voxels in each sublist will be given the same regularization parameter (the one that is the best on average). Parameters ---------- Rstim : array_like, shape (TR, N) Training stimuli with TR time points and N features. Each feature should be Z-scored across time. Rresp : array_like, shape (TR, M) Training responses with TR time points and M different responses (voxels, neurons, what-have-you). Each response should be Z-scored across time. Pstim : array_like, shape (TP, N) Test stimuli with TP time points and N features. Each feature should be Z-scored across time. Presp : array_like, shape (TP, M) Test responses with TP time points and M different responses. Each response should be Z-scored across time. alphas : list or array_like, shape (A,) Ridge parameters that will be tested. Should probably be log-spaced. np.logspace(0, 3, 20) works well. nboots : int The number of bootstrap samples to run. 15 to 30 works well. chunklen : int On each sample, the training data is broken into chunks of this length. This should be a few times longer than your delay/STRF. e.g. for a STRF with 3 delays, I use chunks of length 10. nchunks : int The number of training chunks held out to test ridge parameters for each bootstrap sample. The product of nchunks and chunklen is the total number of training samples held out for each sample, and this product should be about 20 percent of the total length of the training data. corrmin : float in [0..1] Purely for display purposes. After each alpha is tested for each bootstrap sample, the number of responses with correlation greater than this value will be printed. For long-running regressions this can give a rough sense of how well the model works before it's done. joined : None or list of array_like indices If you want the STRFs for two (or more) responses to be directly comparable, you need to ensure that the regularization parameter that they use is the same. To do that, supply a list of the response sets that should use the same ridge parameter here. For example, if you have four responses, joined could be [np.array([0,1]), np.array([2,3])], in which case responses 0 and 1 will use the same ridge parameter (which will be parameter that is best on average for those two), and likewise for responses 2 and 3. singcutoff : float The first step in ridge regression is computing the singular value decomposition (SVD) of the stimulus Rstim. If Rstim is not full rank, some singular values will be approximately equal to zero and the corresponding singular vectors will be noise. These singular values/vectors should be removed both for speed (the fewer multiplications the better!) and accuracy. Any singular values less than singcutoff will be removed. normalpha : boolean Whether ridge parameters (alphas) should be normalized by the largest singular value (LSV) norm of Rstim. Good for rigorously comparing models with different numbers of parameters. single_alpha : boolean Whether to use a single alpha for all responses. Good for identification/decoding. use_corr : boolean If True, this function will use correlation as its metric of model fit. If False, this function will instead use variance explained (R-squared) as its metric of model fit. For ridge regression this can make a big difference -- highly regularized solutions will have very small norms and will thus explain very little variance while still leading to high correlations, as correlation is scale-free while R**2 is not. Returns ------- wt : array_like, shape (N, M) Regression weights for N features and M responses. corrs : array_like, shape (M,) Validation set correlations. Predicted responses for the validation set are obtained using the regression weights: pred = np.dot(Pstim, wt), and then the correlation between each predicted response and each column in Presp is found. alphas : array_like, shape (M,) The regularization coefficient (alpha) selected for each voxel using bootstrap cross-validation. bootstrap_corrs : array_like, shape (A, M, B) Correlation between predicted and actual responses on randomly held out portions of the training set, for each of A alphas, M voxels, and B bootstrap samples. valinds : array_like, shape (TH, B) The indices of the training data that were used as "validation" for each bootstrap sample. """ nresp, nvox = Rresp.shape valinds = [ ] # Will hold the indices into the validation data for each bootstrap Rcmats = [] for bi in counter(range(nboots), countevery=1, total=nboots): logger.info("Selecting held-out test set..") allinds = range(nresp) indchunks = zip(*[iter(allinds)] * chunklen) random.shuffle(indchunks) heldinds = list(itools.chain(*indchunks[:nchunks])) notheldinds = list(set(allinds) - set(heldinds)) valinds.append(heldinds) RRstim = Rstim[notheldinds, :] PRstim = Rstim[heldinds, :] RRresp = Rresp[notheldinds, :] PRresp = Rresp[heldinds, :] # Run ridge regression using this test set Rcmat = ridge_corr(RRstim, PRstim, RRresp, PRresp, alphas, corrmin=corrmin, singcutoff=singcutoff, normalpha=normalpha, use_corr=use_corr, logger=logger) Rcmats.append(Rcmat) # Find best alphas if nboots > 0: allRcorrs = np.dstack(Rcmats) else: allRcorrs = None if not single_alpha: if nboots == 0: raise ValueError( "You must run at least one cross-validation step to assign " "different alphas to each response.") logger.info("Finding best alpha for each voxel..") if joined is None: # Find best alpha for each voxel meanbootcorrs = allRcorrs.mean(2) bestalphainds = np.argmax(meanbootcorrs, 0) valphas = alphas[bestalphainds] else: # Find best alpha for each group of voxels valphas = np.zeros((nvox, )) for jl in joined: # Mean across voxels in the set, then mean across bootstraps jcorrs = allRcorrs[:, jl, :].mean(1).mean(1) bestalpha = np.argmax(jcorrs) valphas[jl] = alphas[bestalpha] else: logger.info("Finding single best alpha..") if nboots == 0: if len(alphas) == 1: bestalphaind = 0 bestalpha = alphas[0] else: raise ValueError( "You must run at least one cross-validation step " "to choose best overall alpha, or only supply one" "possible alpha value.") else: meanbootcorr = allRcorrs.mean(2).mean(1) bestalphaind = np.argmax(meanbootcorr) bestalpha = alphas[bestalphaind] valphas = np.array([bestalpha] * nvox) logger.info("Best alpha = %0.3f" % bestalpha) # Find weights logger.info( "Computing weights for each response using entire training set..") wt = ridge(Rstim, Rresp, valphas, singcutoff=singcutoff, normalpha=normalpha) # Predict responses on prediction set logger.info("Predicting responses for predictions set..") pred = np.dot(Pstim, wt) # Find prediction correlations nnpred = np.nan_to_num(pred) if use_corr: corrs = np.nan_to_num( np.array([ np.corrcoef(Presp[:, ii], nnpred[:, ii].ravel())[0, 1] for ii in range(Presp.shape[1]) ])) else: resvar = (Presp - pred).var(0) Rsqs = 1 - (resvar / Presp.var(0)) corrs = np.sqrt(np.abs(Rsqs)) * np.sign(Rsqs) return wt, corrs, valphas, allRcorrs, valinds
def main(epoch, batch_size, lr, z_dim, bottle_dim, i_c, alpha, n_critic, gpu_id, data_pool): with tf.device('/gpu:%d' % gpu_id): #Placing the ops under devices generator = models.generator #Generator Object discriminator = models.discriminator_wgan_gp #Discriminator Object # inputs Placeholders real = tf.placeholder(tf.float32, shape=[None, 28, 28, 1]) z = tf.placeholder(tf.float32, shape=[None, z_dim]) # generate fake data with the generator fake = generator(z, reuse=False) # Obtaining scores , means and stds for real and fake data from the discriminator r_logit, r_mus, r_sigmas = discriminator(real, reuse=False, gen_train=False, bottleneck_dim=bottle_dim) f_logit, f_mus, f_sigmas = discriminator(fake, gen_train=False, bottleneck_dim=bottle_dim) #Obtaining wasserstein loss and gradient penalty losses to train the discriminator wasserstein_d = losses.wgan_loss(r_logit, f_logit) gp = losses.gradient_penalty(real, fake, discriminator) #We obtain the bottleneck loss in the discriminator #Inputs to this function are bottleneck layer mus and stds for both real and fake data. i_c is the #the information constriant or upperbound. This is an important paramters bottleneck_loss=losses._bottleneck_loss(real_mus=r_mus, fake_mus=f_mus,\ real_sigmas=r_sigmas,fake_sigmas=f_sigmas,i_c=i_c) #This used in lagrangian multiplier optimization. This is paramters also get updated adaptivly. #To read more about duel gradient desenet in deep learning please read - https://medium.com/@jonathan_hui/rl-dual-gradient-descent-fac524c1f049 #Initialize with the zero beta = tf.Variable(tf.zeros([]), name="beta") #Combined both losses (10 is the default hyper paramters given by the paper # - https://arxiv.org/pdf/1704.00028.pdf ) d_loss = -wasserstein_d + gp * 10.0 + beta * bottleneck_loss #We said b also should adaptively get updated. Here we maximize the beta paramters with follwoing function #Please refer to the VDB paper's equation (9) understand more about the update beta_new = tf.maximum(0.0, beta + alpha * bottleneck_loss) #This is the main difference from the pytoch implementation. In tensorlfow we have a static graph. S # to update the beta with above menitoned function we have to use tf.assign() assign_op = tf.assign(beta, beta_new) #beta.assign(beta_new) #This is the generator loss #As described in the paper we have a simple loss to the generator which uses mean scores from #the generated samples f_logit_gen, f_mus_gen, f_sigmas_gen = discriminator( fake, gen_train=True, bottleneck_dim=bottle_dim) g_loss = -tf.reduce_mean(f_logit_gen) #Assigning two optimizers to train both Generator and the Discriminator d_var = tf.trainable_variables('discriminator') g_var = tf.trainable_variables('generator') d_step = tf.train.RMSPropOptimizer(learning_rate=lr).minimize( d_loss, var_list=d_var) g_step = tf.train.RMSPropOptimizer(learning_rate=lr).minimize( g_loss, var_list=g_var) # Tensorbored summaries for plot losses wd = wasserstein_d d_summary = utils.summary({wd: 'wd', gp: 'gp'}) g_summary = utils.summary({g_loss: 'g_loss'}) beta_summary = utils.b_summary(beta) #beta_summary = utils.summary({beta: 'beta'}) #sess= tf.Session() config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) with tf.Session(config=config) as sess: # iteration counter it_cnt, update_cnt = utils.counter() # saver saver = tf.train.Saver( max_to_keep=5 ) #Use to save both generator and discriminator paramters # summary writer summary_writer = tf.summary.FileWriter('./summaries/mnist_wgan_gp', sess.graph) ''' Checking for previuosly trained checkpints''' ckpt_dir = './checkpoints/mnist_wgan_gp' utils.mkdir(ckpt_dir + '/') if not utils.load_checkpoint(ckpt_dir, sess): sess.run(tf.global_variables_initializer()) #Starting the training loop batch_epoch = len(data_pool) // (batch_size * n_critic) max_it = epoch * batch_epoch for it in range(sess.run(it_cnt), max_it): sess.run(update_cnt) # which epoch epoch = it // batch_epoch it_epoch = it % batch_epoch + 1 # train D for i in range( n_critic ): #Fist we train the discriminator for few iterations (Here I used only 1) # batch data real_ipt = data_pool.batch('img') #Read data batch z_ipt = np.random.normal(size=[batch_size, z_dim]) #Sample nosice input d_summary_opt, _ = sess.run([d_summary, d_step], feed_dict={ real: real_ipt, z: z_ipt }) #Discriminator Gradient Update beta_summary_opt = sess.run(beta_summary) #_ = sess.run([d_step], feed_dict={real: real_ipt, z: z_ipt}) sess.run([assign_op], feed_dict={ real: real_ipt, z: z_ipt }) #Adpatively update the beta parameter summary_writer.add_summary(d_summary_opt, it) summary_writer.add_summary(beta_summary_opt, it) # train the geenrator (Here we have a simple generator as in normal Wgan) z_ipt = np.random.normal(size=[batch_size, z_dim]) g_summary_opt, _ = sess.run([g_summary, g_step], feed_dict={z: z_ipt}) #_ = sess.run([g_step], feed_dict={z: z_ipt}) summary_writer.add_summary(g_summary_opt, it) # display training progress if it % 100 == 0: print("Epoch: (%3d) (%5d/%5d)" % (epoch, it_epoch, batch_epoch)) # saving the checpoints after every 1000 interation if (it + 1) % 1000 == 0: save_path = saver.save( sess, '%s/Epoch_(%d)_(%dof%d).ckpt' % (ckpt_dir, epoch, it_epoch, batch_epoch)) print('Model saved in file: % s' % save_path) #This is to save the image generation during the trainign as tiles if (it + 1) % 100 == 0: z_input_sample = np.random.normal(size=[100, z_dim]) #Noise samples f_sample = generator(z) f_sample_opt = sess.run(f_sample, feed_dict={z: z_input_sample}) save_dir = './sample_images_while_training/mnist_wgan_gp' utils.mkdir(save_dir + '/') utils.imwrite( utils.immerge(f_sample_opt, 10, 10), '%s/Epoch_(%d)_(%dof%d).jpg' % (save_dir, epoch, it_epoch, batch_epoch))
return decision, (person, persons[1]) else: print( f"{person.name} perdeu a vez por passar uma decisão inválida!") return decision, (person, ) @persons.event def on_decision(decision): print("Decisão escolhida:", decision.name, end="\n\n") sleep(1) @persons.event def on_dead(person): print(f"{person.name} morreu! Fim de jogo!") while persons.alive: system("cls || clear") # Clear terminal in Windows/Linux person = persons[counter()] print(f"Vez de {person.name} - {person.stats}!", end="\n\n") decision, arguments = person.make_decision(decisions) decision.action(*arguments) sleep(3)
def train(): epoch = 200 batch_size = 1 lr = 0.0002 crop_size = 128 load_size = 128 tar_db_a = "cameron_images.tgz" tar_db_b = "teresa_images.tgz" db_a_i = importer_tar.Importer(tar_db_a) db_b_i = importer_tar.Importer(tar_db_b) image_a_names = db_a_i.get_sorted_image_name() image_b_names = db_b_i.get_sorted_image_name() train_a_size = int(len(image_a_names) * 0.8) train_b_size = int(len(image_b_names) * 0.8) image_a_train_names = image_a_names[0:train_a_size] image_b_train_names = image_b_names[0:train_b_size] image_a_test_names = image_a_names[train_a_size:] image_b_test_names = image_b_names[train_b_size:] print("A train size:{},test size:{}".format(len(image_a_train_names), len(image_a_test_names))) print("B train size:{},test size:{}".format(len(image_b_train_names), len(image_b_test_names))) """ graph """ # models generator_a2b = partial(models.generator, scope='a2b') generator_b2a = partial(models.generator, scope='b2a') discriminator_a = partial(models.discriminator, scope='a') discriminator_b = partial(models.discriminator, scope='b') # operations a_real_in = tf.placeholder(tf.float32, shape=[None, load_size, load_size, 3], name="a_real") b_real_in = tf.placeholder(tf.float32, shape=[None, load_size, load_size, 3], name="b_real") a_real = utils.preprocess_image(a_real_in, crop_size=crop_size) b_real = utils.preprocess_image(b_real_in, crop_size=crop_size) a2b = generator_a2b(a_real) b2a = generator_b2a(b_real) b2a2b = generator_a2b(b2a) a2b2a = generator_b2a(a2b) a_logit = discriminator_a(a_real) b2a_logit = discriminator_a(b2a) b_logit = discriminator_b(b_real) a2b_logit = discriminator_b(a2b) # losses g_loss_a2b = -tf.reduce_mean(a2b_logit) g_loss_b2a = -tf.reduce_mean(b2a_logit) cyc_loss_a = tf.losses.absolute_difference(a_real, a2b2a) cyc_loss_b = tf.losses.absolute_difference(b_real, b2a2b) g_loss = g_loss_a2b + g_loss_b2a + cyc_loss_a * 10.0 + cyc_loss_b * 10.0 wd_a = tf.reduce_mean(a_logit) - tf.reduce_mean(b2a_logit) wd_b = tf.reduce_mean(b_logit) - tf.reduce_mean(a2b_logit) gp_a = gradient_penalty(a_real, b2a, discriminator_a) gp_b = gradient_penalty(b_real, a2b, discriminator_b) d_loss_a = -wd_a + 10.0 * gp_a d_loss_b = -wd_b + 10.0 * gp_b # summaries utils.summary({ g_loss_a2b: 'g_loss_a2b', g_loss_b2a: 'g_loss_b2a', cyc_loss_a: 'cyc_loss_a', cyc_loss_b: 'cyc_loss_b' }) utils.summary({d_loss_a: 'd_loss_a'}) utils.summary({d_loss_b: 'd_loss_b'}) for variable in slim.get_model_variables(): tf.summary.histogram(variable.op.name, variable) merged = tf.summary.merge_all() im1_op = tf.summary.image("real_a", a_real_in) im2_op = tf.summary.image("a2b", a2b) im3_op = tf.summary.image("b2a2b", b2a2b) im4_op = tf.summary.image("real_b", b_real_in) im5_op = tf.summary.image("b2a", b2a) im6_op = tf.summary.image("b2a2b", b2a2b) # optim t_var = tf.trainable_variables() d_a_var = [var for var in t_var if 'a_discriminator' in var.name] d_b_var = [var for var in t_var if 'b_discriminator' in var.name] g_var = [ var for var in t_var if 'a2b_generator' in var.name or 'b2a_generator' in var.name ] d_a_train_op = tf.train.AdamOptimizer(lr, beta1=0.5).minimize(d_loss_a, var_list=d_a_var) d_b_train_op = tf.train.AdamOptimizer(lr, beta1=0.5).minimize(d_loss_b, var_list=d_b_var) g_train_op = tf.train.AdamOptimizer(lr, beta1=0.5).minimize(g_loss, var_list=g_var) """ train """ ''' init ''' # session config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.allow_growth = True sess = tf.Session(config=config) # counter it_cnt, update_cnt = utils.counter() ''' summary ''' summary_writer = tf.summary.FileWriter('./outputs/summaries/', sess.graph) ''' saver ''' saver = tf.train.Saver(max_to_keep=5) ''' restore ''' ckpt_dir = './outputs/checkpoints/' utils.mkdir(ckpt_dir) try: utils.load_checkpoint(ckpt_dir, sess) except: sess.run(tf.global_variables_initializer()) '''train''' try: batch_epoch = min(train_a_size, train_b_size) // batch_size max_it = epoch * batch_epoch for it in range(sess.run(it_cnt), max_it): sess.run(update_cnt) epoch = it // batch_epoch it_epoch = it % batch_epoch + 1 # read data a_real_np = cv2.resize( db_a_i.get_image(image_a_train_names[it_epoch % batch_epoch]), (load_size, load_size)) b_real_np = cv2.resize( db_b_i.get_image(image_b_train_names[it_epoch % batch_epoch]), (load_size, load_size)) # train G sess.run(g_train_op, feed_dict={ a_real_in: [a_real_np], b_real_in: [b_real_np] }) # train discriminator sess.run([d_a_train_op, d_b_train_op], feed_dict={ a_real_in: [a_real_np], b_real_in: [b_real_np] }) # display if it % 100 == 0: # make summary summary = sess.run(merged, feed_dict={ a_real_in: [a_real_np], b_real_in: [b_real_np] }) summary_writer.add_summary(summary, it) print("Epoch: (%3d) (%5d/%5d)" % (epoch, it_epoch, batch_epoch)) # save if (it + 1) % 1000 == 0: save_path = saver.save( sess, '{}/epoch_{}_{}.ckpt'.format(ckpt_dir, epoch, it_epoch)) print('###Model saved in file: {}'.format(save_path)) # sample if (it + 1) % 1000 == 0: a_test_index = int( np.random.uniform(high=len(image_a_test_names))) b_test_index = int( np.random.uniform(high=len(image_b_test_names))) a_real_np = cv2.resize( db_a_i.get_image(image_a_test_names[a_test_index]), (load_size, load_size)) b_real_np = cv2.resize( db_b_i.get_image(image_b_test_names[b_test_index]), (load_size, load_size)) [a_opt, a2b_opt, a2b2a_opt, b_opt, b2a_opt, b2a2b_opt ] = sess.run([a_real, a2b, a2b2a, b_real, b2a, b2a2b], feed_dict={ a_real_in: [a_real_np], b_real_in: [b_real_np] }) sample_opt = np.concatenate( (a_opt, a2b_opt, a2b2a_opt, b_opt, b2a_opt, b2a2b_opt), axis=0) [im1_sum,im2_sum,im3_sum,im4_sum,im5_sum,im6_sum] = \ sess.run([im1_op,im2_op,im3_op,im4_op,im5_op,im6_op], feed_dict={a_real_in: [a_real_np], b_real_in: [b_real_np]}) summary_writer.add_summary(im1_sum, it) summary_writer.add_summary(im2_sum, it) summary_writer.add_summary(im3_sum, it) summary_writer.add_summary(im4_sum, it) summary_writer.add_summary(im5_sum, it) summary_writer.add_summary(im6_sum, it) save_dir = './outputs/sample_images_while_training/' utils.mkdir(save_dir) im.imwrite( im.immerge(sample_opt, 2, 3), '{}/epoch_{}_it_{}.jpg'.format(save_dir, epoch, it_epoch)) except: raise finally: save_path = saver.save( sess, '{}/epoch_{}_{}.ckpt'.format(ckpt_dir, epoch, it_epoch)) print('###Model saved in file: {}'.format(save_path)) sess.close()
def compute_P(self): Winds = self._Winds nonWinds = self._nonWinds n = self._M.shape[0] # pull out part of M for unselected points M_aa = self._M[nonWinds,:][:,nonWinds].tocsc() # pull out part of M that crosses selected and unselected points M_ab = self._M[Winds,:][:,nonWinds] if self._nnz_row is not None: if self._sparse_direction == 'cols': self._threshold = self._nnz_row * (n-self._l) // self._l else: self._threshold = self._nnz_row try: from sksparse.cholmod import cholesky solve_method = 'cholmod' except ImportError: solve_method = 'spsolve' # compute Pprime, part of the dense interpolation matrix if self._threshold is None: if solve_method == 'spsolve': Pprime = sparse.linalg.spsolve(M_aa, -M_ab.T) elif solve_method == 'cholmod': Pprime = cholesky(M_aa).solve_A(-M_ab.T) # compute P, the full dense interpolation matrix P = np.zeros((n, self._l)) P[nonWinds,:] = Pprime.todense() P[Winds,:] = np.eye(self._l) Pnnz = n * self._l if self._soft_p is not None: # don't force P to be exactly identity for known points, # allow it to fudge a little print("Softening P..") M_bb = self._M[Winds,:][:,Winds] soft_eye = sparse.eye(self._l) * self._soft_p to_invert = (M_bb + soft_eye + M_ab.dot(Pprime)).todense() soft_factor = np.linalg.inv(to_invert) * self._soft_p P = P.dot(soft_factor).A else: # Compute the sparse bha if solve_method == 'cholmod': chol_M_aa = cholesky(M_aa) if self._sparse_direction == 'rows': thresh = THR_ROWS(k=self._threshold) Prows = np.empty(self._threshold*(n-self._l)+self._l, dtype=int) Pcols = np.empty(self._threshold*(n-self._l)+self._l, dtype=int) Pvals = np.empty(self._threshold*(n-self._l)+self._l) else: thresh = THR(k=self._threshold) Prows = np.empty(self._threshold*self._l+self._l, dtype=int) Pcols = np.empty(self._threshold*self._l+self._l, dtype=int) Pvals = np.empty(self._threshold*self._l+self._l) chunk_size = 64 # min(self._l // self._njobs, 64) chunks = self._l // chunk_size + ((self._l % chunk_size) > 0) for chunk in counter(range(chunks)): start = chunk*chunk_size end = min(((chunk+1)*chunk_size, self._l)) if solve_method == 'spsolve': sol = sparse.linalg.spsolve(M_aa, -M_ab.T[:, start:end].toarray()) elif solve_method == 'cholmod': sol = chol_M_aa.solve_A(-M_ab.T[:, start:end].toarray()) if self._sparse_direction == 'rows': thresh.fit(sol) else: if self._fit_group: l_i = 0 for l in range(start,end): thresh.fit_partition(sol[:, l_i]) Prows[l*self._threshold:(l+1)*self._threshold] = nonWinds[thresh._idxs] Pvals[l*self._threshold:(l+1)*self._threshold] = thresh._vals l_i += 1 else: l_i = 0 for l in range(start,end): thresh.fit(sol[:, l_i]) Prows[l*self._threshold:(l+1)*self._threshold] = nonWinds[thresh._idxs] Pvals[l*self._threshold:(l+1)*self._threshold] = thresh._vals l_i += 1 if self._sparse_direction == 'rows': cols, vals = thresh.get_best_k() Prows[:(n-self._l)*self._threshold] = np.repeat(nonWinds[np.arange(n-self._l)],self._threshold) Pcols[:(n-self._l)*self._threshold] = cols Pvals[:(n-self._l)*self._threshold] = vals lastnonWindElement = (n-self._l)*self._threshold else: Pcols[:self._l*self._threshold] = np.repeat(np.arange(self._l),self._threshold) lastnonWindElement = self._l*self._threshold # add the identity for indices in W Prows[lastnonWindElement:] = Winds Pcols[lastnonWindElement:] = np.arange(self._l) Pvals[lastnonWindElement:] = 1.0 P = sparse.csr_matrix((Pvals,(Prows, Pcols)), shape=(n,self._l)) P.eliminate_zeros() Pnnz = P.nnz # save values self._nnz = Pnnz self._P = P