def combine(self, cases): strict = [strategy.ordered_signatures,strategy.safe_methods] loose = [strategy.ordered_signatures,strategy.all_methods] primary_names = ['primary%d' % order for order in self.order_when] around_names = ['around%d' % order for order in self.order_around] cases = strategy.separate_qualifiers( cases, before = loose, after =loose, **dict(izip(ichain(primary_names, around_names), repeat(strict))) ) primary = strategy.method_chain(ichain( *[cases.get(primary, []) for primary in primary_names])) if cases.get('after') or cases.get('before'): befores = strategy.method_list(cases.get('before',[])) afters = strategy.method_list(list(cases.get('after',[]))[::-1]) def chain(*args,**kw): for tmp in befores(*args,**kw): pass # toss return values result = primary(*args,**kw) for tmp in afters(*args,**kw): pass # toss return values return result else: chain = primary if (self.order_around): chain = strategy.method_chain(ichain(*([cases.get(around, []) for around in around_names] + [[chain]]))) return chain
def break_chains(chains): prev = -1 broken_chains = chains while not prev == len(broken_chains): prev = len(broken_chains) chain_counter = Counter(ichain(*broken_chains)) doubly_chained = set(i for i, count in chain_counter.iteritems() if count > 1) print prev, len(doubly_chained) broken_chains = set( ichain(*[ fracture_chain(chain, doubly_chained) for chain in broken_chains ])) return [c for c in broken_chains if c]
def elems(self): if self.embedder: elems = ichain((self.elem, ), self.embedder.entity_p_buddy_elems) result = tuple(uniq(elems)) else: result = self.entity_p_buddy_elems return result
def get_languages(): langs = list(filter(None, ichain( (locale.getlocale(locale.LC_MESSAGES)[0],), os.environ.get('LANGUAGE', '').split(':')))) if not langs: langs.append('C') return langs
def elems (self) : if self.embedder : elems = ichain ((self.elem, ), self.embedder.entity_p_buddy_elems) result = tuple (uniq (elems)) else : result = self.entity_p_buddy_elems return result
def re_one_of(xs): """return a regex string that matches one of the xs""" # the "a^" thing matches nothing on purpose (not even with re.M). # it is necessary in the case where `xs` is empty because then the # regex is "()" which then matches the empty string -- oops. escape = re.escape return '|'.join( ichain((escape(x) for x in sorted(xs, key=len, reverse=True)), ["a^"]))
def embedded_elems (self) : def _gen (self) : for e in self.elem.elements : c = e.completer if c is not None : yield c.embedded_elems yield (self.elem, ) return tuple (uniq (ichain (* _gen (self))))
def re_one_of(xs): """return a regex string that matches one of the xs""" # the "a^" thing matches nothing on purpose (not even with re.M). # it is necessary in the case where `xs` is empty because then the # regex is "()" which then matches the empty string -- oops. escape = re.escape return '|'.join(ichain( (escape(x) for x in sorted(xs, key=len, reverse=True)), ["a^"]))
def embedded_elems(self): def _gen(self): for e in self.elem.elements: c = e.completer if c is not None: yield c.embedded_elems yield (self.elem, ) return tuple(uniq(ichain(*_gen(self))))
def entity_p_buddy_elems (self) : result = self.own_elems if self.entity_p : def _gen (self) : yield self.own_elems for e in self.anchor.elements : c = e.completer if c is not None and c.entity_p : yield c.embedded_elems result = tuple (uniq (ichain (* _gen (self)))) return result
def selectors(self): parent = self.parent prefix = self.prefix result = [] if self._selectors: result = list(ichain(*(s(parent) for s in self._selectors))) elif parent and prefix: result = parent.selectors if result and prefix: result = list(prefix.prefixed(s) for s in result) return result
def own_elems (self) : def _gen (self) : ep = self.elem.parent yield (self.elem, ) for n in self.attr.completer.names : elem = ep.get (n) if elem is not None : if elem.completer is not None : yield elem.completer.embedded_elems else : yield (elem, ) return tuple (uniq (ichain (* _gen (self))))
def __init__ (self, * word_lists, ** kw) : self._words = ws = tuple (ichain (* word_lists)) self._keys = set (ws) self._maps = {} self._maps_r = {} self._replacers = {} self._replacers_r = {} self.pop_to_self \ ( kw , "lowercase", "re_head", "re_tail", "skip_language" , prefix = "_" )
def __init__(self, *word_lists, **kw): self._words = ws = tuple(ichain(*word_lists)) self._keys = set(ws) self._maps = {} self._maps_r = {} self._replacers = {} self._replacers_r = {} self.pop_to_self \ ( kw , "lowercase", "re_head", "re_tail", "skip_language" , prefix = "_" )
def own_elems(self): def _gen(self): ep = self.elem.parent yield (self.elem, ) for n in self.attr.completer.names: elem = ep.get(n) if elem is not None: if elem.completer is not None: yield elem.completer.embedded_elems else: yield (elem, ) return tuple(uniq(ichain(*_gen(self))))
def entity_p_buddy_elems(self): result = self.own_elems if self.entity_p: def _gen(self): yield self.own_elems for e in self.anchor.elements: c = e.completer if c is not None and c.entity_p: yield c.embedded_elems result = tuple(uniq(ichain(*_gen(self)))) return result
def combine(self, cases): strict = [strategy.ordered_signatures, strategy.safe_methods] loose = [strategy.ordered_signatures, strategy.all_methods] primary_names = ['primary%d' % order for order in self.order_when] around_names = ['around%d' % order for order in self.order_around] cases = strategy.separate_qualifiers( cases, before=loose, after=loose, **dict(izip(ichain(primary_names, around_names), repeat(strict)))) primary = strategy.method_chain( ichain(*[cases.get(primary, []) for primary in primary_names])) if cases.get('after') or cases.get('before'): befores = strategy.method_list(cases.get('before', [])) afters = strategy.method_list(list(cases.get('after', []))[::-1]) def chain(*args, **kw): for tmp in befores(*args, **kw): pass # toss return values result = primary(*args, **kw) for tmp in afters(*args, **kw): pass # toss return values return result else: chain = primary if (self.order_around): chain = strategy.method_chain( ichain(*([cases.get(around, []) for around in around_names] + [[chain]]))) return chain
def _setup_order_by (self, E_Type, request, data) : s = data.pop ("order_by", "").strip () if s : def _gen (ns) : for n in ns : try : r = self._setup_order_by_1 (E_Type, n) except AttributeError as exc : pass else : yield r order_by = tuple (_gen (s.split (","))) if order_by : self.order_by, criteria = zip (* order_by) self.order_by_q = TFL.Sorted_By (* ichain (* criteria))
def _add_user_defined_indices(cls, e_type, ETW, sa_table): def _gen(sa_table, col_names): for cn in col_names: n = cn.lstrip("-") c = getattr(sa_table.c, n) if cn != n: c = c.desc() yield c for col_names in e_type.use_indices: if not isinstance(col_names, (tuple, list)): col_names = (col_names,) name = "__".join( ichain([sa_table.name], (cn.lstrip("-") for cn in col_names)) ### ensure uniqueness of the index' name ) columns = tuple(_gen(sa_table, col_names)) SA.schema.Index(name, *columns)
class eventClass(eventSuper): # build the constructor if isAbstract: __new__ = Event.__new__ else: def __new__(cls, **kwargs): # construct the tuple containing property values values = tuple( getattr(cls, prop_name).extract(kwargs) for prop_name in cls.__properties__) return tuple.__new__(cls, values) # property names __properties__ = tuple( ichain(eventSuper.__properties__, (p.__name__ for p in properties)))
def make_oai_task_group(oais): """Make a celery group for an OAISet. Since for each OAISet any given record has to be modified by either removing or adding the OAISet.spec, it's save to create a single group per OAISet for all records (no risk of racing conditions in parallel execution). :param oais: OAISet for which the task group is to be made. :type oais: invenio_oaiserver.modules.OAISet """ spec_q = Q('match', **{'_oai.sets': oais.spec}) pattern_q = Q('query_string', query=oais.search_pattern) spec_remove_q = Q('bool', must=spec_q, must_not=pattern_q) spec_add_q = Q('bool', must=pattern_q, must_not=spec_q) return group(ichain(iter_record_oai_tasks(spec_remove_q, oais.spec, remove_oaiset_spec), iter_record_oai_tasks(spec_add_q, oais.spec, add_oaiset_spec)))
def __call__(self, *args, **decls): """Return a dict with `decls` plus any necessary prefixed declarations. """ result = {} name = self.name prefixes = self.prefixes vp_map = self.vp_map if args: v = result[name] = " ".join(args) for p in prefixes: result["-".join((p, name))] = v for k, v in pyk.iteritems(decls): k = k.replace("_", "-") n = "-".join((name, k)) result[n] = v t = k.split("-")[-1] for p in ichain(vp_map.get(t, ()), prefixes): result["-".join((p, n))] = v return result
def _path_splitter(s, _d_match=re.compile(r'\.\d').match): """Split a string into its path components. Assumes a string is a path.""" # If a PathLib Object, use it's functionality to perform the split. if has_pathlib and isinstance(s, PurePath): s = py23_str(s) path_parts = deque() p_appendleft = path_parts.appendleft # Continue splitting the path from the back until we have reached # '..' or '.', or until there is nothing left to split. path_location = s while path_location != os_curdir and path_location != os_pardir: parent_path = path_location path_location, child_path = path_split(parent_path) if path_location == parent_path: break p_appendleft(child_path) # This last append is the base path. # Only append if the string is non-empty. if path_location: p_appendleft(path_location) # Now, split off the file extensions using a similar method to above. # Continue splitting off file extensions until we reach a decimal number # or there are no more extensions. # We are not using built-in functionality of PathLib here because of # the recursive splitting up to a decimal. base = path_parts.pop() base_parts = deque() b_appendleft = base_parts.appendleft while True: front = base base, ext = path_splitext(front) if _d_match(ext) or not ext: # Reset base to before the split if the split is invalid. base = front break b_appendleft(ext) b_appendleft(base) # Return the split parent paths and then the split basename. return ichain(path_parts, base_parts)
def make_oai_task_group(oais): """Make a celery group for an OAISet. Since for each OAISet any given record has to be modified by either removing or adding the OAISet.spec, it's save to create a single group per OAISet for all records (no risk of racing conditions in parallel execution). :param oais: OAISet for which the task group is to be made. :type oais: invenio_oaiserver.modules.OAISet """ spec_q = Q('match', **{'_oai.sets': oais.spec}) pattern_q = Q('query_string', query=oais.search_pattern) spec_remove_q = Q('bool', must=spec_q, must_not=pattern_q) spec_add_q = Q('bool', must=pattern_q, must_not=spec_q) return group( ichain( iter_record_oai_tasks(spec_remove_q, oais.spec, remove_oaiset_spec), iter_record_oai_tasks(spec_add_q, oais.spec, add_oaiset_spec)))
def __call__(self, *args, **decls): """Return a dict with `decls` plus any necessary prefixed declarations. """ result = {} name = self.name prefixes = self.prefixes vp_map = self.vp_map as_text = pyk.text_type if args: v = result[name] = " ".join(as_text(a) for a in args) for p in prefixes: result["-".join((p, name))] = v for k, v in pyk.iteritems(decls): k = k.replace("_", "-") n = "-".join((name, k)) result[n] = v t = k.split("-")[-1] for p in ichain(vp_map.get(t, ()), prefixes): result["-".join((p, n))] = v return result
def _add_user_defined_indices(cls, e_type, ETW, sa_table): def _gen(sa_table, col_names): for cn in col_names: n = cn.lstrip("-") c = getattr(sa_table.c, n) if cn != n: c = c.desc() yield c for col_names in e_type.use_indices: if not isinstance(col_names, (tuple, list)): col_names = (col_names, ) name = "__".join \ ( ichain ( [sa_table.name] ### ensure uniqueness of the index' name , (cn.lstrip ("-") for cn in col_names) ) ) columns = tuple(_gen(sa_table, col_names)) SA.schema.Index(name, *columns)
def path_splitter(s, _d_match=re.compile(r"\.\d").match): """ Split a string into its path components. Assumes a string is a path or is path-like. Parameters ---------- s : str | pathlib.Path Returns ------- split : tuple The path split by directory components and extensions. Examples -------- >>> tuple(path_splitter("this/thing.ext")) ('this', 'thing', '.ext') """ if not isinstance(s, PurePath): s = PurePath(s) # Split the path into parts. *path_parts, base = s.parts # Now, split off the file extensions until we reach a decimal number at # the beginning of the suffix or there are no more extensions. suffixes = PurePath(base).suffixes try: digit_index = next(i for i, x in enumerate(reversed(suffixes)) if _d_match(x)) except StopIteration: pass else: digit_index = len(suffixes) - digit_index suffixes = suffixes[digit_index:] base = base.replace("".join(suffixes), "") return filter(None, ichain(path_parts, [base], suffixes))
def _handle_pycompile (self, cao) : P = self._P (cao) cwd = self.pbl.cwd root = pjoin (cao.root_path, cao.apply_to_version) dirs = [cao.app_dir] + cao.lib_dir args = tuple \ ( ichain ( ["-m", "compileall"] , cao.compile_options , ((["-x"] + cao.skip_modules) if cao.skip_modules else []) , dirs ) ) clean = self.pbl ["find"] \ [tuple (dirs + ["-name", "*.py[co]", "-delete"])] with cwd (root) : if cao.verbose or cao.dry_run : print ("cd", self.pbl.path ()) print (clean) print (P.python, " ".join (args)) if not cao.dry_run : clean () P.python (* args)
def main(args=None): if args is None: args = sys.argv[1:] args = parse_args(args) torch.autograd.set_detect_anomaly(True) curr_run_name = args.run_name EPOCHS = args.epochs if 'steps_per_epoch' in args: STEPS_PER_EPOCH = args.steps_per_epoch else: STEPS_PER_EPOCH = None #region preparations #region DATASETS_DIR DATASETS_DIR = os.path.join(os.path.abspath('./'), 'datasets') try: EnsureDirectoryExists(DATASETS_DIR) except: print('datasets directory couldn`t be found and couldn`t be created:\n%s' % DATASETS_DIR) raise FileNotFoundError('datasets directory couldn`t be found and couldn`t be created:\n%s' % DATASETS_DIR) #endregion #region data_dir dataset_name = 'mnist' data_dir = os.path.join(DATASETS_DIR, dataset_name) try: EnsureDirectoryExists(data_dir) except: print('data_dir couldn`t be found and couldn`t be created:\n%s' % data_dir) raise FileNotFoundError('data_dir couldn`t be found and couldn`t be created:\n%s' % data_dir) #endregion #region logs_dir logs_dir = os.path.join(os.path.abspath('./'), 'logs', curr_run_name) try: EnsureDirectoryExists(logs_dir) except: print('logs directory couldn`t be found and couldn`t be created:\n%s' % logs_dir) raise FileNotFoundError('logs directory couldn`t be found and couldn`t be created:\n%s' % logs_dir) #endregion #region imgs_dir imgs_dir = os.path.join(os.path.abspath('./'), 'logs', curr_run_name, 'images') try: EnsureDirectoryExists(imgs_dir) except: print('output images directory couldn`t be found and couldn`t be created:\n%s' % imgs_dir) raise FileNotFoundError('output images directory couldn`t be found and couldn`t be created:\n%s' % imgs_dir) #endregion #region scripts_backup_dir scripts_backup_dir = os.path.join(os.path.abspath('./'), 'scripts_backup', curr_run_name) try: EnsureDirectoryExists(scripts_backup_dir) except: print('scripts backup directory couldn`t be found and couldn`t be created:\n%s' % scripts_backup_dir) raise FileNotFoundError('scripts backup directory couldn`t be found and couldn`t be created:\n%s' % scripts_backup_dir) #endregion #region models_dir models_dir = os.path.join(os.path.abspath('./'), 'logs', curr_run_name, 'models') try: EnsureDirectoryExists(models_dir) except: print('models snapshots directory couldn`t be found and couldn`t be created:\n%s' % models_dir) raise FileNotFoundError('models snapshots directory couldn`t be found and couldn`t be created:\n%s' % models_dir) #endregion # region models_dir tboard_dir = os.path.join(os.path.abspath('./'), 'logs', curr_run_name, 'TBoard') try: EnsureDirectoryExists(tboard_dir) except: print('Tensorboard directory couldn`t be found and couldn`t be created:\n%s' % tboard_dir) raise FileNotFoundError('Tensorboard directory directory couldn`t be found and couldn`t be created:\n%s' % tboard_dir) # endregion # endregion preparations #region backing up the scripts configuration ignore_func = lambda dir, files: [f for f in files if (isfile(join(dir, f)) and f[-3:] != '.py')] + [d for d in files if ((isdir(d)) & (d.endswith('scripts_backup') | d.endswith('.ipynb_checkpoints') | d.endswith('__pycache__') | d.endswith('build') | d.endswith('datasets') | d.endswith('logs') | d.endswith('runs') | d.endswith('snapshots')))] copytree_multi('./', scripts_backup_dir, ignore=ignore_func) #endregion #region setting training parameters num_workers = args.num_workers if args.debug: num_workers = 0 batch_size = args.batch_size lr = 1e-4 b1 = 0.5 b2 = 0.9 #99 decay = 2.5*1e-5 n_skip_iter = 1 #5 img_size = 256 channels = 1 # Latent space info latent_dim = args.latent_dim n_c = args.cat_num betan = 10 betac = 10 wass_metric = args.wass_metric mtype = 'van' if (wass_metric): mtype = 'wass' x_shape = (channels, img_size, img_size) #endregion cuda = True if torch.cuda.is_available() else False # device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') if cuda: torch.cuda.set_device(0) # Loss function bce_loss = torch.nn.BCELoss() xe_loss = torch.nn.CrossEntropyLoss() mse_loss = torch.nn.MSELoss() # Initialize generator and discriminator generator = Generator_CNN(latent_dim, n_c, x_shape, verbose=True) encoder = Encoder_CNN(latent_dim, n_c) discriminator = Discriminator_CNN(wass_metric=wass_metric) if cuda: generator.cuda() encoder.cuda() discriminator.cuda() bce_loss.cuda() xe_loss.cuda() mse_loss.cuda() Tensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor # Configure training data loader def collation(data): x_tensors = [d[0] for d in data] y_tensors = [d[1] for d in data] x_tensor = torch.cat(x_tensors, dim=0) y_tensor = torch.cat(y_tensors, dim=0) return x_tensor, y_tensor train_dataset = CustomMNISTdataset(mnist_fname=os.path.join(DATASETS_DIR, 'mnist', 'MNIST', 'mnist.npz'), train_set=True, augment=True) train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers) # collate_fn=collation) # dataloader = get_dataloader(data_dir=data_dir, # batch_size=batch_size, # num_workers=num_workers, # train_set=True, # augment=True) # Test data loader # test_dataloader = get_dataloader(data_dir=data_dir, batch_size=batch_size, train_set=False, augment=False) test_dataset = CustomMNISTdataset(mnist_fname=os.path.join(DATASETS_DIR, 'mnist', 'MNIST', 'mnist.npz'), train_set=False, augment=True) test_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers) # collate_fn=collation) ge_chain = ichain(generator.parameters(), encoder.parameters()) optimizer_GE = torch.optim.Adam(ge_chain, lr=lr, betas=(b1, b2), weight_decay=decay) optimizer_D = torch.optim.Adam(discriminator.parameters(), lr=lr, betas=(b1, b2)) #optimizer_D = torch.optim.Adam(discriminator.parameters(), lr=lr, betas=(b1, b2), weight_decay=decay) # ---------- # Training # ---------- ge_l = [] d_l = [] c_zn = [] c_zc = [] c_i = [] if STEPS_PER_EPOCH is None: STEPS_PER_EPOCH = len(train_dataset)//train_dataloader.batch_size if STEPS_PER_EPOCH * train_dataloader.batch_size < len(train_dataset): STEPS_PER_EPOCH = STEPS_PER_EPOCH+1 EVAL_STEPS = len(test_dataset)//test_dataloader.batch_size if EVAL_STEPS*test_dataloader.batch_size < len(test_dataset): EVAL_STEPS = EVAL_STEPS+1 #region saving train details for in-train-time analysis train_details = {'run_name': curr_run_name, 'EPOCHS': EPOCHS, 'STEPS_PER_EPOCH': STEPS_PER_EPOCH, 'EVAL_STEPS': EVAL_STEPS, 'learning_rate': lr, 'beta_1': b1, 'beta_2': b2, 'weight_decay': decay, 'n_skip_iter': n_skip_iter, 'latent_dim': latent_dim, 'n_classes': n_c, 'beta_n': betan, 'beta_c': betac, 'wass_metric': wass_metric, } # this file will be rewritten lately with open(os.path.join(logs_dir, 'train_details.pkl'), 'wb') as f: pickle.dump(train_details, f) # region saving train details for in-train-time analysis writer = SummaryWriter(log_dir=tboard_dir) #region Training loop print('\nBegin training session with %i epochs...\n'%(EPOCHS)) for epoch in range(EPOCHS): generator.train() encoder.train() for idx, (imgs, itruth_label) in tqdm(enumerate(train_dataloader), total=STEPS_PER_EPOCH): if idx >= STEPS_PER_EPOCH: break # Zero gradients for models generator.zero_grad() encoder.zero_grad() discriminator.zero_grad() # Configure input real_imgs = Variable(imgs.type(Tensor)) #region Train Generator + Encoder optimizer_GE.zero_grad() # Sample random latent variables zn, zc, zc_idx = sample_z(shape=imgs.shape[0], latent_dim=latent_dim, n_c=n_c, cuda_available=cuda) # Generate a batch of images gen_imgs = generator(zn, zc) # Discriminator output from real and generated samples D_gen = discriminator(gen_imgs) D_real = discriminator(real_imgs) # if args.debug: # continue # Step for Generator & Encoder, n_skip_iter times less than for discriminator if (idx % n_skip_iter == 0): # Encode the generated images enc_gen_zn, enc_gen_zc, enc_gen_zc_logits = encoder(gen_imgs) # Calculate losses for z_n, z_c zn_loss = mse_loss(enc_gen_zn, zn) zc_loss = xe_loss(enc_gen_zc_logits, zc_idx) #zc_loss = cross_entropy(enc_gen_zc_logits, zc) # Check requested metric if wass_metric: # Wasserstein GAN loss ge_loss = torch.mean(D_gen) + betan * zn_loss + betac * zc_loss else: # Vanilla GAN loss valid = Variable(Tensor(gen_imgs.size(0), 1).fill_(1.0), requires_grad=False) v_loss = bce_loss(D_gen, valid) ge_loss = v_loss + betan * zn_loss + betac * zc_loss ge_loss.backward(retain_graph=True) optimizer_GE.step() #endregion #region Train Discriminator optimizer_D.zero_grad() # Measure discriminator's ability to classify real from generated samples if wass_metric: # Gradient penalty term grad_penalty = calc_gradient_penalty(discriminator, real_imgs, gen_imgs, cuda_available=cuda) # Wasserstein GAN loss w/gradient penalty d_loss = torch.mean(D_real) - torch.mean(D_gen) + grad_penalty else: # Vanilla GAN loss fake = Variable(Tensor(gen_imgs.size(0), 1).fill_(0.0), requires_grad=False) real_loss = bce_loss(D_real, valid) fake_loss = bce_loss(D_gen, fake) d_loss = (real_loss + fake_loss) / 2 d_loss.backward() optimizer_D.step() #endregion # if args.debug: # continue # Save training losses d_l.append(d_loss.item()) ge_l.append(ge_loss.item()) # Generator in eval mode generator.eval() encoder.eval() # Set number of examples for cycle calcs n_sqrt_samp = 5 n_samp = n_sqrt_samp * n_sqrt_samp #region Cycle through test real -> enc -> gen print('cycle evaluation...') cycle_eval_mse_losses = [] for idx, (eval_imgs, eval_labels) in tqdm(enumerate(test_dataloader), total=EVAL_STEPS): # test_imgs, test_labels = next(iter(testdata)) eval_imgs = Variable(eval_imgs.type(Tensor)) # t_imgs, t_label = test_imgs.data, test_labels # Encode sample real instances e_tzn, e_tzc, e_tzc_logits = encoder(eval_imgs) # Generate sample instances from encoding teg_imgs = generator(e_tzn, e_tzc) # Calculate cycle reconstruction loss img_mse_loss = mse_loss(eval_imgs, teg_imgs) # Save img reco cycle loss cycle_eval_mse_losses.append(img_mse_loss.item()) c_i.append(np.mean(cycle_eval_mse_losses)) #endregion #region Cycle through randomly sampled encoding -> generator -> encoder zn_samp, zc_samp, zc_samp_idx = sample_z(shape=n_samp, latent_dim=latent_dim, n_c=n_c, cuda_available=cuda) # Generate sample instances gen_imgs_samp = generator(zn_samp, zc_samp) # Encode sample instances zn_e, zc_e, zc_e_logits = encoder(gen_imgs_samp) # Calculate cycle latent losses lat_mse_loss = mse_loss(zn_e, zn_samp) lat_xe_loss = xe_loss(zc_e_logits, zc_samp_idx) #lat_xe_loss = cross_entropy(zc_e_logits, zc_samp) # Save latent space cycle losses c_zn.append(lat_mse_loss.item()) c_zc.append(lat_xe_loss.item()) #endregion #region write losses to tensorboard writer.add_scalar('Loss/train/gen_enc_loss', ge_loss.item(), epoch) writer.add_scalar('Loss/train/dsc_loss', d_loss.item(), epoch) writer.add_scalar('Loss/train/cycle/lat_mse_loss', lat_mse_loss.item(), epoch) writer.add_scalar('Loss/train/cycle/lat_xe_loss', lat_xe_loss.item(), epoch) writer.add_scalar('Loss/train/cycle/img_mse_loss', img_mse_loss.item(), epoch) #endregion #region Save cycled and generated examples! r_imgs, i_label = real_imgs.data[:n_samp], itruth_label[:n_samp] e_zn, e_zc, e_zc_logits = encoder(r_imgs) reg_imgs = generator(e_zn, e_zc) save_image(r_imgs.data[:n_samp], '%s/real_%06i.png' %(imgs_dir, epoch), nrow=n_sqrt_samp, normalize=True) save_image(reg_imgs.data[:n_samp], '%s/reg_%06i.png' %(imgs_dir, epoch), nrow=n_sqrt_samp, normalize=True) save_image(gen_imgs_samp.data[:n_samp], '%s/gen_%06i.png' %(imgs_dir, epoch), nrow=n_sqrt_samp, normalize=True) #endregion #region Generate samples for specified classes stack_imgs = [] for idx_gen in range(n_c): # Sample specific class zn_samp, zc_samp, zc_samp_idx = sample_z(shape=n_c, latent_dim=latent_dim, n_c=n_c, fix_class=idx_gen, cuda_available=cuda) # Generate sample instances gen_imgs_samp = generator(zn_samp, zc_samp) if (len(stack_imgs) == 0): stack_imgs = gen_imgs_samp else: stack_imgs = torch.cat((stack_imgs, gen_imgs_samp), 0) # Save class-specified generated examples! save_image(stack_imgs, '%s/gen_classes_%06i.png' %(imgs_dir, epoch), nrow=n_c, normalize=True) #endregion #region snapshots if args.snapshots_period == -1 or args.snapshots_period == 0: pass elif args.snapshots_period > 1: if epoch % args.snapshots_period == 0: model_list = [discriminator, encoder, generator] save_models(models=model_list, out_dir=models_dir, stage=epoch) #endregion print ('[Epoch %d/%d]' % (epoch, EPOCHS)) print("\tModel Losses: [D: %f] [GE: %f]" % (d_loss.item(), ge_loss.item())) print("\tCycle Losses: [x: %f] [z_n: %f] [z_c: %f]" % (img_mse_loss.item(), lat_mse_loss.item(), lat_xe_loss.item())) #endregion # if args.debug: # sys.exit() # Save training results train_details = {'run_name' : curr_run_name, 'EPOCHS' : EPOCHS, 'STEPS_PER_EPOCH' : STEPS_PER_EPOCH, 'learning_rate' : lr, 'beta_1' : b1, 'beta_2' : b2, 'weight_decay' : decay, 'n_skip_iter' : n_skip_iter, 'latent_dim' : latent_dim, 'n_classes' : n_c, 'beta_n' : betan, 'beta_c' : betac, 'wass_metric' : wass_metric, 'gen_enc_loss' : {'label' : 'G+E', 'data' : ge_l}, 'disc_loss' : {'label' : 'D', 'data' : d_l}, 'zn_cycle_loss' : {'label' : '$||Z_n-E(G(x))_n||$', 'data' : c_zn}, 'zc_cycle_loss' : {'label' : '$||Z_c-E(G(x))_c||$', 'data' : c_zc}, 'img_cycle_loss' : {'label' : '$||X-G(E(x))||$', 'data' : c_i} } with open(os.path.join(logs_dir, 'train_details.pkl'), 'wb') as f: pickle.dump(train_details, f) # Plot some training results plot_train_loss(train_details=train_details, var_list=['gen_enc_loss', 'disc_loss'], figname=os.path.join(logs_dir, 'training_model_losses.png') ) plot_train_loss(train_details=train_details, var_list=['zn_cycle_loss', 'zc_cycle_loss', 'img_cycle_loss'], figname=os.path.join(logs_dir, 'training_cycle_loss.png') ) # Save current state of trained models model_list = [discriminator, encoder, generator] save_models(models=model_list, out_dir=models_dir)
def words (self) : return sorted (ichain (* (g8r.words for g8r in self.rereps)))
, "ß" : "ss" , "ä" : "ae" , "ö" : "oe" , "ü" : "ue" } _diacrit_rep = Dict_Replacer (_diacrit_map) _graph_rep = Re_Replacer \ ( "(%s)+" % "|".join (re.escape (c) for c in ("^!$%&([{}]) ?`'*+#:;<>|" '"')) , "_" ) _non_print_rep = Re_Replacer \ ( "|".join (re.escape (chr (i)) for i in ichain (range (0, 32), [127])) , "" ) _quote_map = \ { "«" : "<<" , "»" : ">>" , "´" : "'" , "\u2018" : "'" , "\u2019" : "'" , "\u201A" : "'" , "\u201B" : "'" , "\u201C" : '"' , "\u201D" : '"' , "\u201E" : '"' , "\u201F" : '"'
def main(): global args parser = argparse.ArgumentParser( description="Convolutional NN Training Script") parser.add_argument("-r", "--run_name", dest="run_name", default="dec", help="Name of training run") parser.add_argument("-n", "--n_epochs", dest="n_epochs", default=50, type=int, help="Number of epochs") parser.add_argument("-b", "--batch_size", dest="batch_size", default=256, type=int, help="Batch size") parser.add_argument("-s", "--dataset_name", dest="dataset_name", default='mnist', choices=dataset_list, help="Dataset name") parser.add_argument("-p", "--pretrain", dest="pretrain", default=False, help="pretrain ae") args = parser.parse_args() run_name = args.run_name dataset_name = args.dataset_name # make directory run_dir = os.path.join(RUNS_DIR, dataset_name, run_name) data_dir = os.path.join(DATASETS_DIR, dataset_name) imgs_dir = os.path.join(run_dir, 'images') models_dir = os.path.join(run_dir, 'models') log_path = os.path.join(run_dir, 'logs') os.makedirs(data_dir, exist_ok=True) os.makedirs(run_dir, exist_ok=True) os.makedirs(imgs_dir, exist_ok=True) os.makedirs(models_dir, exist_ok=True) os.makedirs(log_path, exist_ok=True) #------train var------- sgd_lr = 0.01 momentum = 0.9 epochs = args.n_epochs batch_size = args.batch_size pretrain_epochs = 20 pretrain = args.pretrain n_cluster = 10 lr_adam = 1e-4 # 0.0001, 用于预训练autoencoder b1 = 0.5 b2 = 0.9 #99 decay = 2.5 * 1e-5 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") alpha = 1.0 #------test var-------- test_batch_size = 1500 # net encoder = Encoder_CNN(n_cluster=n_cluster, batch_size=batch_size) decoder = Decoder_CNN(n_cluster=n_cluster, batch_size=batch_size, img_feature=(1, 28, 28)) # 组合参数 autoencoder_params = ichain(encoder.parameters(), decoder.parameters()) # optimization auto_op = torch.optim.Adam(autoencoder_params, lr=lr_adam, betas=(b1, b2), weight_decay=decay) # dataloader dataloader = get_dataloader(dataset_path=data_dir, dataset_name=dataset_name, batch_size=batch_size, train=True) # loss auto_loss = nn.MSELoss() # to cuda encoder.to(device) decoder.to(device) auto_loss.to(device) # ----pretrain---- if pretrain: print('...Pretraining...') logger = open(os.path.join(log_path, "log.txt"), 'a') logger.write("pretraining...\n") logger.close() for epoch in range(pretrain_epochs): for i, (data, target) in enumerate(dataloader): data, target = data.to(device), target.to(device) encoder.train() decoder.train() encoder.zero_grad() decoder.zero_grad() z = encoder(data) output = decoder(z) loss = auto_loss(data, output) loss.backward() auto_op.step() # caculate acc of autoencoder data, target = next(iter(dataloader)) data = data.to(device) encoder.eval() decoder.eval() features = encoder(data) save_images(decoder(features), "%s/%s_%d.png" % (imgs_dir, 'pretrain_test', epoch)) data, target = features.data.cpu(), target.numpy() km = KMeans(n_clusters=n_cluster, n_init=20) y_pred = km.fit_predict(features.data.cpu()) acc = metrics.acc(target, y_pred) nmi = metrics.nmi(target, y_pred) print(' ' * 8 + '|==> acc: %.4f, nmi: %.4f <==|' % (acc, nmi)) # log pretrain info logger = open(os.path.join(log_path, "log.txt"), 'a') logger.write("iter={}, acc={:.4f}, nmi={:.4f}\n".format( epoch, acc, nmi)) logger.close() # save model params torch.save(encoder.state_dict(), os.path.join(models_dir, 'encoder.pkl')) torch.save(decoder.state_dict(), os.path.join(models_dir, 'decoder.pkl')) else: encoder.load_state_dict( torch.load(os.path.join(models_dir, 'encoder.pkl'))) decoder.load_state_dict( (torch.load(os.path.join(models_dir, 'decoder.pkl')))) #============================DEC=============================== dec = DEC(encoder=encoder, n_cluster=n_cluster, batch_size=batch_size, alpha=alpha) dec_op = torch.optim.SGD(dec.parameters(), lr=sgd_lr, momentum=momentum) dec.to(device) # init mu data, target = next(iter(dataloader)) dec.get_assign_cluster_centers_op(encoder(data.to(device))) logger = open(os.path.join(log_path, "log.txt"), 'a') logger.write( "============================DEC===============================\n") logger.close() for epoch in range(epochs): for i, (data, target) in enumerate(dataloader): data, target = data.to(device), target.to(device) dec.train() dec.zero_grad() dec_op.zero_grad() q, p = dec(data) loss = kl_divergence(p, q) loss.backward() dec_op.step() # test encoder.eval() dec.eval() _data, _target = next(iter(dataloader)) q, p = dec(_data.cuda()) _loss = kl_divergence(p, q) _pred = torch.argmax(q, dim=1) _acc = metrics.acc(_target.numpy(), _pred.cpu().numpy()) _nmi = metrics.nmi(_target.numpy(), _pred.cpu().numpy()) print("[DEC] epoch: {}\tloss: {}\tacc: {}\tnmi: {}".format( epoch, _loss, _acc, _nmi)) logger = open(os.path.join(log_path, "log.txt"), 'a') logger.write("[DEC] epoch: {}\tloss: {}\tacc: {}\tnmi: {}\n".format( epoch, _loss, _acc, _nmi)) logger.close() # save dec model torch.save(dec.state_dict(), os.path.join(models_dir, 'dec.pkl'))
def from_MT (cls, mt) : args = tuple (ichain (* (c [:2] for c in mt.cols))) return cls (* args)
def _write_headers(self): headers = ''.join(ichain([self.requestline], self.message.headers)) self.endpoint.sendall(headers) self.endpoint.sendall('\r\n')
def main(): global args parser = argparse.ArgumentParser( description="Convolutional NN Training Script") parser.add_argument("-r", "--run_name", dest="run_name", default='clusgan', help="Name of training run") parser.add_argument("-n", "--n_epochs", dest="n_epochs", default=200, type=int, help="Number of epochs") parser.add_argument("-b", "--batch_size", dest="batch_size", default=64, type=int, help="Batch size") parser.add_argument("-s", "--dataset_name", dest="dataset_name", default='mnist', choices=dataset_list, help="Dataset name") args = parser.parse_args() run_name = args.run_name dataset_name = args.dataset_name # Make directory structure for this run run_dir = os.path.join(RUNS_DIR, dataset_name, run_name) data_dir = os.path.join(DATASETS_DIR, dataset_name) imgs_dir = os.path.join(run_dir, 'images') models_dir = os.path.join(run_dir, 'models') os.makedirs(data_dir, exist_ok=True) os.makedirs(run_dir, exist_ok=True) os.makedirs(imgs_dir, exist_ok=True) os.makedirs(models_dir, exist_ok=True) print('\nResults to be saved in directory %s\n' % (run_dir)) # Training details n_epochs = args.n_epochs batch_size = args.batch_size test_batch_size = 5000 lr = 1e-4 b1 = 0.5 b2 = 0.9 #99 decay = 2.5 * 1e-5 n_skip_iter = 1 #5 img_size = 28 channels = 1 # Latent space info latent_dim = 30 n_c = 10 betan = 10 betac = 10 # Wasserstein metric flag wass_metric = True #wass_metric = False x_shape = (channels, img_size, img_size) cuda = True if torch.cuda.is_available() else False device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') # Loss function bce_loss = torch.nn.BCELoss() xe_loss = torch.nn.CrossEntropyLoss() mse_loss = torch.nn.MSELoss() # Initialize generator and discriminator generator = Generator_CNN(latent_dim, n_c, x_shape) encoder = Encoder_CNN(latent_dim, n_c) discriminator = Discriminator_CNN(wass_metric=wass_metric) if cuda: generator.cuda() encoder.cuda() discriminator.cuda() bce_loss.cuda() xe_loss.cuda() mse_loss.cuda() Tensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor # Configure training data loader dataloader = get_dataloader(dataset_name=dataset_name, data_dir=data_dir, batch_size=batch_size) # Test data loader testdata = get_dataloader(dataset_name=dataset_name, data_dir=data_dir, batch_size=test_batch_size, train_set=False) test_imgs, test_labels = next(iter(testdata)) test_imgs = Variable(test_imgs.type(Tensor)) ge_chain = ichain(generator.parameters(), encoder.parameters()) optimizer_GE = torch.optim.Adam(ge_chain, lr=lr, betas=(b1, b2), weight_decay=decay) optimizer_D = torch.optim.Adam(discriminator.parameters(), lr=lr, betas=(b1, b2)) #optimizer_D = torch.optim.Adam(discriminator.parameters(), lr=lr, betas=(b1, b2), weight_decay=decay) # ---------- # Training # ---------- ge_l = [] d_l = [] c_zn = [] c_zc = [] c_i = [] # Training loop print('\nBegin training session with %i epochs...\n' % (n_epochs)) for epoch in range(n_epochs): for i, (imgs, itruth_label) in enumerate(dataloader): # Ensure generator/encoder are trainable generator.train() encoder.train() # Zero gradients for models generator.zero_grad() encoder.zero_grad() discriminator.zero_grad() # Configure input real_imgs = Variable(imgs.type(Tensor)) # --------------------------- # Train Generator + Encoder # --------------------------- optimizer_GE.zero_grad() # Sample random latent variables zn, zc, zc_idx = sample_z(shape=imgs.shape[0], latent_dim=latent_dim, n_c=n_c) # Generate a batch of images gen_imgs = generator(zn, zc) # Discriminator output from real and generated samples D_gen = discriminator(gen_imgs) D_real = discriminator(real_imgs) # Step for Generator & Encoder, n_skip_iter times less than for discriminator if (i % n_skip_iter == 0): # Encode the generated images enc_gen_zn, enc_gen_zc, enc_gen_zc_logits = encoder(gen_imgs) # Calculate losses for z_n, z_c zn_loss = mse_loss(enc_gen_zn, zn) zc_loss = xe_loss(enc_gen_zc_logits, zc_idx) #zc_loss = cross_entropy(enc_gen_zc_logits, zc) # Check requested metric if wass_metric: # Wasserstein GAN loss ge_loss = torch.mean( D_gen) + betan * zn_loss + betac * zc_loss else: # Vanilla GAN loss valid = Variable(Tensor(gen_imgs.size(0), 1).fill_(1.0), requires_grad=False) v_loss = bce_loss(D_gen, valid) ge_loss = v_loss + betan * zn_loss + betac * zc_loss ge_loss.backward(retain_graph=True) optimizer_GE.step() # --------------------- # Train Discriminator # --------------------- optimizer_D.zero_grad() # Measure discriminator's ability to classify real from generated samples if wass_metric: # Gradient penalty term grad_penalty = calc_gradient_penalty(discriminator, real_imgs, gen_imgs) # Wasserstein GAN loss w/gradient penalty d_loss = torch.mean(D_real) - torch.mean(D_gen) + grad_penalty else: # Vanilla GAN loss fake = Variable(Tensor(gen_imgs.size(0), 1).fill_(0.0), requires_grad=False) real_loss = bce_loss(D_real, valid) fake_loss = bce_loss(D_gen, fake) d_loss = (real_loss + fake_loss) / 2 d_loss.backward() optimizer_D.step() # Save training losses d_l.append(d_loss.item()) ge_l.append(ge_loss.item()) # Generator in eval mode generator.eval() encoder.eval() # Set number of examples for cycle calcs n_sqrt_samp = 5 n_samp = n_sqrt_samp * n_sqrt_samp ## Cycle through test real -> enc -> gen t_imgs, t_label = test_imgs.data, test_labels #r_imgs, i_label = real_imgs.data[:n_samp], itruth_label[:n_samp] # Encode sample real instances e_tzn, e_tzc, e_tzc_logits = encoder(t_imgs) # Generate sample instances from encoding teg_imgs = generator(e_tzn, e_tzc) # Calculate cycle reconstruction loss img_mse_loss = mse_loss(t_imgs, teg_imgs) # Save img reco cycle loss c_i.append(img_mse_loss.item()) ## Cycle through randomly sampled encoding -> generator -> encoder zn_samp, zc_samp, zc_samp_idx = sample_z(shape=n_samp, latent_dim=latent_dim, n_c=n_c) # Generate sample instances gen_imgs_samp = generator(zn_samp, zc_samp) # Encode sample instances zn_e, zc_e, zc_e_logits = encoder(gen_imgs_samp) # Calculate cycle latent losses lat_mse_loss = mse_loss(zn_e, zn_samp) lat_xe_loss = xe_loss(zc_e_logits, zc_samp_idx) #lat_xe_loss = cross_entropy(zc_e_logits, zc_samp) # Save latent space cycle losses c_zn.append(lat_mse_loss.item()) c_zc.append(lat_xe_loss.item()) # Save cycled and generated examples! r_imgs, i_label = real_imgs.data[:n_samp], itruth_label[:n_samp] e_zn, e_zc, e_zc_logits = encoder(r_imgs) reg_imgs = generator(e_zn, e_zc) save_image(r_imgs.data[:n_samp], '%s/real_%06i.png' % (imgs_dir, epoch), nrow=n_sqrt_samp, normalize=True) save_image(reg_imgs.data[:n_samp], '%s/reg_%06i.png' % (imgs_dir, epoch), nrow=n_sqrt_samp, normalize=True) save_image(gen_imgs_samp.data[:n_samp], '%s/gen_%06i.png' % (imgs_dir, epoch), nrow=n_sqrt_samp, normalize=True) ## Generate samples for specified classes stack_imgs = [] for idx in range(n_c): # Sample specific class zn_samp, zc_samp, zc_samp_idx = sample_z(shape=n_c, latent_dim=latent_dim, n_c=n_c, fix_class=idx) # Generate sample instances gen_imgs_samp = generator(zn_samp, zc_samp) if (len(stack_imgs) == 0): stack_imgs = gen_imgs_samp else: stack_imgs = torch.cat((stack_imgs, gen_imgs_samp), 0) # Save class-specified generated examples! save_image(stack_imgs, '%s/gen_classes_%06i.png' % (imgs_dir, epoch), nrow=n_c, normalize=True) print ("[Epoch %d/%d] \n"\ "\tModel Losses: [D: %f] [GE: %f]" % (epoch, n_epochs, d_loss.item(), ge_loss.item()) ) print("\tCycle Losses: [x: %f] [z_n: %f] [z_c: %f]" % (img_mse_loss.item(), lat_mse_loss.item(), lat_xe_loss.item())) # Save training results train_df = pd.DataFrame({ 'n_epochs': n_epochs, 'learning_rate': lr, 'beta_1': b1, 'beta_2': b2, 'weight_decay': decay, 'n_skip_iter': n_skip_iter, 'latent_dim': latent_dim, 'n_classes': n_c, 'beta_n': betan, 'beta_c': betac, 'wass_metric': wass_metric, 'gen_enc_loss': ['G+E', ge_l], 'disc_loss': ['D', d_l], 'zn_cycle_loss': ['$||Z_n-E(G(x))_n||$', c_zn], 'zc_cycle_loss': ['$||Z_c-E(G(x))_c||$', c_zc], 'img_cycle_loss': ['$||X-G(E(x))||$', c_i] }) train_df.to_csv('%s/training_details.csv' % (run_dir)) # Plot some training results plot_train_loss(df=train_df, arr_list=['gen_enc_loss', 'disc_loss'], figname='%s/training_model_losses.png' % (run_dir)) plot_train_loss( df=train_df, arr_list=['zn_cycle_loss', 'zc_cycle_loss', 'img_cycle_loss'], figname='%s/training_cycle_loss.png' % (run_dir)) # Save current state of trained models model_list = [discriminator, encoder, generator] save_model(models=model_list, out_dir=models_dir)
def path_splitter(s, _d_match=re.compile(r"\.\d").match): """ Split a string into its path components. Assumes a string is a path or is path-like. Parameters ---------- s : str | pathlib.Path Returns ------- split : tuple The path split by directory components and extensions. Examples -------- >>> tuple(path_splitter("this/thing.ext")) ({u}'this', {u}'thing', {u}'.ext') """ if has_pathlib and isinstance(s, PurePath): s = py23_str(s) path_parts = deque() p_appendleft = path_parts.appendleft # Continue splitting the path from the back until we have reached # '..' or '.', or until there is nothing left to split. path_location = s while path_location != os_curdir and path_location != os_pardir: parent_path = path_location path_location, child_path = path_split(parent_path) if path_location == parent_path: break p_appendleft(child_path) # This last append is the base path. # Only append if the string is non-empty. # Make sure the proper path separator for this OS is used # no matter what was actually given. if path_location: p_appendleft(py23_str(os_sep)) # Now, split off the file extensions using a similar method to above. # Continue splitting off file extensions until we reach a decimal number # or there are no more extensions. # We are not using built-in functionality of PathLib here because of # the recursive splitting up to a decimal. base = path_parts.pop() base_parts = deque() b_appendleft = base_parts.appendleft while True: front = base base, ext = path_splitext(front) if _d_match(ext) or not ext: # Reset base to before the split if the split is invalid. base = front break b_appendleft(ext) b_appendleft(base) # Return the split parent paths and then the split basename. return ichain(path_parts, base_parts)
def finalize(self): tn_map = self.tn_map names = ichain([""], sorted(k for k, etw in pyk.iteritems(self.et_map) if etw.e_type.has_identity)) for tid, tn in enumerate(names): tn_map.update(((tid, tn), (tn, tid)))
def _of_bases (self, cls) : result = TFL.defaultdict (list) for b in reversed (cls.__bases__) : for k, cs in pyk.iteritems (self.of_class (b)) : result [k] = list (uniq (ichain (result [k], cs))) return result
testdata = torch.utils.data.DataLoader( datasets.MNIST( "../../data/mnist", train=False, download=True, transform=transforms.Compose( [transforms.ToTensor()] ), ), batch_size=batch_size, shuffle=True, ) test_imgs, test_labels = next(iter(testdata)) test_imgs = Variable(test_imgs.type(Tensor)) ge_chain = ichain(generator.parameters(), encoder.parameters()) optimizer_GE = torch.optim.Adam(ge_chain, lr=lr, betas=(b1, b2), weight_decay=decay) optimizer_D = torch.optim.Adam(discriminator.parameters(), lr=lr, betas=(b1, b2)) # ---------- # Training # ---------- ge_l = [] d_l = [] c_zn = [] c_zc = [] c_i = [] # Training loop
def run(self): for reference, method_name in itertools.ichain(self.callbacks, self.single_callbacks): getattr(pride.objects[reference], method_name)() del self.single_callbacks[:]
def from_MT(cls, mt): args = tuple(ichain(*(c[:2] for c in mt.cols))) return cls(*args)
, "ß" : "ss" , "ä" : "ae" , "ö" : "oe" , "ü" : "ue" } _diacrit_rep = Dict_Replacer(_diacrit_map) _graph_rep = Re_Replacer \ ( "(%s)+" % "|".join (re.escape (c) for c in ("^!$%&([{}]) ?`'*+#:;<>|" '"')) , "_" ) _non_print_rep = Re_Replacer \ ( "|".join (re.escape (chr (i)) for i in ichain (range (0, 32), [127])) , "" ) _quote_map = \ { "«" : "<<" , "»" : ">>" , "´" : "'" , "\u2018" : "'" , "\u2019" : "'" , "\u201A" : "'" , "\u201B" : "'" , "\u201C" : '"' , "\u201D" : '"' , "\u201E" : '"' , "\u201F" : '"'
def value (self, o, renderer) : return ", ".join \ ( str (nw.net_address) for nw in ichain (o.ip4_networks, o.ip6_networks) )
def _main (cmd) : Sender = PMA.Sender_Tester if cmd.Debug else PMA.Sender smtp = Sender \ ( local_hostname = cmd.mail_local_hostname , mail_host = cmd.mail_host , mail_port = cmd.mail_port , password = cmd.mail_word , user = cmd.mail_user , use_tls = cmd.tls ) attach = sorted \ ( ichain ( ichain ( * tuple ( TFL.CAO.Rel_Path.resolved_paths ( cmd.msg_base_dirs, m , single_match = False , skip_missing = True ) for m in cmd.Attach_Message ) ) , cmd.attach ) ) bcc = None if cmd.Bcc : try : alias_mgr = PMA.Alias_Mgr \ ("/etc/aliases", "~/.aliases", "~/.mh_aliases") alias = alias_mgr [cmd.Bcc] bcc = str (alias) except KeyError : bcc = cmd.Bcc except Exception as exc : print (exc) comp = PMA.Composer \ ( cmd.editor, cmd.user, cmd.domain, smtp , attach = attach , bcc = bcc , debug = cmd.Debug , receiver = cmd.To , rst2html = cmd.HTML , short_body = cmd.Short_body , subject = cmd.subject ) def message_from_arg (cmd, arg) : try : return first (PMA.messages_from_args ([arg], cmd.msg_base_dirs)) except LookupError : raise SystemExit (1) if cmd.forward : comp.forward (message_from_arg (cmd, cmd.forward)) elif cmd.reply : comp.reply (message_from_arg (cmd, cmd.reply)) elif cmd.Reply_all : comp.reply_all (message_from_arg (cmd, cmd.Reply_all)) elif cmd.bounce : comp.resend (message_from_arg (cmd, cmd.bounce)) else : comp.compose ()
def __init__(self, x, y, config, ids=None, outcomes=None, client_id: int = 0, hardw_acc_flag: bool = False, output_folder=None): # Training details self.n_epochs = config['n_local_epochs'] self.lr = config['learning_rate'] self.b1 = config['beta_1'] self.b2 = config['beta_2'] self.decay = config['decay'] self.n_skip_iter = config['d_step'] # Data dimensions self.x_shape = config['x_shape'] # Latent space info self.latent_dim = config['latent_dim'] self.n_c = config['n_clusters'] self.betan = config['betan'] self.betac = config['betac'] # Wasserstein+GP metric flag self.wass_metric = config['wass_metric'] print('Using metric {}'.format( 'Wassestrain' if self.wass_metric else 'Vanilla')) self.cuda = True if torch.cuda.is_available( ) and hardw_acc_flag else False self.device = torch.device('cuda:0' if self.cuda else 'cpu') print('Using device {}'.format(self.device)) torch.autograd.set_detect_anomaly(True) # Loss function self.bce_loss = torch.nn.BCELoss() self.xe_loss = torch.nn.CrossEntropyLoss() self.mse_loss = torch.nn.MSELoss() # Initialize NNs if config['conv_net']: self.generator = ConvGeneratorCNN(self.latent_dim, self.n_c, self.x_shape) self.encoder = ConvEncoderCNN(self.latent_dim, self.n_c) self.discriminator = ConvDiscriminatorCNN( wass_metric=self.wass_metric) else: self.generator = GeneratorCNN(latent_dim=self.latent_dim, n_c=self.n_c, gen_dims=config['gen_dims'], x_shape=self.x_shape) self.encoder = EncoderCNN(latent_dim=self.latent_dim, enc_dims=config['enc_dims'], n_c=self.n_c) self.discriminator = DiscriminatorCNN( disc_dims=config['disc_dims'], wass_metric=self.wass_metric) if self.cuda: self.generator.cuda() self.encoder.cuda() self.discriminator.cuda() self.bce_loss.cuda() self.xe_loss.cuda() self.mse_loss.cuda() self.TENSOR = torch.cuda.FloatTensor if self.cuda else torch.FloatTensor # Configure data loader self.batch_size = config['batch_size'] train_idx, test_idx = split_dataset(x=x, splits=config['splits'], shuffle=config['shuffle'], fold_n=config['fold_n']) self.x_train = x[train_idx] self.y_train = y[train_idx] self.x_test = x[test_idx] self.y_test = y[test_idx] if outcomes is None or ids is None: self.trainloader = DataLoader(PrepareDataSimple(self.x_train, y=self.y_train), batch_size=self.batch_size) self.testloader = DataLoader(PrepareDataSimple(self.x_test, y=self.y_test), batch_size=self.batch_size) else: self.id_train = ids[train_idx] self.outcomes_train = outcomes[train_idx] self.id_test = ids[test_idx] self.outcomes_test = outcomes[test_idx] self.trainloader = DataLoader(PrepareData( x=self.x_train, y=self.y_train, ids=self.id_train, outcomes=self.outcomes_train), batch_size=self.batch_size) self.testloader = DataLoader(PrepareData( x=self.x_test, y=self.y_test, ids=self.id_test, outcomes=self.outcomes_test), batch_size=self.batch_size) self.ge_chain = ichain(self.generator.parameters(), self.encoder.parameters()) self.optimizer_GE = torch.optim.Adam(self.ge_chain, lr=self.lr, betas=(self.b1, self.b2), weight_decay=self.decay) self.optimizer_D = torch.optim.Adam(self.discriminator.parameters(), lr=self.lr, betas=(self.b1, self.b2)) # ---------- # Training # ---------- self.ge_l = [] self.d_l = [] self.c_zn = [] self.c_zc = [] self.c_i = [] # metrics self.img_mse_loss = None self.lat_mse_loss = None self.lat_xe_loss = None # leghts of NN parameters to send and receive self.g_w_l = len(self.generator.state_dict().items()) self.d_w_l = len(self.discriminator.state_dict().items()) self.e_w_l = len(self.encoder.state_dict().items()) # initiliazing to zero the federated epochs counter self.f_epoch = 0 # for saving images self.save_images = config['save_images'] self.client_id = client_id if output_folder is None: self.out_dir = output_folder self.img_dir = 'client_%d_images' % (self.client_id) os.makedirs(self.img_dir, exist_ok=True) else: self.out_dir = pathlib.Path(output_folder) os.makedirs(self.out_dir, exist_ok=True)
def words(self): return sorted(ichain(*(g8r.words for g8r in self.rereps)))
def _response_body (self, resource, request, response) : Bad_Req = resource.Status.Bad_Request req_data = request.req_data result = {} user = resource.user_restriction scope = resource.scope CNDB = scope.CNDB ipid = req_data.get ("interface") ppid = req_data.get ("pool") if ipid is None : raise Bad_Req (_T ("Request must include interface pid")) try : iface = scope.pid_query (ipid) except LookupError as exc : raise (Bad_Req (str (exc))) node = iface.my_node if ppid is not None : try : pool = scope.pid_query (ppid) except LookupError as exc : raise (Bad_Req (str (exc))) else : pools = [pool] else : def _query (ETM, node, user) : bitlen = ETM.left.net_address.P_Type.bitlen return ETM.query \ ( Q.OR ( Q.right.node == node , Q.AND ( ~ Q.right.node , Q.right.group_links.right.member_links .left == user # possibly, # a Q-expression checking various quotas ) ) & Q.OR ( ~ Q.right.netmask_interval.upper , Q.right.netmask_interval.upper == bitlen ) ).attr (Q.right).distinct () pools = list \ ( p for p in ichain ( _query (CNDB.IP4_Network_in_IP4_Pool, node, user) , _query (CNDB.IP6_Network_in_IP6_Pool, node, user) ) if p.can_allocate () ) n_pools = len (pools) ### XXX ### filter pools by quota ### if none remain ### --> send feedback about the full pools available template = resource.top.Templateer.get_template \ ("html/dashboard/app.m.jnj") if not n_pools : result ["feedback"] = _T \ ( "No pools available that allow allocation " "by user %(user)s and node %(node)s" % dict (user = user, node = node) ) elif n_pools == 1 : pool = TFL.first (pools) nw0 = TFL.first (pool.ip_networks) try : ipa = pool.allocate (nw0.net_address.bitlen, user) except Exception as exc : result ["feedback"] = str (exc) else : iii = scope.CNDB.Net_Interface_in_IP_Network \ (iface, ipa, mask_len = ipa.pool.net_address.mask) result ["row"] = template.call_macro \ ("e_type_object", resource, iii) else : result ["menu"] = template.call_macro \ ("action_button_allocate_ip_pool_menu", resource, pools) return result
def _of_bases(self, cls): result = TFL.defaultdict(list) for b in reversed(cls.__bases__): for k, cs in pyk.iteritems(self.of_class(b)): result[k] = list(uniq(ichain(result[k], cs))) return result
from _TFL import TFL from _TFL.pyk import pyk from _TFL.Regexp import Regexp, Re_Replacer, Dict_Replacer, re import _TFL.CAO from itertools import chain as ichain import unicodedata _diacrit_map = {"Ä": "Ae", "Ö": "Oe", "Ü": "Ue", "ß": "ss", "ä": "ae", "ö": "oe", "ü": "ue"} _diacrit_rep = Dict_Replacer(_diacrit_map) _graph_rep = Re_Replacer("(%s)+" % "|".join(re.escape(c) for c in ("^!$%&([{}]) ?`'*+#:;<>|" '"')), "_") _non_print_rep = Re_Replacer("|".join(re.escape(chr(i)) for i in ichain(range(0, 32), [127])), "") _quote_map = { "«": "<<", "»": ">>", "\u2018": "'", "\u2019": "'", "\u201A": "'", "\u201B": "'", "\u201C": '"', "\u201D": '"', "\u201E": '"', "\u201F": '"', "\u2039": "'", "\u203A": "'", }