def coupler_filter(self, coupler_set): coupler_sites = set([]) for (i, j) in coupler_set: coupler_sites.add(i) coupler_sites.add(j) filtered_sites = set([]) for site in self.sites: if site.index in coupler_sites: filtered_sites.add(site.index) filtered_couplers = [] for i, j in self.couplers: if (i.index in filtered_sites and j.index in filtered_sites): coupler = (i.index, j.index) if coupler in coupler_set: filtered_couplers.append(coupler) if len(filtered_couplers) != len(coupler_sites): print_err( 'warning: given a coupler set of size {} but found only {} active couplings from this set' .format(len(coupler_sites), len(filtered_couplers))) filtered_sites = set([]) for (i, j) in filtered_couplers: filtered_sites.add(i) filtered_sites.add(j) return ChimeraQPU(filtered_sites, filtered_couplers, self.cell_size, self.chimera_degree, self.site_range, self.coupler_range, self.chimera_degree_view, self.chip_id, self.endpoint, self.solver_name)
def _rescale(fields, couplings, offset, site_range, coupler_range): assert (site_range.lb + site_range.ub == 0.0) assert (coupler_range.lb + coupler_range.ub == 0.0) scaling_factor = 1.0 scale = 1.0 for field in fields.values(): if field != 0: if field < site_range.lb: scaling_factor = min(scaling_factor, site_range.lb / float(field)) if field > site_range.ub: scaling_factor = min(scaling_factor, site_range.ub / float(field)) for coupling in couplings.values(): if coupling != 0: if coupling < coupler_range.lb: scaling_factor = min(scaling_factor, coupler_range.lb / float(coupling)) if coupling > coupler_range.ub: scaling_factor = min(scaling_factor, coupler_range.ub / float(coupling)) if scaling_factor < 1.0: print_err( 'info: rescaling field to {} and couplings to {} with scaling factor {}' .format(site_range, coupler_range, scaling_factor)) fields = {k: v * scaling_factor for k, v in fields.items()} couplings = {k: v * scaling_factor for k, v in couplings.items()} offset = offset * scaling_factor scale = 1 / scaling_factor return fields, couplings, offset, scale
def main(): parser = argparse.ArgumentParser() parser.add_argument('edges_filename', default='interim/customer_product_counts.csv') parser.add_argument('customers_filename', default='data/publicChallenge.csv') parser.add_argument('-o', '--output', default='rwalks.probas') parser.add_argument('-l', '--walk-length', default=2, type=int) parser.add_argument('-w', '--no-walks', default=20, type=int) parser.add_argument('-N', '--top-n', default=20, type=int) parser.add_argument('-s', '--seed', default=101, type=int) args = parser.parse_args() rw = RandomWalks(args.seed) # customers, products, G = nx_load_data(edges_filename) common.print_err("Loading...") rw.load(args.edges_filename) common.print_err("Loaded") # customers_to_predict = customers # with open(customers_to_predict_filename, 'rb') as f: # customers_to_predict = ['c'+cid.strip() for cid in f if cid.strip() != ""] customers_to_predict = customer.load_ids(args.customers_filename, add_prefix='c') # walk_length = 2 # no_of_walks = 20 # 100 rw.predict(customers_to_predict, args.walk_length, args.no_walks, args.top_n, args.output)
def handle_request(request): print_err("access: " + request.fields[0]) print_err("params: " + request.fields[2]) try: if request.is_error: return handle_err(request) else: return handle_dat(request) except (SecurityViolation, InvalidRequest) as e: fields = map(str, e.args) return unet_request.Request(*fields, is_error=True) except Exception as e: traceback.print_exc()
def back_track(findme): '''given a depth ( 0 = subsec, 4 = sub3para) [findme] cycles backwards through list [alist] and returns information found in bracket at that sublevel. (e.g. "(iii) A tenant shall..." -> "iii" ) If there's no brackets, returns section itself e.g. "90.100"''' num = 0 while True: if alist[cnt-1-num][0] == findme: if (in_bracs(alist[cnt-1-num][1])) is None: return alist[cnt-1-num][1] else: return in_bracs(alist[cnt-1-num][1]) num += 1 if cnt-1-num < 0: return "NOT FOUND" print_err(0, f'(Did not find item of depth of {findme} anywhere before line {cnt-1}.)')
def sub_levels(depth, parent, data): # text, cls): global subs_list if in_bracs(data[1]) == start[depth]: try: subs_list[depth] = parent.ol(class_='lvl' + str(data[0])) except Exception as e: print_err( e, f'Couldn\'t add new depth {depth} for #{data[0]} for: {data[1]} at {data[2]}' ) return False try: subs_list[depth].li(data[1], title=str(data[2])) return True except Exception as e: print_err( e, f'Couldn\'t add {data[0]} with depth {depth} for: {data[2]}') return False
def handle_dat(request): allowed_directory = "global/" file = allowed_directory + request.fields[0] args = request.fields[2:] if not in_directory(file, allowed_directory): raise SecurityViolation("Forbidden", request.fields[0]) if is_executable(file): p = subprocess.run([file] + args, input=request.fields[1].encode("utf-8") if len(request.fields) >= 1 else b'', capture_output=True) if len(p.stderr) > 0: print_err("stderr: " + p.stderr.decode("utf-8")) return unet_request.Request(p.stdout) elif is_readable(file): with open(file, "rb") as f: return unet_request.Request(f.read()) else: raise InvalidRequest("Not Found", file[len(allowed_directory) - 1:])
def fix_subs(cur, lst): if cur == 'eL': # for capital L, scroll up until you find an 'h/H' for back in range(count): if in_bracs(lst[(count - back)][1]) == 'h': return 2 if in_bracs(lst[(count - back)][1]) == 'H': return 3 elif cur == 'romanish': # for ambiguous roman characters # TODO see todos below combine these a little better? Make separate function. back_one = lst[(count - 1)] # first look back one paragraph, that may answer it if back_one[0] == 4 or back_one[0] == 5: # if (i) or (I)... return 4 elif back_one[0] == 3: # if a subpara (A), did last para (a) match previous letter in alphabet? for back in range(count): if lst[back][0] == 2: if in_bracs(lst[(count - back)][1]) == prior_para[in_bracs(lst[count][1])]: return 2 # e.g., (i) is just a para after (h) else: return 4 # e.g., (i) is start of roman numeral list i, ii, iii... else: return 2 elif cur == 'ROMANISH': # for ambiguous ROMAN characters, look back one paragraph. back_one = lst[(count - 1)][0] if back_one == 5: # if it's a sub3 para (I), then this too is sub3para (II). # TODO not necesarily. Should still check. Could be (g), **(h),** (A), (B), (i), (ii), **(i),** (j)... return 5 elif back_one == 4: # if a sub2para (i), did last subpara (A) match previous letter? for back in range(count, 1, -1): # TODO could probalby use backward count above and for entire piece where we need to look back. # todo see back_track function elsewhere & maybe incorporate it if lst[back][0] == 3: if in_bracs(lst[back][1]) == prior_para[in_bracs(lst[count][1]).lower()]: return 3 else: return 5 else: return 3 elif cur == 'dunno': print_err(f'unclassified line ("dunno") remains', f'Line #{count} for {cur}') return cur else: return cur
def check_index(il, lst): global count if il[0] == 'subtitle': # replace subtitles coming after index terms as within index. try: temp = 0 while lst[count + temp][0] == 'sub2title' or lst[count + temp][0] == 'subtitle' or \ lst[count + temp][0] == 'dunno' or lst[count + temp][0] == 'index': if lst[count + temp][0] == 'index': il[0] = 'index_sub' break temp += 1 except Exception as e: print_err(e, f'line #{count} for: {il}') if il[0] == 'sub2title' or il[0] == 'dunno': # replace subtitles coming after index terms as within index. try: temp = 0 while lst[count + temp][0] == 'sub2title' or lst[count + temp][0] == 'subtitle' or \ lst[count + temp][0] == 'dunno' or lst[count + temp][0] == 'index': if lst[count + temp][0] == 'index': il[0] = 'index2sub' temp += 1 except Exception as e: print_err(e, f'sub2title -> index reclass in line# {count} for: {il}')
def load_config(args): config_file_path = args.config_file if os.path.isfile(config_file_path): with open(config_file_path, 'r') as config_file: try: config_data = json.load(config_file) for key, value in config_data.items(): if isinstance(value, dict): print_err( 'invalid value for configuration key "%s", only single values are allowed' % config_file_path) quit() if not hasattr(args, key) or getattr(args, key) == None: if isinstance(value, unicode): value = value.encode('ascii', 'ignore') if isinstance(value, list): new_list = [] for item in value: if isinstance(item, unicode): item = item.encode('ascii', 'ignore') new_list.append(item) value = new_list setattr(args, key, value) else: print_err( 'skipping the configuration key "%s", it already has a value of %s' % (key, str(getattr(args, key)))) except ValueError: print_err( 'the config file does not appear to be a valid json document: %s' % config_file_path) quit() else: if config_file_path != DEFAULT_CONFIG_FILE: print_err('unable to open conifguration file: %s' % config_file_path) quit() return args
def get_split_i(splitter, customer_ids, min_x, min_y): for customer_id, customer_records in customer.get_mult_records_i(customer_ids): if len(customer.get_unique_times(customer_records)) < 2: common.print_err("Skipped:", customer_id) continue x_orders_set, y_row = splitter.split(customer_records) if len(x_orders_set) < min_x: common.print_err("Skipped due to x too small: {} ({})".format(customer_id, len(x_orders_set))) continue if len(y_row) < min_y: common.print_err("Skipped due to y too small: {} ({})".format(customer_id, len(y_row))) continue yield customer_id, x_orders_set, y_row
def cur_children(line, my_div): global err global slug # todo carve out form piece below into new section global form if line[0] == 'form_start': # trying to create new form box based on parentage try: form_id = line[ 2] # pulls in parent of form type from prior line todo: although why not just do that now? if form_id == 0: form = slug.div(class_='form-box') elif str(form_id).isnumeric(): if 1 <= form_id <= 5: form = subs_list[form_id - 1].div(class_='form-box') else: form = my_div.div(line[1], class_='form-box') print_err(f'Form not created within section text', f'at {line}') except Exception as e: print_err({e}, f'form failed at {line}') return True # // end form carve out elif line[0] == 0: slug = my_div.p(line[1], class_='flush') return True elif line[0] == 'form': try: form.p(line[1], class_='form-box') except Exception as e: print_err({e}, f'attempted to add form from {line} to form box') return True elif str(line[0]).isnumeric(): if line[0] == 1: return sub_levels(0, my_div, line) if 2 <= line[0] <= 5: return sub_levels(line[0] - 1, subs_list[line[0] - 2], line)
def build_case(args): if not args.seed is None: print_err('setting random seed to: {}'.format(args.seed)) random.seed(args.seed) #print_err(args.chimera_edge_set) qpu = get_qpu(args.profile, args.ignore_connection, args.hardware_chimera_degree) #print_err(qpu) if args.chimera_degree != None: print_err('filtering QPU to chimera of degree {}'.format( args.chimera_degree)) qpu = qpu.chimera_degree_filter(args.chimera_degree) if args.chimera_cell_limit != None: print_err('filtering QPU to the first {} chimera cells'.format( args.chimera_cell_limit)) qpu = qpu.cell_filter(args.chimera_cell_limit) if args.chimera_cell_box != None: chimera_cell_1 = tuple(args.chimera_cell_box[0:2]) chimera_cell_2 = tuple(args.chimera_cell_box[2:4]) print_err('filtering QPU to the chimera cell box {} by {}'.format( chimera_cell_1, chimera_cell_2)) qpu = qpu.chimera_cell_box_filter(chimera_cell_1, chimera_cell_2) if args.spin_set != None: print_err('filtering QPU to the spin set {}'.format(args.spin_set)) qpu = qpu.spin_filter(args.spin_set) if args.coupler_set != None: print_err('filtering QPU to the coupler set {}'.format( args.coupler_set)) qpu = qpu.coupler_filter(args.coupler_set) if args.generator == 'const': qpu_config = generator.generate_disordered( qpu, [args.coupling], [1.0], [args.field], [1.0], args.random_gauge_transformation) elif args.generator == 'ran': qpu_config = generator.generate_ran(qpu, args.probability, args.steps, args.field, args.scale, args.simple_ground_state) elif args.generator == 'gd': qpu_config = generator.generate_disordered( qpu, args.coupling_values, args.coupling_probabilities, args.field_values, args.field_probabilities, args.random_gauge_transformation) elif args.generator == 'cbfm': qpu_config = generator.generate_disordered( qpu, [args.j1_val, args.j2_val], [args.j1_pr, args.j2_pr], [args.h1_val, args.h2_val], [args.h1_pr, args.h2_pr], args.random_gauge_transformation) elif args.generator == 'fl': qpu_config = generator.generate_fl( qpu, args.steps, args.alpha, args.multicell, args.cluster_chimera_cells, args.simple_ground_state, args.min_loop_length, args.loop_reject_limit, args.loop_sample_limit) elif args.generator == 'wscn': if args.chimera_cell_limit != None: print_err( 'weak-strong cluster networks cannot be constricted with a cell limit.' ) quit() if qpu.chimera_degree_view < 6: print_err( 'weak-strong cluster networks require a qpu with chimera degree of at least 6, the given degree is {}.' .format(qpu.chimera_degree_view)) quit() effective_chimera_degree = 3 * (qpu.chimera_degree_view // 3) if effective_chimera_degree != qpu.chimera_degree_view: print_err( 'the weak-strong cluster network will occupy a space of chimera degree {}.' .format(effective_chimera_degree)) qpu = qpu.chimera_degree_filter(effective_chimera_degree) qpu_config = generator.generate_wscn(qpu, args.weak_field, args.strong_field) elif args.generator == 'fclg': qpu_config = generator.generate_fclg(qpu, args.steps, args.alpha, args.gadget_fraction, args.simple_ground_state, args.min_loop_length, args.loop_reject_limit, args.loop_sample_limit) else: assert (False) # CLI failed if args.include_zeros: config = qpu_config if isinstance(qpu_config, QPUAssignment): config = qpu_config.qpu_config for site in config.qpu.sites: if not site in config.fields: config.fields[site] = 0.0 for coupler in config.qpu.couplers: if not coupler in config.couplings: config.couplings[coupler] = 0.0 #print_err(qpu_config) if args.omit_solution: if isinstance(qpu_config, QPUAssignment): qpu_config = qpu_config.qpu_config data = qpu_config.build_dict(args.include_zeros) data[ 'description'] = 'This is a randomly generated B-QP built by D-WIG (https://github.com/lanl-ansi/dwig) using the {} algorithm.'.format( args.generator) if not args.seed is None: data['description'] = data[ 'description'] + ' A random number seed of {} was used.'.format( args.seed) data['metadata'] = build_metadata(args, qpu) return data
def get_qpu(profile, ignore_connection, hardware_chimera_degree): chip_id = None endpoint = None solver_name = None cell_size = 8 global _qpu_remote if not ignore_connection: if _qpu_remote == None: try: with Client.from_config(config_file=os.getenv("HOME") + "/dwave.conf", profile=profile) as client: endpoint = client.endpoint solver = client.get_solver() solver_name = solver.name couplers = solver.undirected_edges sites = solver.nodes solver_chimera_degree = int( math.ceil(math.sqrt(len(sites) / cell_size))) if hardware_chimera_degree != solver_chimera_degree: print_err( 'Warning: the hardware chimera degree was specified as {}, while the solver {} has a degree of {}' .format(hardware_chimera_degree, solver_name, solver_chimera_degree)) hardware_chimera_degree = solver_chimera_degree site_range = Range(*solver.properties['h_range']) coupler_range = Range(*solver.properties['j_range']) chip_id = solver.properties['chip_id'] #TODO remove try/except logic, if there is a better way to check the connection except Exception as e: print_err( 'QPU connection details not found or there was a connection error' ) print_err(' ' + str(e)) print_err( 'assuming full yield square chimera of degree {}'.format( hardware_chimera_degree)) ignore_connection = True else: print_err('info: using cached QPU details') return _qpu_remote if ignore_connection: site_range = Range(-2.0, 2.0) coupler_range = Range(-1.0, 1.0) # the hard coded 4 here assumes an 4x2 unit cell graph = dwave_networkx.chimera_graph(hardware_chimera_degree, hardware_chimera_degree, cell_size // 2) edges = graph.edges() #arcs = get_chimera_adjacency(hardware_chimera_degree, hardware_chimera_degree, cell_size//2) #print(arcs) # turn arcs into couplers # this step is nessisary to be consistent with the solver.properties['couplers'] data couplers = [] for i, j in edges: assert (i != j) if i < j: couplers.append((i, j)) else: couplers.append((j, i)) couplers = set(couplers) sites = set([coupler[0] for coupler in couplers] + [coupler[1] for coupler in couplers]) # sanity check on coupler consistency across both branches for i, j in couplers: assert (i < j) if _qpu_remote == None: _qpu_remote = ChimeraQPU(sites, couplers, cell_size, hardware_chimera_degree, site_range, coupler_range, chip_id=chip_id, endpoint=endpoint, solver_name=solver_name) return _qpu_remote
def learn( self, x, validation_split=0.0, validation_data=None, log_file_prefix=None, per_epoch_callback_funs=[], callbacks=[], verbose='print_epochs', #one of: 'none' # 'print_epochs' # 'progress_bars' deadline=None, max_duration=None): optim_fun = getattr(optim, self.params.optimizer) optimizer = optim_fun(self.model.parameters(), **self.params.optimizer_params) # optional validation split if validation_split > 0: assert validation_data is None perm = np.random.permutation(x.shape[0]) split_idx = round(validation_split * x.shape[0]) val, train = perm[:split_idx], perm[split_idx:] validation_data, x = x[val, :], x[train, :] validation = (validation_data is not None) # learn data normalization self.normalizer.learn(x) x = self.normalizer.normalize(x) if validation: validation_data = self.normalizer.normalize(validation_data) # optionally add callbacks keras_callbacks = [] # 'built-in' callbacks '''if log_file_prefix: #keras_callbacks.append(keras.callbacks.CSVLogger(log_file_prefix + ".log")) if self.params.log_weights: keras_callbacks.append(WeightLogger(self.vae_loss, log_file_prefix)) if self.params.log_loss: keras_callbacks.append(LossLogger(log_file_prefix, per_patch=self.log_loss_per_patch)) if self.params.log_loss and validation: keras_callbacks.append(LossLogger(log_file_prefix, loss='val_loss')) # externally defined keras callback objects for callback in callbacks: keras_callbacks.extend(callbacks) # externally defined callbacks functions for callback in per_epoch_callback_funs: class CB(keras.callbacks.Callback): def on_train_begin(self, logs={}): callback() def on_epoch_end(self, epoch, logs={}): callback() keras_callbacks.append(CB()) if self.params.early_stopping: if self.params.early_stopping == True: monitor = ('val_loss' if validation else 'loss') else: monitor = self.params.early_stopping keras_callbacks.append(keras.callbacks.EarlyStopping( monitor=monitor, patience=self.params.early_stopping_patience)) if self.reduce_lr_on_plateau: if self.reduce_lr_on_plateau == True: monitor = ('val_loss' if validation else 'loss') else: monitor = self.reduce_lr_on_plateau keras_callbacks.append(keras.callbacks.ReduceLROnPlateau( monitor=monitor, factor=self.reduce_lr_factor, patience=self.params.reduce_lr_patience))''' '''if verbose == 'none': verbose = 0 elif verbose == 'print_epochs': verbose = 2 elif verbose == 'progress_bars': verbose = 1 else: assert False # invalid verbosity''' loss_fun = vae_loss if self.params.prediction_log_var_min is not None: # TODO: Now reqularization strength depends on the type (size) of # x_log_var. Should probably unify? x_log_var_regularizer = lambda xlv: (torch.sum( F.relu(self.params.prediction_log_var_min - xlv))) loss_fun = lambda x, x_mean, x_log_var, z_mean, z_log_var: ( vae_loss(x, x_mean, x_log_var, z_mean, z_log_var ) + x_log_var_regularizer(x_log_var)) if self.cuda: import torch.backends.cudnn as cudnn #cudnn.benchmark = True if max_duration is not None: new_dl = datetime.datetime.now() + max_duration if deadline is None or new_dl < deadline: deadline = new_dl #train progress_interval = 100 n_samples = x.shape[0] n_patches = n_samples // self.params.batch_size best_es_target_loss = float('inf') best_state = self.model.state_dict() epochs_no_improvement = 0 #print(self.model) start_time = time.perf_counter() for epoch in range(self.params.n_epochs): #print(self.model.state_dict()) # train epoch_start_time = time.perf_counter() self.model.train() train_loss = 0 perm = np.random.permutation(x.shape[0]) batches = [ perm[i:i + self.params.batch_size] for i in range(0, len(perm), self.params.batch_size) ] for batch_idx, batch_rows in enumerate(batches): batch_data = torch.from_numpy(x[batch_rows, :]).float() batch_data = Variable(batch_data) if self.cuda: batch_data = batch_data.cuda() optimizer.zero_grad() recon_mean, recon_log_var, z_mean, z_log_var = self.model( batch_data) loss = loss_fun(batch_data, recon_mean, recon_log_var, z_mean, z_log_var) loss.backward() train_loss += loss.data[0] optimizer.step() if (batch_idx + 1) % progress_interval == 0: #print(loss.data[0]) print_err( 'Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'. format(epoch, batch_idx * self.params.batch_size, n_samples, 100. * batch_idx / len(batches), loss.data[0] / len(batch_rows)), flush=True) train_loss /= n_samples print_err("Epoch: %4d Avg. loss: %.3f" % (epoch, train_loss), end='', flush=True) # validation if validation: self.model.eval() val_loss = 0 log_prob = 0 kl_dist = 0 perm = range(validation_data.shape[0]) batches = [ perm[i:i + self.params.batch_size] for i in range(0, len(perm), self.params.batch_size) ] for batch_idx, batch_rows in enumerate(batches): batch_data = torch.from_numpy( validation_data[batch_rows, :]).float() batch_data = Variable(batch_data, volatile=True) if self.cuda: batch_data = batch_data.cuda() recon_mean, recon_log_var, z_mean, z_log_var = self.model( batch_data) loss = loss_fun(batch_data, recon_mean, recon_log_var, z_mean, z_log_var) val_loss += loss.data[0] log_prob += torch.sum( gaussian_log_prob(batch_data, recon_mean, recon_log_var)).data[0] kl_dist += torch.sum( std_gaussian_kl_dist(z_mean, z_log_var)).data[0] val_loss /= validation_data.shape[0] log_prob /= validation_data.shape[0] kl_dist /= validation_data.shape[0] print_err(" Val. loss: %.3f (re: %.3f, kl: %.3f)" % (val_loss, log_prob, kl_dist), end='', flush=True) epoch_elapsed = time.perf_counter() - epoch_start_time elapsed_str = str(datetime.timedelta(seconds=epoch_elapsed)) print_err(" Time: %s" % (elapsed_str), flush=True) for cb in per_epoch_callback_funs: cb() # early stopping (loss not improving) if self.params.early_stopping: if self.params.early_stopping_target == 'auto': if validation: es_target_loss = val_loss else: es_target_loss = loss elif self.params.early_stopping_target == 'val_loss': es_target_loss = val_loss elif self.params.early_stopping_target == 'loss': es_target_loss = train_loss else: assert False, "invalid early_stopping_target" if es_target_loss < best_es_target_loss: best_es_target_loss = es_target_loss best_state = self.model.state_dict() epochs_no_improvement = 0 else: epochs_no_improvement += 1 if epochs_no_improvement > self.params.early_stopping_patience: self.model.load_state_dict(best_state) print("Early stopping as no improvement in %d epochs" % (self.params.early_stopping_patience)) break # time based stopping if datetime.datetime.now() >= deadline: print_err('Epoch %d: Time based stopping' % (epoch)) break #print(self.model.state_dict()) '''self.vae_loss.fit(x, x,
def build_models(self): class VAEModel(nn.Module): def __init__(self, params): super(VAEModel, self).__init__() self.params = params # encoding layers self.enc_layers = nn.ModuleList() enc_dims = list(self.params.enc_dims) enc_activations = auto_expand(self.params.enc_activations) if self.params.input_dropout: self.enc_layers.append( nn.Dropout(self.params.input_dropout)) last_dim = self.params.input_dim for i in range(self.params.n_enc): #add_bias_terms = not self.params.batch_normalization add_bias_terms = True self.enc_layers.append( nn.Linear(last_dim, enc_dims[i], bias=add_bias_terms)) #'enc_hidden_%d'%(i+1) if i == 0: self.first_dense_enc_layer = self.enc_layers[-1] #if self.params.batch_normalization: # self.enc_layers.append(BatchNormalization(mode=2)) self.enc_layers.append( create_activation(enc_activations[i])) if self.params.enc_dropout: self.enc_layers.append( nn.Dropout( auto_expand(self.params.enc_dropout)[i])) last_dim = enc_dims[i] # latent layers self.enc_z_mean = nn.Linear(last_dim, self.params.latent_dim) self.enc_z_log_var = nn.Linear(last_dim, self.params.latent_dim) #self.enc_z_draw = Gaussian() # decoding layers self.dec_layers = nn.ModuleList() dec_dims = list(self.params.dec_dims) dec_activations = auto_expand(self.params.dec_activations) if self.params.latent_dropout: self.dec_layers.append(Dropout(self.params.latent_dropout)) last_dim = self.params.latent_dim for i in range(self.params.n_dec): #add_bias_terms = not (self.params.batch_normalization and i != self.params.n_dec) add_bias_terms = True self.dec_layers.append( nn.Linear(last_dim, dec_dims[i], bias=add_bias_terms)) if i == self.params.n_dec: self.last_dense_dec_layer = self.dec_layers[-1] #if self.params.batch_normalization and i != self.params.n_dec: # self.dec_layers.append(BatchNormalization(mode=2)) self.dec_layers.append( create_activation(dec_activations[i])) if self.params.dec_dropout: self.dec_layers.append( nn.Dropout( auto_expand(self.params.dec_dropout)[i])) last_dim = dec_dims[i] # prediction layers self.dec_x_mean = nn.Linear(last_dim, self.params.input_dim) self.dec_x_mean_act = None if self.params.prediction_mean_activation: self.dec_x_mean_act = create_activation( self.params.prediction_mean_activation) #pvname = 'pred_log_var' if self.params.prediction_var in [ 'persample_independent', 'pi' ]: self.dec_x_log_var = nn.Linear(last_dim, self.params.input_dim) self.dec_x_log_var.weight.data.zero_() elif self.params.prediction_var in ['persample_same', 'ps']: self.dec_x_log_var = nn.Linear(last_dim, 1) self.dec_x_log_var.weight.data.zero_() elif self.params.prediction_var in [ 'global_independent', 'gi' ]: #self.dec_x_log_var = Const(shape=(self.params.input_dim,), trainable=True, name=pvname) self.dec_x_log_var = nn.Parameter( torch.Tensor(self.params.input_dim)) self.dec_x_log_var.data.zero_() elif self.params.prediction_var in ['global_same', 'gs']: self.dec_x_log_var = nn.Parameter(torch.Tensor(1)) self.dec_x_log_var.data.zero_() else: self.dec_x_log_var = Variable(torch.from_numpy( self.params.prediction_var).float(), requires_grad=False) #self.dec_x_log_var = torch.from_numpy(self.params.prediction_var).float() #self.dec_x_draw = Gaussian(name='pred_draw') self.dec_x_log_var_act = None if self.params.prediction_log_var_activation: self.dec_x_log_var_act = create_activation( self.params.prediction_log_var_activation) # normalization layers #if self.params.normalize_input_scale is not None: # scale = K.variable(self.params.normalize_input_scale) # self.enc_layers.insert(0, Lambda(lambda x: # x / scale)) # self.dec_layers.append(Lambda(lambda x: # x * scale)) #if self.params.normalize_input_mean is not None: # mean = K.variable(self.params.normalize_input_mean) # self.enc_layers.insert(0, Lambda(lambda x: # x - mean)) # self.dec_layers.append(Lambda(lambda x: # x + mean)) self.cuda_ = False def cuda(self): self.cuda_ = True super(VAEModel, self).cuda() def cpu(self): self.cuda_ = False super(VAEModel, self).cpu() def encode(self, x): encoded = x for layer in self.enc_layers: encoded = layer(encoded) encoded_z_mean = self.enc_z_mean(encoded) encoded_z_log_var = self.enc_z_log_var(encoded) return encoded_z_mean, encoded_z_log_var def decode(self, z): decoded = z for layer in self.dec_layers: decoded = layer(decoded) decoded_x_mean = self.dec_x_mean(decoded) if self.dec_x_mean_act: decoded_x_mean = self.dec_x_mean_act(decoded_x_mean) if isinstance(self.dec_x_log_var, nn.Module): decoded_x_log_var = self.dec_x_log_var(decoded) else: decoded_x_log_var = self.dec_x_log_var if self.dec_x_log_var_act: decoded_x_log_var = self.dec_x_log_var_act( decoded_x_log_var) return decoded_x_mean, decoded_x_log_var def forward(self, x): #encoded_z_mean, encoded_z_log_var = self.encode(x.view(-1, self.params.input_dim)) encoded_z_mean, encoded_z_log_var = self.encode(x) z = rand_normal_reparametrised(encoded_z_mean, encoded_z_log_var) return self.decode(z) + (encoded_z_mean, encoded_z_log_var) self.model = VAEModel(self.params) if torch.cuda.is_available() and torch.cuda.device_count() > 0: print_err("GPU available -> using CUDA", flush=True) self.cuda = True self.model.cuda() else: self.cuda = False print_err("no GPU available -> using CPU", flush=True) '''kl_dist = StdGaussianKLDist(name='kl_dist') \
def build_sections(sections_list, mn): global sec_div global cur_div global form global err global subs_list ll, sec_div, cur_div, form, err, = None, None, None, None, None counter = 0 for line in sections_list[1:]: counter += 1 if line[0] == 'index' or line[0] == 'index_sub' or line[ 0] == 'index2sub': pass elif line[0] == 'subtitle': mn.h3(line[1], class_='subtitle') elif line[0] == 'sub2title': mn.h4(line[1], class_='sub2title') elif line[0] == 'or_sec': sec_div = mn.div(id=line[1], class_='section') ll = sec_div.p(line[1], class_='or_sec') err = None cur_div = sec_div reboot_cur() elif line[0] == 'temp_sec': sec_div = mn.div(class_='section temporary') ll = sec_div.p(line[1], class_='or_sec') cur_div = sec_div reboot_cur() elif line[0] == 'da_sec': alt_div = cur_div.div(class_='alternate') cur_div = alt_div cur_div.p(line[1], class_='or_sec') reboot_cur() elif line[0] == 'leadline': ll.span(' ' + line[1], class_='leadline') elif line[0] == 'note_next': mn.p(line[1], class_='note') else: if sec_children(line, sec_div): pass elif cur_children(line, cur_div): pass else: print_err('Generated HTML for unclassified line', f'For {line}') if err is None: try: err = cur_div.div(class_='unknown') except Exception as e: print_err(e, f'creating new error div failed for {line}') err = mn.div(class_='unknown') err.p("** WARNING ** ", style='font-weight:bold;text-align:center') err.p( "Lines below are part of this section, but may not have parsed correctly.", style='font-weight:bold;text-align:center') ptemp = err.p( "View original from the ", style= 'font-weight:bold;text-align:center;margin-bottom:20px' ) ptemp.a( 'Oregon Legislature Website', href= 'https://www.oregonlegislature.gov/bills_laws/Pages/ORS.aspx' ) ptemp += '.' try: err.p(line[1]) except Exception as e: print_err( e, f' adding to existing err message failed for {line}')
def get_qpu(url, token, proxy, solver_name, hardware_chimera_degree): chip_id = None cell_size = 8 if not url is None and not token is None and not solver_name is None: print_err( 'QPU connection details found, accessing "{}" at "{}"'.format( solver_name, url)) if proxy is None: remote_connection = RemoteConnection(url, token) else: remote_connection = RemoteConnection(url, token, proxy) solver = remote_connection.get_solver(solver_name) couplers = solver.properties['couplers'] couplers = set([tuple(coupler) for coupler in couplers]) sites = solver.properties['qubits'] solver_chimera_degree = int( math.ceil(math.sqrt(len(sites) / cell_size))) if hardware_chimera_degree != solver_chimera_degree: print_err( 'Warning: the hardware chimera degree was specified as {}, while the solver {} has a degree of {}' .format(hardware_chimera_degree, solver_name, solver_chimera_degree)) hardware_chimera_degree = solver_chimera_degree site_range = Range(*solver.properties['h_range']) coupler_range = Range(*solver.properties['j_range']) chip_id = solver.properties['chip_id'] else: print_err( 'QPU connection details not found, assuming full yield square chimera of degree {}' .format(hardware_chimera_degree)) site_range = Range(-2.0, 2.0) coupler_range = Range(-1.0, 1.0) # the hard coded 4 here assumes an 4x2 unit cell arcs = get_chimera_adjacency(hardware_chimera_degree, hardware_chimera_degree, cell_size / 2) # turn arcs into couplers # this step is nessisary to be consistent with the solver.properties['couplers'] data couplers = [] for i, j in arcs: assert (i != j) if i < j: couplers.append((i, j)) else: couplers.append((j, i)) couplers = set(couplers) sites = set([coupler[0] for coupler in couplers] + [coupler[1] for coupler in couplers]) # sanity check on coupler consistency across both branches for i, j in couplers: assert (i < j) return ChimeraQPU(sites, couplers, cell_size, hardware_chimera_degree, site_range, coupler_range, chip_id=chip_id)