def format_event_data_group_update(data): """Return a structure describing a group update event.""" event_data = ns() event_data.member_lists = list() event_data.header = "Group update" if data.change_by: event_data.header += " by " + format_message(data.change_by.name) if data.group_name: event_data.name = data.group_name if data.new_members and len(data.new_members) > 0: member_list = format_member_list("New members:", data.new_members) event_data.member_lists.append(member_list) if data.deleted_members and len(data.deleted_members) > 0: member_list = format_member_list( "Deleted members:", data.deleted_members ) event_data.member_lists.append(member_list) if data.members and len(data.members) > 0: member_list = format_member_list("Members:", data.members) event_data.member_lists.append(member_list) return event_data
def try_clustering(model, pairwise_similarity, names=None, verbose=True): pairwise_distance = pairwise_similarity.max() - pairwise_similarity output_ids = model.fit_predict(pairwise_distance) unk_labels, cluster_sizes = np.unique(output_ids, return_counts=True) num_clusters = np.count_nonzero(unk_labels >= 0) non_outliers = np.flatnonzero(output_ids >= 0) num_outliers = len(output_ids) - len(non_outliers) if len(non_outliers): score = silhouette_score(pairwise_distance[np.ix_( non_outliers, non_outliers)], output_ids[non_outliers], metric='precomputed') else: score = -1 result = ns(score=score, num_clusters=num_clusters, labels=output_ids, num_outliers=num_outliers, unk_labels=unk_labels, cluster_sizes=cluster_sizes) if verbose: show_clusters(result, names) return result
def search(self, pat, line, num): found = search(pat, line) if found: grp = found.groupdict() grp['line'] = line grp['num'] = num return ns(**grp)
def parse(message: str): details = False if message.endswith('?'): details = True message = message[:-1] components = is_this_for_me(message) if not components: return None if components == ['apua']: return ns(action='apua') modifier = 0 try: modifier = int(components[-1]) components = components[:-1] except IndexError: return except ValueError: pass try: if components[-1] in ('+', '-'): if components[-1] == '-': modifier = -modifier components = components[:-1] except IndexError: return move = ' '.join(components) if not move: return return ns( action='move', move=move, modifier=modifier, details=details, )
def test_multiply(self): point = Point2D(2, 3) tests = [ (Point2D(1, 1), (2, 3)), ((1, 1), (2, 3)), ((0.5, 4.5), (1, 13.5)), (1, (2, 3)), (1.0, (2, 3)), (ns(x=1, y=1), (2, 3)), ] for other, expected in tests: with self.subTest(other): result = point * other self.assertEqual(result, expected)
def test_subtract(self): point = Point2D(2, 3) tests = [ (Point2D(1, 1), (1, 2)), ((1, 1), (1, 2)), ((0.5, 4.5), (1.5, -1.5)), (1, (1, 2)), (1.0, (1, 2)), (ns(x=1, y=1), (1, 2)), ] for other, expected in tests: with self.subTest(other): result = point - other self.assertEqual(result, expected)
def test_divide(self): point = Point2D(2, 3) tests = [ (Point2D(1, 1), (2, 3)), ((1, 1), (2, 3)), ((0.5, 6.0), (4, 0.5)), (1, (2, 3)), (1.0, (2, 3)), (ns(x=1, y=1), (2, 3)), ] for other, expected in tests: with self.subTest(other): result = point / other self.assertEqual(result, expected)
def test_floordiv(self): point = Point2D(2, 3) tests = [ (Point2D(1, 1), (2, 3)), ((1, 1), (2, 3)), ((0.4, -6.0), (4.0, -1.0)), (1, (2, 3)), (1.0, (2, 3)), (ns(x=1, y=1), (2, 3)), ] for other, expected in tests: with self.subTest(other): result = point // other self.assertEqual(result, expected)
def format_member_list(header: str, member_list): """Return a list of printable group members belonging to a category (e.g. new members).""" people = ns() people.header = header people.members = list() for member in member_list: designation = "" if not member.match_from_phone: if member.phone: designation = f"{format_message(member.name)} ({format_message(member.phone)})" else: designation = f"{format_message(member.name)}" else: if member.name is None or member.name == member.phone: designation = f"{format_message(member.phone)}" else: designation = f"{format_message(member.phone)} ~ {format_message(member.name)}" if member.admin: designation += " (admin)" people.members.append(designation) return people
def __init__(self, source_data): prop = 'function' if callable(source_data) else 'constant' super().__init__(ns(view=self), prop) self.data = source_data
from animations import directions as dirs from types import SimpleNamespace as ns from gui import gui, terrain_colors # from random import choice from hex_tile import get_hex_neighbors, pixel_to_hex, hex_to_pixel, tile_hash # dice_pools = [] state = ns( **{ "run": True, "hex_map": {}, "valid": {}, "selected": None, "offset": (0, 0), "active_player": 0, "board_size": 36, "terrain_tiles": { "plains": 12, "forest": 9, "towns": 6, "hills": 6, "mounts": 3 }, "selected_terrain": "plains" }) ms_st = ns(**{"but_1": 0, "but_2": 0, "last": (0, 0)}) # # Dice functions # def generate_dice(count = 6,size = 6): # if count < 6: # count = 6 # return {hash(die):die[0] for die in[(roll_die((size,)),x)
import colors as c from types import SimpleNamespace as ns from animation import Animation as anim from hex_tile import hex_to_pixel from gui import draw_box, draw_hex, draw_text, screen directions = ns(**{"TOP": 0, "LEFT": 1, "BOTTOM": 3, "RIGHT": 4}) def white_hex_blink(selected, loop=2, ms=200): def step(percent, off=(0, 0)): r, g, b = c.WHITE pos, ci = selected coords = hex_to_pixel(pos) highlight = (r, g, b, 255 * percent) bc = c.YELLOW draw_hex(coords, color=highlight, bc=bc, off=off) return anim(ms, step, loop) def slide_in(pos, size, drtn, ms=1000): ss = screen.get_size() x1, y1 = [(0 - l) if drtn < 2 else (sl + l) for l, sl in zip(size, ss)] Δx, Δy = [(a - b) for a, b in zip(pos, (x1, y1))] x2, y2 = pos vert = drtn % 2 == 0 def step(percent, off=(0, 0)): x = (x1 + Δx * percent) if not vert else x2 y = (y1 + Δy * percent) if vert else y2
def train( traindir, *, datadir='data/conll2003', epochs=100, bert_model='bert-base-multilingual-cased', ): save_json(locals(), f'{traindir}/params.json') summary_writer = SummaryWriter(f'{traindir}/summaries') DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu') train_data = list(read_conll2003(f'{datadir}/eng.train')) testa_data = list(read_conll2003(f'{datadir}/eng.testa')) testb_data = list(read_conll2003(f'{datadir}/eng.testb')) feats = FeatsBert(bert_model=bert_model) train, testa, testb = feats.encode(train_data, testa_data, testb_data) feats.save(traindir) # save vocabularies labels_vocab = feats.vocab['labels'] train_batches = Batcher(train, batch_size=10, shuffle=True, max_seqlen=60).to( DEVICE) # BERT needs to limit seq length (OOM) testa_batches = Batcher(testa, batch_size=32).to(DEVICE) testb_batches = Batcher(testb, batch_size=32).to(DEVICE) model = BertTagger(bert_model, labels_vocab_size=len(labels_vocab)).to(DEVICE) # for p in model.bert.parameters(): # p.requires_grad = False optimizer = torch.optim.SGD([ { 'params': model.bert.parameters(), 'lr': 0.00015 }, { 'params': model.output.parameters(), 'lr': 0.15 }, ], lr=1.5, momentum=0.0, weight_decay=0.000001) schedule = torch.optim.lr_scheduler.LambdaLR( optimizer, lambda epoch: 1. / (1. + 0.05 * epoch)) eval_metric = MetricSet({ 'acc': TokenAndRecordAccuracyBert(), 'entity': F1ScoreBert(labels_vocab=labels_vocab), 'loss': CrossEntropyLoss(), }) train_metric = MetricSet({ 'acc': TokenAndRecordAccuracyBert(), }) def cross_entropy(x, labels): x = x.transpose(1, 2) return F.cross_entropy(x, labels, ignore_index=0, reduction='mean') global_step = 0 best = ns(f1=0.0, epoch=-1) for epoch in trange(epochs, desc='epoch'): schedule.step() summary_writer.add_scalar('lr', optimizer.param_groups[0]['lr'], global_step=global_step) train_metric.reset() for count, (words, labels) in enumerate(tqdm(train_batches, desc='step'), start=1): optimizer.zero_grad() x = model(words) loss = cross_entropy(x, labels) loss.backward() optimizer.step() # accumulate metrics train_metric.append(x, labels, loss=loss.item()) global_step += 1 if count % 50 == 0: summ = train_metric.summary tqdm.write( f'epoch: {epoch:5}, step: {count:6}, loss: {summ["acc.loss"]:10.4f} ' f'tacc: {summ["acc.tacc"]:6.4f}, racc: {summ["acc.racc"]:6.4f}' ) summary_writer['train'].add_scalar_metric( summ, global_step=global_step) with torch.no_grad(): model.train(False) eval_metric.reset().update( (model(x), y) for x, y in tqdm(testa_batches, desc='dev')) summ = eval_metric.summary f1 = summ['entity.f1'] if f1 > best.f1: best.f1 = f1 best.epoch = epoch torch.save(model, f'{traindir}/model.pickle') tqdm.write( f'Dev: loss: {summ["loss.loss"]:6.4f}, tacc: {summ["acc.tacc"]:6.4f}, racc: {summ["acc.racc"]:6.4f}, ' f'entity.f1: {summ["entity.f1"]:6.4f}, best.f1: {best.f1:6.4f} at epoch {best.epoch}' ) summary_writer['dev'].add_scalar_metric(summ, global_step=global_step) model.train(True) model = torch.load(f'{traindir}/model.pickle') with torch.no_grad(): model.train(False) metric = MetricSet({ 'acc': TokenAndRecordAccuracyBert(), 'entity': F1ScoreBert(labels_vocab=labels_vocab), 'viterbi': F1ScoreBert(labels_vocab=labels_vocab, entity_decoder='viterbi'), # this is sloow 'loss': CrossEntropyLoss(), }) tqdm.write('Evaluating the best model on testa') metric.reset().update( (model(x), y) for x, y in tqdm(testa_batches, desc="dev")) tqdm.write(repr(metric.summary)) summary_writer['final-dev'].add_scalar_metric(metric.summary) tqdm.write('Evaluating the best model on testb') metric.reset().update( (model(x), y) for x, y in tqdm(testb_batches, desc="test")) tqdm.write(repr(metric.summary)) summary_writer['final-test'].add_scalar_metric(metric.summary) summary_writer.close()
class Activation(): def __init__(self, f, df): self.f = f self.df = df def __call__(self, *args, **kwargs): return self.f(*args, **kwargs) # index into the derivative of the activation function def d(activation): return activation.df # activation functions _sigmoid = lambda z: 1 / (1 + np.exp(-z)) activations = ns( sigmoid = Activation(_sigmoid, lambda z: _sigmoid(z) * (1 - _sigmoid(z))), relu = Activation(lambda z: z*(z>0), lambda z: (z > 0) + 1e-12) ) class Autoencoder(): def __init__(self, layers): logging.info("Setting up autoencoder") self.layers = layers self.params = [] def train(self, x, tau=0.01, epochs=100, seed=112358): np.random.seed(seed) p, n = x.shape # wire up layers L = len(self.layers) # inputs, outputs, weights, biases, and activation functions
def examples(cls): return [ ns(use_exception_handler=False), ns(use_exception_handler=True), ]
import ui, console, objc_util, keychain from types import SimpleNamespace as ns font = 'Apple SD Gothic Neo' UIColor = objc_util.ObjCClass('UIColor') objc_black = UIColor.darkGrayColor().CGColor() light_theme = ns(front='black', back='white', secondary='darkgrey', shadow=objc_black) dark_theme = ns(front='white', back='black', secondary='grey', shadow=objc_black) blue_theme = ns(front='#1976D2', back='white', secondary='#03A9F4', shadow=objc_black) green_theme = ns(front='#009688', back='white', secondary='#80CBC4', shadow=objc_black) red_theme = ns(front='#E53935', back='white', secondary='#FFA726', shadow=objc_black) cyan_dark_theme = ns(front='#4DD0E1', back='black', secondary='#00897B', shadow=objc_black)
def cross_entropy(x, labels): x = x.transpose(1, 2) return F.cross_entropy(x, labels, ignore_index=0, reduction='mean') global_step = 0 best = ns(f1=0.0, epoch=-1) for epoch in trange(epochs, desc='epoch'): schedule.step() summary_writer.add_scalar('lr', optimizer.param_groups[0]['lr'], global_step=global_step) train_metric.reset() for count, (words, labels) in enumerate(tqdm(train_batches, desc='step'), start=1): optimizer.zero_grad() x = model(words) loss = cross_entropy(x, labels) loss.backward() optimizer.step() # accumulate metrics train_metric.append(x, labels, loss=loss.item()) global_step += 1 if count % 50 == 0: summ = train_metric.summary tqdm.write( f'epoch: {epoch:5}, step: {count:6}, loss: {summ["acc.loss"]:10.4f} ' f'tacc: {summ["acc.tacc"]:6.4f}, racc: {summ["acc.racc"]:6.4f}' ) summary_writer['train'].add_scalar_metric( summ, global_step=global_step) with torch.no_grad(): model.train(False) eval_metric.reset().update( (model(x), y) for x, y in tqdm(testa_batches, desc='dev')) summ = eval_metric.summary f1 = summ['entity.f1'] if f1 > best.f1: best.f1 = f1 best.epoch = epoch torch.save(model, f'{traindir}/model.pickle') tqdm.write( f'Dev: loss: {summ["loss.loss"]:6.4f}, tacc: {summ["acc.tacc"]:6.4f}, racc: {summ["acc.racc"]:6.4f}, ' f'entity.f1: {summ["entity.f1"]:6.4f}, best.f1: {best.f1:6.4f} at epoch {best.epoch}' ) summary_writer['dev'].add_scalar_metric(summ, global_step=global_step) model.train(True) model = torch.load(f'{traindir}/model.pickle') with torch.no_grad(): model.train(False) metric = MetricSet({ 'acc': TokenAndRecordAccuracyBert(), 'entity': F1ScoreBert(labels_vocab=labels_vocab), 'viterbi': F1ScoreBert(labels_vocab=labels_vocab, entity_decoder='viterbi'), # this is sloow 'loss': CrossEntropyLoss(), }) tqdm.write('Evaluating the best model on testa') metric.reset().update( (model(x), y) for x, y in tqdm(testa_batches, desc="dev")) tqdm.write(repr(metric.summary)) summary_writer['final-dev'].add_scalar_metric(metric.summary) tqdm.write('Evaluating the best model on testb') metric.reset().update( (model(x), y) for x, y in tqdm(testb_batches, desc="test")) tqdm.write(repr(metric.summary)) summary_writer['final-test'].add_scalar_metric(metric.summary) summary_writer.close()