def test_insertConnection(self): connection = models.Connection() connection.CallSid = "test" connection.phone_number = "test" connection.From = "test" connection.To = "test" connection.Duration = 60 connection.put() self.assertEqual(1, len(models.Connection().all().fetch(2)))
def on_logged_in(self, userid, gadget, wavelet): waveid = wavelet.wave_id if not models.Login.exists(waveid, userid): # Wait, this user has no login. Bail out. gadget.set_status(Gadget.LOGIN) return blogger = utils.Blogger.open(wavelet.wave_id, userid) if not blogger: # There was some error. Bail out. gadget.set_status(Gadget.LOGIN) return blogs = blogger.get_blogs() if len(blogs) == 0: # User has no blogs. Bail out for now. gadget.set_status(Gadget.LOGIN) return blog = blogs[0]['id'] connection = models.Connection.get(waveid) if connection: connection.owner = userid connection.blog = blog connection.post = None else: connection = models.Connection(waveid=waveid, owner=userid, blog=blog) connection.put() gadget.set_owner(userid) gadget.set_status(Gadget.CONNECTED) gadget.set('blogs', utils.to_json(blogs)) gadget.set_blog(blog)
def main() -> None: connection = models.Connection(_db=db_name, _host=db_host, _password=db_passwd, _port=db_port, _user=db_user) scan(connection) connection.commit()
def decode(model_file, target_file, gpu=-1, save_to=None): context = utils.load_context(model_file) if context.seed is not None: utils.set_random_seed(context.seed, gpu) Log.i("random seed: {}".format(context.seed)) framework_utils.set_debug(App.debug) loader = context.loader Log.i('load test dataset from {}'.format(target_file)) test_dataset = loader.load(target_file, train=False, size=16 if utils.is_dev() else None) Log.i('#samples {}'.format(len(test_dataset))) Log.v('') Log.v("initialize ...") Log.v('--------------------------------') Log.i('# gpu: {}'.format(gpu)) Log.i('# tagset size: {}'.format(len(loader.tag_map))) Log.i('# model layers: {}'.format(context.models)) Log.i('# context: {}'.format(context)) Log.v('--------------------------------') Log.v('') models.USE_ORTHONORMAL = False # Set up a neural network model layers = [ models.Input( word_embeddings=loader.get_embeddings('word'), char_embeddings=loader.get_embeddings('char'), char_feature_size=50, dropout=0.5, ), models.Recurrent(n_layers=2, in_size=loader.get_embeddings('word').shape[1] + 50, out_size=400, dropout=0.5), models.Tagger(in_size=400 * 2, out_size=len(loader.tag_map), units=100, dropout=0.5) if context.models[2] is models.Tagger else models.GoldTagger(out_size=len(loader.tag_map)), ] if models.Parser in context.models: layers.extend([ models.Connection(in_size=400 * 2, out_size=800, tagset_size=len(loader.tag_map), tag_embed_size=50, dropout=0.5), models.Parser(in_size=850, n_deprels=len(loader.rel_map), n_blstm_layers=1, lstm_hidden_size=400, parser_mlp_units=800, dropout=0.50), ]) model = models.MTL(*layers) chainer.serializers.load_npz(model_file, model) if gpu >= 0: framework_utils.set_model_to_device(model, device_id=gpu) # Setup an evaluator evaluator = models.Evaluator(loader, target_file, save_to) evaluator.add_target(model) # Start decoding framework_utils.chainer_train_off() evaluator.on_epoch_validate_begin({'epoch': 0}) for batch_index, batch in enumerate( test_dataset.batch(context.batch_size, colwise=True, shuffle=False)): xs, ts = batch[:-1], batch[-1] evaluator.on_batch_begin({'train': False, 'xs': xs, 'ts': ts}) model(*xs) evaluator.on_batch_end({'train': False, 'xs': xs, 'ts': ts}) evaluator.on_epoch_validate_end({'epoch': 0})
def train( train_file, test_file=None, embed_file=None, embed_size=100, n_epoch=20, batch_size=32, lr=0.001, l2_lambda=0.0, grad_clip=5.0, tasks='tp', gpu=-1, save_to=None, seed=None): if seed is not None: utils.set_random_seed(seed, gpu) Log.i("random seed: {}".format(seed)) framework_utils.set_debug(App.debug) # Select Task with_tagging_task = False with_parsing_task = False for char in tasks: if char == 't': with_tagging_task = True elif char == 'p': with_parsing_task = True else: raise ValueError("Invalid task specified: {}".format(char)) if not any([with_tagging_task, with_parsing_task]): raise RuntimeError("No valid task specified") Log.i('Task: tagging={}, parsing={}' .format(with_tagging_task, with_parsing_task)) # Transition System transition_system = transition.ArcStandard if with_parsing_task: Log.i('Transition System: {}'.format(transition_system)) # Load files Log.i('initialize DataLoader with embed_file={} and embed_size={}' .format(embed_file, embed_size)) loader = dataset.DataLoader(word_embed_file=embed_file, word_embed_size=embed_size, char_embed_size=10, transition_system=transition_system) Log.i('load train dataset from {}'.format(train_file)) train_dataset = loader.load(train_file, train=True, size=120 if utils.is_dev() else None) if test_file: Log.i('load test dataset from {}'.format(test_file)) test_dataset = loader.load(test_file, train=False, size=16 if utils.is_dev() else None) else: test_dataset = None Log.v('') Log.v("initialize ...") Log.v('--------------------------------') Log.i('# Minibatch-size: {}'.format(batch_size)) Log.i('# epoch: {}'.format(n_epoch)) Log.i('# gpu: {}'.format(gpu)) Log.i('# tagset size: {}'.format(len(loader.tag_map))) Log.v('--------------------------------') Log.v('') # Set up a neural network model layers = [ models.Input( word_embeddings=loader.get_embeddings('word'), char_embeddings=loader.get_embeddings('char'), char_feature_size=50, dropout=0.5, ), models.Recurrent( n_layers=2, in_size=loader.get_embeddings('word').shape[1] + 50, out_size=400, dropout=0.5), models.Tagger( in_size=400 * 2, out_size=len(loader.tag_map), units=100, dropout=0.5) if with_tagging_task else models.GoldTagger(out_size=len(loader.tag_map)), ] if with_parsing_task: layers.extend([ models.Connection( tagset_size=len(loader.tag_map), tag_embed_size=50, dropout=0.5), models.Parser( in_size=850, n_deprels=len(loader.rel_map), n_blstm_layers=2, lstm_hidden_size=400, parser_mlp_units=800, dropout=0.50, transition_system=transition_system), ]) model = models.MTL(*layers) if gpu >= 0: framework_utils.set_model_to_device(model, device_id=gpu) # Setup an optimizer optimizer = chainer.optimizers.Adam( alpha=lr, beta1=0.9, beta2=0.999, eps=1e-08) optimizer.setup(model) if l2_lambda > 0.0: optimizer.add_hook(chainer.optimizer.WeightDecay(l2_lambda)) else: l2_lambda = False if grad_clip > 0.0: optimizer.add_hook(chainer.optimizer.GradientClipping(grad_clip)) else: grad_clip = False # optimizer.add_hook( # framework_utils.optimizers.ExponentialDecayAnnealing( # initial_lr=lr, decay_rate=0.75, decay_step=5000, lr_key='alpha')) Log.i('optimizer: Adam(alpha={}, beta1=0.9, ' 'beta2=0.999, eps=1e-08), grad_clip={}, ' 'regularization: WeightDecay(lambda={})' .format(lr, grad_clip, l2_lambda)) # Setup a trainer trainer = Trainer(optimizer, model, loss_func=model.compute_loss, accuracy_func=model.compute_accuracy) trainer.configure(framework_utils.config) if test_dataset: evaluator = models.Evaluator(loader, test_file, save_to) evaluator.add_target(model) trainer.attach_callback(evaluator) if save_to is not None: accessid = Log.getLogger().accessid date = Log.getLogger().accesstime.strftime('%Y%m%d') trainer.attach_callback( framework_utils.callbacks.Saver( model, basename="{}-{}".format(date, accessid), directory=save_to, context=dict(App.context, models=[type(layer) for layer in layers], loader=loader))) # Start training trainer.fit(train_dataset, None, batch_size=batch_size, epochs=n_epoch, validation_data=test_dataset, verbose=App.verbose)