def route_message(request):
    username = current_user(request)
    # 如果此时用户未登录,重定向到 '/'
    if username == '游客':
        return redirect('/login')

    # 判断 POST 请求
    if request.method == 'POST':
        # 先加载原有数据
        form = request.form()
        t = Message.new(form)
        # 加个时间
        t.time = current_time()
        item = t.saveMessage()
        save(item, 'data/Message.txt')
        # 将 list 转换成 str
        body = templateM('message.html', messages=item)
    elif request.method == 'GET':
        # 也就是说,当我第一次访问 http://localhost:3000/messages 时,会先发送 GET 请求
        # 定向到了新的 url
        # http://localhost:3000/messages?message=gua
        if any(request.query) == False:
            # 说明是进入网页的时候提交的 GET 请求
            # 提取出现有的 Message.
            path = 'data/Message.txt'
            data = load(path)
            body = templateM('message.html', messages=data)
    header = 'HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n'
    r = header + '\r\n' + body
    return r.encode(encoding='utf-8')
def main():
    args = parser.parse_args()
    ds = datasources.load(args.text_path, ds_name=args.text_type,
                        max_length=args.max_length,
                        buffer_size=args.buffer_size,
                        batch_size=args.batch_size,
                        to_lower=args.to_lower)

    model = models.load(ds.vocab_size, embedding_dim=args.embedding_dim, units=args.units)

    checkpoint_prefix = os.path.splitext(os.path.basename(args.text_path))[0]
    trainer = Trainer(datasource=ds, model=model, optimizer=tf.train.AdamOptimizer(),
                      checkpoint_dir=args.checkpoint_dir, checkpoint_prefix=checkpoint_prefix)

    trainer.restore_last_checkpoint()

    print('Insert separated by spaces:')
    print('- number of samples that should be generated')
    print('- size of text to be generated')
    print('- the start string of the text')
    print('- temperature (higher = more creative, lower=more predictable')
    print('Example of input: 10 100 test 1.2')
    

    while True:
        repeat, num_char_generate, start_string, temperature = input().split()
        repeat = int(repeat)
        temperature = float(temperature)
        num_char_generate = int(num_char_generate)
        for i in range(repeat):
            generated_text = trainer.sample(num_char_generate=num_char_generate, start_string=start_string, temperature=temperature)
            if args.text_type == 'sentences' and generated_text in ds.text:
                print ('Not so creative this exact sample of text is in the train file.')
            print (generated_text)
            print ('-' * 30)
Beispiel #3
0
def main():
    args = parser.parse_args()
    ds = datasources.load(args.text_path,
                          ds_name=args.text_type,
                          max_length=args.max_length,
                          buffer_size=args.buffer_size,
                          batch_size=args.batch_size,
                          to_lower=args.to_lower)

    model = models.load(ds.vocab_size,
                        embedding_dim=args.embedding_dim,
                        units=args.units)

    checkpoint_prefix = os.path.splitext(os.path.basename(args.text_path))[0]
    trainer = Trainer(datasource=ds,
                      model=model,
                      optimizer=tf.train.AdamOptimizer(),
                      checkpoint_dir=args.checkpoint_dir,
                      checkpoint_prefix=checkpoint_prefix)

    trainer.train(epochs=args.epochs,
                  verbose=args.verbose,
                  num_char_generate=args.num_char_generate,
                  start_string=args.start_string,
                  temperature=args.temperature)
Beispiel #4
0
def super_train(args, built_model, status_track):
    # take out the parts that self_play needs from the model
    X, Y, Z, _, _, params, loss = built_model
    with tf.name_scope("train"):
        if args.which_cycle == 0: lr = 1e-2
        else: lr = 1e-3
        train_step = tf.train.AdamOptimizer(lr).minimize(loss)

    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        model_dir = status_track.get_sl_starter()
        assert (args.save_dir is not None) and (model_dir is not None), "save_dir and model_dir needs to be specified for super_training"
        sess.run(load(params, os.path.join(args.save_dir, model_dir)))
        print("loaded model {} at dir {} as super_training starter".format(args.save_dir, model_dir))

        # data for supervised training
        dump_trace = os.path.join(args.dump_dir, args.dump_file)
        with open(dump_trace, 'rb') as sl_file:
            sl_Buffer = pickle.load(sl_file)

        # supervised training cycle
        for i in range(args.sl_num_steps + 1):
            batch = sl_Buffer.sample(args.sl_nbatch)
            feed_dict = { X: batch[0], Y: batch[1], Z: batch[2] }
            sess.run(train_step, feed_dict)
            if i > 0 and i % args.sl_ncheckpoint == 0:
                new_model_dir = status_track.generate_new_model()
                print("checkpoint model {}".format(new_model_dir))
                ps = sess.run(params)
                save(ps, os.path.join(args.save_dir, new_model_dir))
def main(input_model_file, output_weights_file):
    # Load the model
    print('\n---- Loading Keras model (' + input_model_file + ')...\n')
    model = models.load("chestxray", input_model_file)
    print('\nDone loading model\n')

    extract_and_save_weights(model, output_weights_file)
def main(model_arch, input_model_file, output_model_file):
    # Load model
    print('\n---- Loading Keras model (' + input_model_file + ')...\n')
    model = models.load(model_arch, input_model_file)
    print('\nDone loading model\n')

    convert_and_save_model(model, output_model_file)
Beispiel #7
0
 def validate_register(self):
     path = self.db_path()
     models = load(path)
     for up in models:
         if self.username == up['username']:
             return False
     if len(self.username) > 2 and len(self.password) > 2:
         return True
Beispiel #8
0
def run(model_settings, dataset_settings, _log):
    _log.info('dataset_settings: ' + str(dataset_settings))
    _log.info('model_settings: ' + str(model_settings))
    dataset = datasets.load(dataset_settings)
    model_settings.update({'dataset': dataset})
    model = models.load(model_settings)

    train(model, dataset)
    evaluate(model, dataset)
Beispiel #9
0
 def all(cls,page = 0,size =5):
     path = cls.db_path()
     models = load(path)
     ms = [cls._new_from_dict(m) for m in models]
     if page == 0:
         return ms
     else:
         star = (page-1)*size
         end =page*size
         return ms[star:end]
Beispiel #10
0
def self_play(args, built_model, status_track):
    # take out the parts that self_play need from the model
    X, _, _, p, v, params, _ = built_model

    # within a tensorflow session, run MCT objects with model
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        model_dir = status_track.get_model_dir()
        if (args.save_dir is not None) and (model_dir is not None):
            sess.run(load(params, os.path.join(args.save_dir, model_dir)))
            print("loaded model {} at dir {} for selfplay".format(args.save_dir, model_dir))
        else:
            # this is the initial random parameter! let's save it in hard drive!
            ps = sess.run(params)
            model_dir = status_track.generate_new_model()
            save(ps, os.path.join(args.save_dir, model_dir))

        # initialize a list of MCT and run self_play
        MCTList = []
        for i in status_track.get_nbatch_index(args.nbatch, args.n_train_files):
            MCTList.append(MCT(args.train_path, i, args.max_clause, args.max_var, args.nrepeat, 
                tau = lambda x: 1.0 if x <= 30 else 0.0001, resign = 400)) 
        pi_matrix = np.zeros((args.nbatch, 2 * args.max_var), dtype = np.float32)
        v_array = np.zeros((args.nbatch,), dtype = np.float32)
        needMore = np.ones((args.nbatch,), dtype = np.bool)
        while True:
            states = []
            pi_v_index = 0
            for i in range(args.nbatch):
                if needMore[i]:
                    temp = MCTList[i].get_state(pi_matrix[pi_v_index], v_array[pi_v_index])
                    pi_v_index += 1
                    if temp is None:
                        needMore[i] = False
                    else:
                        states.append(temp)
            if not np.any(needMore):
                break
            pi_matrix, v_array = sess.run([p, v], feed_dict = {X: np.asarray(states, dtype = np.float32)})

        print("loop finished and save Pi graph to slBuffer")
        # bring sl_buffer to memory
        os.makedirs(args.dump_dir, exist_ok = True)
        dump_trace = os.path.join(args.dump_dir, args.dump_file)
        if os.path.isfile(dump_trace):
            with open(dump_trace, 'rb') as sl_file:
                sl_Buffer = pickle.load(sl_file)
        else:
            sl_Buffer = slBuffer_allFile(args.sl_buffer_size, args.train_path, args.n_train_files)
        # write in sl_buffer
        for i in range(args.nbatch):
            MCTList[i].write_data_to_buffer(sl_Buffer)
        # write sl_buffer back to disk
        with open(dump_trace, 'wb') as sl_file:
            pickle.dump(sl_Buffer, sl_file, -1)
def app(name, settings):
    u"""起動するアプリケーションをロードする
    """
    options.log_file_prefix = settings["logging_path"]
    logging.getLogger().setLevel(settings['logging_level'])

    models.load(settings['db'])

    with open(settings['server_apps_conf_path'], 'r') as f:
        app_conf = yaml.load(f)
        server = app_conf['common'][name]
        if settings['env'] in app_conf:
            server.update(app_conf[settings['env']][name])

    ui = {'ui_modules': {}, 'ui_methods': {}}
    for ui_key in ui.keys():
        for package in server.get(ui_key, {}).keys():
            for key in server[ui_key][package].keys():
                name = server[ui_key][package][key]
                module= importlib.import_module("mylib.%s.%s" % \
                        (ui_key, package))
                ui[ui_key].update({key: getattr(module, name)})
    settings.update(ui)

    routes = []
    for package in server['handlers'].keys():
        for uri in server['handlers'][package].keys():
            name = server['handlers'][package][uri]
            handler = importlib.import_module("handlers.%s" % package)
            routes.append((r"%s" % uri, getattr(handler, name)))

    application = Application(routes, **settings)
    try:
        application.sentry_client = AsyncSentryClient(settings['sentry'])
    except:
        pass

    return dict(
        app = application, 
        port = server['port'],
        worker = server['worker_processes']
    )
Beispiel #12
0
def main():
    config = configure()
    world = worlds.load(config)
    model = models.load(config)
    trainer = trainers.load(config, world, model)
    #trainer.train(model, world)
    if (trainer.test(model, world) < 0.7):
        return False

    with model.session.as_default():
        trainer.transfer(model, world)
Beispiel #13
0
 def validate_login(self):
     # 先通过db_path()取得user.txt文件
     path = self.db_path()
     # 读取文件将json变为list
     models = load(path)
     # 循环List中的dict匹配post给的值是否存在,存在则登录成功.
     for up in models:
         if self.username == up['username'] and self.password == up[
                 'password']:
             return True
     return False
Beispiel #14
0
def main(model_arch,
         input_model_file,
         output_model_file,
         input_name=INPUT_NAME_DEFAULT,
         output_name=OUTPUT_NAME_DEFAULT):
    # Load model
    print('\n---- Loading Keras model (' + input_model_file + ')...\n')
    model = models.load(model_arch, input_model_file)
    print('\nDone loading model\n')

    convert_and_save_model(model, output_model_file, input_name, output_name)
Beispiel #15
0
def run(model_settings, dataset_settings, num_experiments, _log):
    _log.info('dataset_settings: ' + str(dataset_settings))
    _log.info('model_settings: ' + str(model_settings))

    ex.info['evaluations'] = []
    for i in range(num_experiments):
        dataset = datasets.load(dataset_settings)
        model_settings.update({'dataset': dataset})
        model = models.load(model_settings)
        train(model, dataset)
        ex.info['evaluations'].append(evaluate(model, dataset))
Beispiel #16
0
def main():
    parser = argparse.ArgumentParser('Runs single experiment for a single seed.')
    parser.add_argument('config', nargs='?', type=str, default='config.yaml')
    args = parser.parse_args()
    config = configure(file=args.config)
    world = worlds.load(config)
    model = models.load(config)
    trainer = trainers.load(config)
    tf.compat.v1.disable_eager_execution()
    tf.get_logger().setLevel('INFO') # DEBUG/INFO/....
    trainer.train(model, world)
Beispiel #17
0
def setup_model(env, config):
    shared_model = models.load(config, env.observation_space.shape[0],
                               env.action_space)
    if config.load:
        saved_state = torch.load('{0}{1}.dat'.format(config.load_model_dir,
                                                     config.env),
                                 map_location=lambda storage, loc: storage)
        shared_model.load_state_dict(saved_state)
    shared_model.share_memory()  # NOTE Hogwild style

    return shared_model
def route_admin(request):
    # 只允许 role 为 1 的用户组访问
    if current_role(request) == 1:
        data = load('data/User.txt')
        # list 转换为 str
        headers = {'Content-Type': 'text/html'}
        body = templateM('admin.html', users=data)
        header = response_with_headers(headers)
        r = header + '\r\n' + body
        return r.encode(encoding='utf-8')
    else:
        return redirect('/login')
Beispiel #19
0
 def test_load(self):
     r = [
         {
             'id': 1,
             'name': 'gua',
         },
         {
             'id': 2,
             'name': 'gw'
         },
     ]
     self.assertListEqual(load('Student'), r)
Beispiel #20
0
def app(name, settings):
    u"""起動するアプリケーションをロードする
    """
    options.log_file_prefix = settings["logging_path"]
    logging.getLogger().setLevel(settings['logging_level'])

    models.load(settings['db'])

    with open(settings['server_apps_conf_path'], 'r') as f:
        app_conf = yaml.load(f)
        server = app_conf['common'][name]
        if settings['env'] in app_conf:
            server.update(app_conf[settings['env']][name])

    ui = {'ui_modules': {}, 'ui_methods': {}}
    for ui_key in ui.keys():
        for package in server.get(ui_key, {}).keys():
            for key in server[ui_key][package].keys():
                name = server[ui_key][package][key]
                module= importlib.import_module("mylib.%s.%s" % \
                        (ui_key, package))
                ui[ui_key].update({key: getattr(module, name)})
    settings.update(ui)

    routes = []
    for package in server['handlers'].keys():
        for uri in server['handlers'][package].keys():
            name = server['handlers'][package][uri]
            handler = importlib.import_module("handlers.%s" % package)
            routes.append((r"%s" % uri, getattr(handler, name)))

    application = Application(routes, **settings)
    try:
        application.sentry_client = AsyncSentryClient(settings['sentry'])
    except:
        pass

    return dict(app=application,
                port=server['port'],
                worker=server['worker_processes'])
def run(model_settings, dataset_settings, _log):
    _log.info('dataset_settings: ' + str(dataset_settings))
    dataset = datasets.load(dataset_settings)

    model_settings.update({
        'input_shape': dataset.input_shape,
        'num_classes': dataset.num_classes,
    })
    _log.info('model_settings: ' + str(model_settings))
    model = models.load(model_settings)

    train(model, dataset)
    evaluate(model, dataset)
def run(model_settings, dataset_settings, num_experiments, _log):
    _log.info('dataset_settings: ' + str(dataset_settings))
    _log.info('model_settings: ' + str(model_settings))
    ex.info['evaluations'] = []
    for i in range(1, num_experiments+1):
        print('#'*10, 'Run', i, '#'*10)
        dataset_settings['train_size'] = i/num_experiments
        dataset = datasets.load(dataset_settings)
        model_settings.update({'dataset': dataset})
        model = models.load(model_settings)
        train(model, dataset)
        ex.info['evaluations'].append(evaluate(model, dataset))
    ex.info['sota'] = dataset.sota
Beispiel #23
0
    def __init__(self, conf):
        self.model_path = conf['model_path']
        self.tokenizer = conf.get('tokenizer', default_tokenizer)
        self.untokenizer = conf.get('untokenizer', default_untokenizer)
        self.lowercase = conf.get('lowercase', True)

        x = T.imatrix('x')
        net, _ = models.load(self.model_path, 1, x)

        self.predict=theano.function(inputs=[x], outputs=net.y)
        self.word_vocabulary=net.x_vocabulary
        self.punctuation_vocabulary=net.y_vocabulary
        self.reverse_punctuation_vocabulary = {v:k for k,v in self.punctuation_vocabulary.items()}
        self.human_readable_punctuation_vocabulary = [p[0] for p in self.punctuation_vocabulary if p != data.SPACE]
Beispiel #24
0
def _subprocess(gpu_queue, args, models_dir, model_name, cv_index, cv_size,
                split_seed, tta_index):
    """子プロセスの予測処理。"""
    if 'CUDA_VISIBLE_DEVICES' not in os.environ:
        # GPUの固定
        gpu_id = gpu_queue.get()
        os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_id)
        # TensorFlowのログ抑止
        os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
        # モデルの読み込み
        import models
        _subprocess_context['model'] = models.load(models_dir / model_name)
    else:
        import models

    result_path = pathlib.Path(models_dir / _CACHE_NAME_FORMAT.format(
        target=args.target, model=model_name, tta=tta_index))
    if result_path.is_file():
        print('スキップ: {}'.format(result_path))
        return
    else:
        seed = 1234 + tta_index
        np.random.seed(seed)

        assert args.target in ('val', 'test')
        if args.target == 'val':
            _, (X_target, _), _ = data.load_data(cv_index, cv_size, split_seed)
        else:
            _, _, X_target = data.load_data(cv_index, cv_size, split_seed)

        pattern_index = len(_SIZE_PATTERNS) * tta_index // args.tta_size
        img_size = _SIZE_PATTERNS[pattern_index]
        batch_size = int(_BATCH_SIZE * ((_BASE_SIZE**2) /
                                        (img_size[0] * img_size[1]))**1.5)

        gen = models.create_generator(img_size, mixup=False)
        proba_target = _subprocess_context['model'].predict_generator(
            gen.flow(X_target,
                     batch_size=batch_size,
                     data_augmentation=True,
                     shuffle=False,
                     random_state=seed),
            steps=gen.steps_per_epoch(len(X_target), batch_size),
            verbose=0)

        result_path.parent.mkdir(parents=True, exist_ok=True)
        joblib.dump(proba_target, result_path)
Beispiel #25
0
def main():
    if len(sys.argv) <= 1:
        print(
            "please specify a model name (e.g. models.baseline.random_handle)")
        sys.exit(1)
    module_name = sys.argv[1]
    hyper_parameters = models.parse_hyper_parameters(sys.argv[2:])
    model = models.load(module_name, hyper_parameters)

    data.load_devel1000()

    print("Model:", module_name, hyper_parameters)

    accuracy, correct, tests = evaluate(
        model, tqdm(data.DEVEL1000, dynamic_ncols=True))

    print(f"Label accuracy: {correct}/{tests} ({accuracy:%})")
Beispiel #26
0
    def restore(self, modelpath, isshowunnk, text):

        if len(modelpath) > 1:
            model_file = modelpath
        else:
            print("Model file path argument missing")

        show_unk = False
        if len(isshowunnk):
            show_unk = bool(int(isshowunnk))

        x = T.imatrix('x')

        print("Loading model parameters...")
        net, _ = models.load(model_file, 1, x)

        print("Building model...")
        predict = theano.function(inputs=[x], outputs=net.y)
        word_vocabulary = net.x_vocabulary
        punctuation_vocabulary = net.y_vocabulary
        reverse_word_vocabulary = {v: k for k, v in net.x_vocabulary.items()}
        reverse_punctuation_vocabulary = {
            v: k
            for k, v in net.y_vocabulary.items()
        }

        human_readable_punctuation_vocabulary = [
            p[0] for p in punctuation_vocabulary if p != data.SPACE
        ]
        tokenizer = word_tokenize
        untokenizer = lambda text: text.replace(" '", "'").replace(
            " n't", "n't").replace("can not", "cannot")

        words = [
            w for w in untokenizer(' '.join(tokenizer(text))).split()
            if w not in punctuation_vocabulary
            and w not in human_readable_punctuation_vocabulary
        ]

        finalstring = self.punctuate(predict, word_vocabulary,
                                     punctuation_vocabulary,
                                     reverse_punctuation_vocabulary,
                                     reverse_word_vocabulary, words, show_unk)

        return finalstring
Beispiel #27
0
 def all(cls):
     """
     all 方法(类里面的函数叫方法)使用 load 函数得到所有的 models
     """
     path = cls.db_path()
     models = load(path)
     # 这里用了列表推导生成一个包含所有 实例 的 list
     # 因为这里是从 存储的数据文件 中加载所有的数据
     # 所以用 _new_from_dict 这个特殊的函数来初始化一个数据
     ms = []
     for m in models:
         m = cls._new_from_dict(m)
         if m.ct > 0:
             format = '%Y/%m/%d %H:%M:%S'
             value = time.localtime(m.ct)
             m.dt = time.strftime(format, value)
         ms.append(m)
     return ms
Beispiel #28
0
 def load(self):
     """Load pretrain model."""
     print "Loading model."
     self.net, _ = models.load(self.model_path, 1, self.x)
     self._predict = theano.function(inputs=[self.x], outputs=self.net.y)
     self.word_vocabulary = self.net.x_vocabulary
     self.punctuation_vocabulary = self.net.y_vocabulary
     self.reverse_word_vocabulary = \
         {v: k for k, v in self.net.x_vocabulary.items()}
     self.reverse_punctuation_vocabulary = \
         {0: u'?',
          1: u'!',
          2: u' ',
          3: u',',
          4: u'-',
          5: u':',
          6: u';',
          7: u'.'}
Beispiel #29
0
def main():
    config = configure()
    session = tf.Session()
    task = tasks.load(config)
    channel = channels.load(config)
    model = models.load(config)
    desc_model = models.desc_im.DescriptionImitationModel()
    translator = translators.load(config)

    rollout_ph = experience.RolloutPlaceholders(task, config)
    replay_ph = experience.ReplayPlaceholders(task, config)
    reconst_ph = experience.ReconstructionPlaceholders(task, config)
    channel.build(config)
    model.build(task, rollout_ph, replay_ph, channel, config)
    desc_model.build(task, rollout_ph, replay_ph, channel, config)
    translator.build(task, reconst_ph, channel, model, config)

    if config.task.train:
        trainer.run(task, rollout_ph, replay_ph, reconst_ph, model, desc_model,
                    translator, session, config)
    else:
        trainer.load(session, config)

    if config.task.lexicon:
        lex = lexicographer.run(task, rollout_ph, reconst_ph, model,
                                desc_model, translator, session, config)

    if config.task.visualize:
        visualizer.run(lex, task, config)

    if config.task.calibrate:
        calibrator.run(task, rollout_ph, model, desc_model, lexicographer,
                       session, config)

    if config.task.evaluate:
        evaluator.run(task, rollout_ph, replay_ph, reconst_ph, model,
                      desc_model, lex, session, config)
        sem_evaluator.run(task, rollout_ph, reconst_ph, model, desc_model,
                          translator, lex, session, config)

    if config.task.turkify:
        turkifier.run(task, rollout_ph, model, lex, session, config)
Beispiel #30
0
def main(training_path: str, audio_path: str, weights_path: str):
    log = get_logger(name=__name__)

    save_file = load(training_path, weights_path)

    try:
        log.info('---------------------------------------------')
        log.info('Using net:')
        log.info("{}".format(training_path))
        log.info('---------------------------------------------')
        log.info('Audio path:')
        log.info(audio_path)
        log.info('---------------------------------------------')
        log.info('Predicted transcription:')
        log.info(predict(save_file.model, save_file.alphabet, audio_path))
    except FileNotFoundError as ex:
        log.error('%s' % ex)
    except ModelNotFoundError as ex:
        log.error('%s' % ex)
    except Exception as ex:
        log.error('%s' % ex)
def punctuate(text_data):

    model_file = "Model_models.py_h256_lr0.02.pcl"
    vocab_len = len(data.read_vocabulary(data.WORD_VOCAB_FILE))
    x_len = vocab_len if vocab_len < data.MAX_WORD_VOCABULARY_SIZE else data.MAX_WORD_VOCABULARY_SIZE + data.MIN_WORD_COUNT_IN_VOCAB
    x = np.ones((x_len, main.MINIBATCH_SIZE)).astype(int)

    print("Loading model parameters...")
    net, _ = models.load(model_file, x)

    print("Building model...")
        
    word_vocabulary = net.x_vocabulary
    punctuation_vocabulary = net.y_vocabulary
    reverse_punctuation_vocabulary = {v:k for k,v in net.y_vocabulary.items()}

    text = re.sub('[,?!ред]', '', text_data)
    # text = text_data
    punctuated = play_with_model.punctuate(word_vocabulary, punctuation_vocabulary, reverse_punctuation_vocabulary, text, net)

    return punctuated
Beispiel #32
0
        while True:
            resp = raw_input("Found an existing model with the name %s. Do you want to:\n[c]ontinue training the existing model?\n[r]eplace the existing model and train a new one?\n[e]xit?\n>" % model_file_name)
            resp = resp.lower().strip()
            if resp not in ('c', 'r', 'e'):
                continue
            if resp == 'e':
                sys.exit()
            elif resp == 'c':
                continue_with_previous = True
            break

    if continue_with_previous:
        print "Loading previous model state" 

        net, state = models.load(model_file_name, MINIBATCH_SIZE, x)
        gsums, learning_rate, validation_ppl_history, starting_epoch, rng = state
        best_ppl = min(validation_ppl_history)

    else:
        rng = np.random
        rng.seed(1)

        print "Building model..."
        net = models.GRU(
            rng=rng,
            x=x,
            minibatch_size=MINIBATCH_SIZE,
            n_hidden=num_hidden,
            x_vocabulary=word_vocabulary,
            y_vocabulary=punctuation_vocabulary
    print "Blog %s has %d pages"%(blogdir, entries[-1])

    for page in entries:
        doc_dom = parse(os.path.join(blogdir, str(page)))
        posts = extract_posts(doc_dom)
        print "Page %d/%d had %d posts"%(page, entries[-1], len(posts))
        for post_dom in posts:
            blog.add_doc(text_from_post(post_dom))
    
#    blog.vectorize()
    # Save it
    return blog

if __name__ == "__main__":
    print "Main longer implemented."
    args = parser.parse_args()
    newdir = os.path.join('data', 'blogs')
    # Load the models files
    load()
    if not os.path.exists(newdir): os.mkdir(newdir)

    for dir in args.blogdir:
        dir = dir.rstrip('/')
        print "Reading %s"%dir
        docs = make_dataset(dir)
        name = os.path.basename(dir)
        print "Writing %s"%name
        with open(os.path.join(newdir, name), 'w') as f:
            cPickle.dump(docs, f, -1)

Beispiel #34
0
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("--batch-size", type=int, default=30, help='batch size')
    parser.add_argument("--dataload-workers-nums", type=int, default=8, help='number of workers for dataloader')
    parser.add_argument('--tta', action='store_true', help='test time augmentation')
    parser.add_argument('--seed', type=int, default=None, help='manual seed for deterministic')
    parser.add_argument("--threshold", type=float, default=0.5, help='probability threshold')
    parser.add_argument("--output-prefix", type=str, default='noprefix', help='prefix string for output files')
    parser.add_argument('--resize', action='store_true', help='resize to 128x128 instead of reflective padding')
    parser.add_argument("model", help='a pretrained neural network model')
    args = parser.parse_args()

    use_gpu = torch.cuda.is_available()
    print('use_gpu', use_gpu)

    print("loading model...")
    model = models.load(args.model)
    model.float()

    if use_gpu:
        if args.seed is not None:
            torch.manual_seed(args.seed)
            torch.backends.cudnn.deterministic = True
        else:
            torch.backends.cudnn.benchmark = True
        model.cuda()

    print("testing %s..." % args.model)
    since = time.time()
    test()
    time_elapsed = time.time() - since
    time_str = 'total time elapsed: {:.0f}h {:.0f}m {:.0f}s '.format(time_elapsed // 3600, time_elapsed % 3600 // 60,
Beispiel #35
0
def train(**kwargs):
    """
    Train model

    Load the whole train data in memory for faster operations

    args: **kwargs (dict) keyword arguments that specify the model hyperparameters
    """

    # Roll out the parameters
    batch_size = kwargs["batch_size"]
    n_batch_per_epoch = kwargs["n_batch_per_epoch"]
    nb_epoch = kwargs["nb_epoch"]
    model_name = kwargs["model_name"]
    generator = kwargs["generator"]
    image_data_format = kwargs["image_data_format"]
    img_dim = kwargs["img_dim"]
    patch_size = kwargs["patch_size"]
    bn_mode = kwargs["bn_mode"]
    label_smoothing = kwargs["use_label_smoothing"]
    label_flipping = kwargs["label_flipping"]
    dset = kwargs["dset"]
    use_mbd = kwargs["use_mbd"]

    epoch_size = n_batch_per_epoch * batch_size
    # Setup environment (logging directory etc)
    #general_utils.setup_logging(model_name)

    # Load and rescale data
    #X_full_train, X_sketch_train, X_full_val, X_sketch_val = data_utils.load_data(dset, image_data_format)
    img_dim = (256,256,3) # Manual entry

    # Get the number of non overlapping patch and the size of input image to the discriminator
    nb_patch, img_dim_disc = data_utils.get_nb_patch(img_dim, patch_size, image_data_format)

    try:

        # Create optimizers
        opt_dcgan = Adam(lr=1E-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
        # opt_discriminator = SGD(lr=1E-3, momentum=0.9, nesterov=True)
        opt_discriminator = Adam(lr=1E-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)

        # Load generator model
        generator_model = models.load("generator_unet_%s" % generator,
                                      img_dim,
                                      nb_patch,
                                      bn_mode,
                                      use_mbd,
                                      batch_size)
        # Load discriminator model
        discriminator_model = models.load("DCGAN_discriminator",
                                          img_dim_disc,
                                          nb_patch,
                                          bn_mode,
                                          use_mbd,
                                          batch_size)

        generator_model.compile(loss="mae", optimizer=opt_discriminator)
        discriminator_model.trainable = False

        DCGAN_model = models.DCGAN(generator_model,
                                   discriminator_model,
                                   img_dim,
                                   patch_size,
                                   image_data_format)

        loss = [l1_loss, 'binary_crossentropy']
        loss_weights = [1E1, 1]
        DCGAN_model.compile(loss=loss, loss_weights=loss_weights, optimizer=opt_dcgan)

        discriminator_model.trainable = True
        discriminator_model.compile(loss='binary_crossentropy', optimizer=opt_discriminator)

        gen_loss = 100
        disc_loss = 100
        best_loss=[100]*3

        # Start training
        print("Start training")
        for e in range(nb_epoch):
            # Initialize progbar and batch counter
            progbar = generic_utils.Progbar(epoch_size)
            batch_counter = 1
            start = time.time()

            for X_full_batch, X_sketch_batch in data_utils.facades_generator(img_dim,batch_size=batch_size):

                X_gen, X_gen_target = next(data_utils.facades_generator(img_dim,batch_size=batch_size))
                generator_model.train_on_batch(X_gen, X_gen_target)
                # Create a batch to feed the discriminator model
                X_disc, y_disc = data_utils.get_disc_batch(X_full_batch,
                                                           X_sketch_batch,
                                                           generator_model,
                                                           batch_counter,
                                                           patch_size,
                                                           image_data_format,
                                                           label_smoothing=label_smoothing,
                                                           label_flipping=label_flipping)

                # Update the discriminator
                disc_loss = discriminator_model.train_on_batch(X_disc, y_disc) # X_disc, y_disc
                # Create a batch to feed the generator model
                X_gen, X_gen_target = next(data_utils.facades_generator(img_dim,batch_size=batch_size))
                y_gen = np.zeros((X_gen.shape[0], 2), dtype=np.uint8)
                y_gen[:, 1] = 1

                # Freeze the discriminator
                discriminator_model.trainable = False
                gen_loss = DCGAN_model.train_on_batch(X_gen, [X_gen_target, y_gen])
                # Unfreeze the discriminator
                discriminator_model.trainable = True

                batch_counter += 1
                progbar.add(batch_size, values=[("D logloss", disc_loss),
                                                ("G tot", gen_loss[0]),
                                                ("G L1", gen_loss[1]),
                                                ("G logloss", gen_loss[2])])

                # Save images for visualization
                if batch_counter % (n_batch_per_epoch / 2) == 0:
                    # Get new images from validation
                    figure_name = "training_"+str(e)
                    data_utils.plot_generated_batch(X_full_batch, X_sketch_batch, generator_model,
                                                    batch_size, image_data_format, figure_name)

                if batch_counter >= n_batch_per_epoch:
                    break

            print("")
            print('Epoch %s/%s, Time: %s' % (e + 1, nb_epoch, time.time() - start))

            if e % 5 == 0:
                gen_weights_path = os.path.join('../../models/%s/gen_weights_epoch%s.h5' % (model_name, e))
                generator_model.save_weights(gen_weights_path, overwrite=True)

                disc_weights_path = os.path.join('../../models/%s/disc_weights_epoch%s.h5' % (model_name, e))
                discriminator_model.save_weights(disc_weights_path, overwrite=True)

                DCGAN_weights_path = os.path.join('../../models/%s/DCGAN_weights_epoch%s.h5' % (model_name, e))
                DCGAN_model.save_weights(DCGAN_weights_path, overwrite=True)
                
                Best_gen_L1_weights_path = os.path.join('../../models/%s/best_gen_L1_weights_epoch.h5' % (model_name))
                if(gen_loss[1]<=best_loss[1]):
                        generator_model.save_weights(Best_gen_L1_weights_path, overwrite=True)
                        best_loss[1]=gen_loss[1]
                Best_gen_Totweights_path = os.path.join('../../models/%s/best_gen_Totweights_epoch.h5' % (model_name))
                if(gen_loss[0]<=best_loss[0]):
                        generator_model.save_weights(Best_gen_Totweights_path, overwrite=True)
                        best_loss[0]=gen_loss[0]
                         

    except KeyboardInterrupt:
        pass
Beispiel #36
0
    if len(sys.argv) > 2:
        output_file = sys.argv[2]
    else:
        sys.exit("Output file path argument missing")

    use_pauses = len(sys.argv) > 3 and bool(int(sys.argv[3]))

    x = T.imatrix('x')
    
    if use_pauses:
    
        p = T.matrix('p')

        print("Loading model parameters...")
        net, _ = models.load(model_file, 1, x, p)

        print("Building model...")
        predict = theano.function(
            inputs=[x, p],
            outputs=net.y
        )

    else:

        print("Loading model parameters...")
        net, _ = models.load(model_file, 1, x)

        print("Building model...")
        predict = theano.function(
            inputs=[x],
def eval(**kwargs):

    # Roll out the parameters
    batch_size = kwargs["batch_size"]
    generator = kwargs["generator"]
    model_name = kwargs["model_name"]
    image_dim_ordering = kwargs["image_dim_ordering"]
    img_dim = kwargs["img_dim"]
    cont_dim = (kwargs["cont_dim"],)
    cat_dim = (kwargs["cat_dim"],)
    noise_dim = (kwargs["noise_dim"],)
    bn_mode = kwargs["bn_mode"]
    noise_scale = kwargs["noise_scale"]
    dset = kwargs["dset"]
    epoch = kwargs["epoch"]

    # Setup environment (logging directory etc)
    general_utils.setup_logging(model_name)

    # Load and rescale data
    if dset == "RGZ":
        X_real_train = data_utils.load_RGZ(img_dim, image_dim_ordering)
    if dset == "mnist":
        X_real_train, _, _, _ = data_utils.load_mnist(image_dim_ordering)
    img_dim = X_real_train.shape[-3:]

    # Load generator model
    generator_model = models.load("generator_%s" % generator,
                                  cat_dim,
                                  cont_dim,
                                  noise_dim,
                                  img_dim,
                                  bn_mode,
                                  batch_size,
                                  dset=dset)

    # Load colorization model
    generator_model.load_weights("../../models/%s/gen_weights_epoch%s.h5" %
                                 (model_name, epoch))

    X_plot = []
    # Vary the categorical variable
    for i in range(cat_dim[0]):
        X_noise = data_utils.sample_noise(noise_scale, batch_size, noise_dim)
        X_cont = data_utils.sample_noise(noise_scale, batch_size, cont_dim)
        X_cont = np.repeat(X_cont[:1, :], batch_size, axis=0)  # fix continuous noise
        X_cat = np.zeros((batch_size, cat_dim[0]), dtype='float32')
        X_cat[:, i] = 1  # always the same categorical value

        X_gen = generator_model.predict([X_cat, X_cont, X_noise])
        X_gen = data_utils.inverse_normalization(X_gen)

        if image_dim_ordering == "th":
            X_gen = X_gen.transpose(0,2,3,1)

        X_gen = [X_gen[i] for i in range(len(X_gen))]
        X_plot.append(np.concatenate(X_gen, axis=1))
    X_plot = np.concatenate(X_plot, axis=0)

    plt.figure(figsize=(8,10))
    if X_plot.shape[-1] == 1:
        plt.imshow(X_plot[:, :, 0], cmap="gray")
    else:
        plt.imshow(X_plot)
    plt.xticks([])
    plt.yticks([])
    plt.ylabel("Varying categorical factor", fontsize=28, labelpad=60)

    plt.annotate('', xy=(-0.05, 0), xycoords='axes fraction', xytext=(-0.05, 1),
                 arrowprops=dict(arrowstyle="-|>", color='k', linewidth=4))
    plt.tight_layout()
    plt.savefig("../../figures/varying_categorical.png")
    plt.clf()
    plt.close()

    # Vary the continuous variables
    X_plot = []
    # First get the extent of the noise sampling
    x = np.ravel(data_utils.sample_noise(noise_scale, batch_size * 20000, cont_dim))
    # Define interpolation points
    x = np.linspace(x.min(), x.max(), num=batch_size)
    for i in range(batch_size):
        X_noise = data_utils.sample_noise(noise_scale, batch_size, noise_dim)
        X_cont = np.concatenate([np.array([x[i], x[j]]).reshape(1, -1) for j in range(batch_size)], axis=0)
        X_cat = np.zeros((batch_size, cat_dim[0]), dtype='float32')
        X_cat[:, 1] = 1  # always the same categorical value

        X_gen = generator_model.predict([X_cat, X_cont, X_noise])
        X_gen = data_utils.inverse_normalization(X_gen)
        if image_dim_ordering == "th":
            X_gen = X_gen.transpose(0,2,3,1)
        X_gen = [X_gen[i] for i in range(len(X_gen))]
        X_plot.append(np.concatenate(X_gen, axis=1))
    X_plot = np.concatenate(X_plot, axis=0)

    plt.figure(figsize=(10,10))
    if X_plot.shape[-1] == 1:
        plt.imshow(X_plot[:, :, 0], cmap="gray")
    else:
        plt.imshow(X_plot)
    plt.xticks([])
    plt.yticks([])
    plt.ylabel("Varying continuous factor 1", fontsize=28, labelpad=60)
    plt.annotate('', xy=(-0.05, 0), xycoords='axes fraction', xytext=(-0.05, 1),
                 arrowprops=dict(arrowstyle="-|>", color='k', linewidth=4))
    plt.xlabel("Varying continuous factor 2", fontsize=28, labelpad=60)
    plt.annotate('', xy=(1, -0.05), xycoords='axes fraction', xytext=(0, -0.05),
                 arrowprops=dict(arrowstyle="-|>", color='k', linewidth=4))
    plt.tight_layout()
    plt.savefig("../../figures/varying_continuous.png")
    plt.clf()
    plt.close()
Beispiel #38
0
    import argparse
    from pathlib import Path
    from torchvision.transforms import Compose
    from torch.utils.data import DataLoader
    from transforms import PrepareImageAndMask, PadToNxN, HWCtoCHW
    from datasets import SaltIdentification

    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("--input", type=str, help='input directory')
    parser.add_argument("--output", type=str, default='swa_model.pth', help='output model file')
    parser.add_argument("--batch-size", type=int, default=16, help='batch size')
    args = parser.parse_args()

    directory = Path(args.input)
    files = [f for f in directory.iterdir() if f.suffix == ".pth"]
    assert(len(files) > 1)

    net = models.load(files[0])
    for i, f in enumerate(files[1:]):
        net2 = models.load(f)
        moving_average(net, net2, 1. / (i + 2))

    img_size = 128
    batch_size = 16
    train_transform = Compose([PrepareImageAndMask(), PadToNxN(img_size), HWCtoCHW()])
    train_dataset = SaltIdentification(mode='train', transform=train_transform, preload=True)
    train_dataloader = DataLoader(train_dataset, shuffle=True, batch_size=batch_size, drop_last=True)
    net.cuda()
    bn_update(train_dataloader, net)

    models.save(net, args.output)
def train(**kwargs):
    """
    Train model

    Load the whole train data in memory for faster operations

    args: **kwargs (dict) keyword arguments that specify the model hyperparameters
    """

    # Roll out the parameters
    batch_size = kwargs["batch_size"]
    n_batch_per_epoch = kwargs["n_batch_per_epoch"]
    nb_epoch = kwargs["nb_epoch"]
    generator = kwargs["generator"]
    model_name = kwargs["model_name"]
    image_dim_ordering = kwargs["image_dim_ordering"]
    img_dim = kwargs["img_dim"]
    cont_dim = (kwargs["cont_dim"],)
    cat_dim = (kwargs["cat_dim"],)
    noise_dim = (kwargs["noise_dim"],)
    bn_mode = kwargs["bn_mode"]
    label_smoothing = kwargs["label_smoothing"]
    label_flipping = kwargs["label_flipping"]
    noise_scale = kwargs["noise_scale"]
    dset = kwargs["dset"]
    use_mbd = kwargs["use_mbd"]
    epoch_size = n_batch_per_epoch * batch_size

    # Setup environment (logging directory etc)
    general_utils.setup_logging(model_name)

    # Load and rescale data
    if dset == "celebA":
        X_real_train = data_utils.load_celebA(img_dim, image_dim_ordering)
    if dset == "mnist":
        X_real_train, _, _, _ = data_utils.load_mnist(image_dim_ordering)
    img_dim = X_real_train.shape[-3:]

    try:

        # Create optimizers
        opt_dcgan = Adam(lr=1E-4, beta_1=0.5, beta_2=0.999, epsilon=1e-08)
        opt_discriminator = Adam(lr=1E-4, beta_1=0.5, beta_2=0.999, epsilon=1e-08)
        # opt_discriminator = SGD(lr=1E-4, momentum=0.9, nesterov=True)

        # Load generator model
        generator_model = models.load("generator_%s" % generator,
                                      cat_dim,
                                      cont_dim,
                                      noise_dim,
                                      img_dim,
                                      bn_mode,
                                      batch_size,
                                      dset=dset,
                                      use_mbd=use_mbd)
        # Load discriminator model
        discriminator_model = models.load("DCGAN_discriminator",
                                          cat_dim,
                                          cont_dim,
                                          noise_dim,
                                          img_dim,
                                          bn_mode,
                                          batch_size,
                                          dset=dset,
                                          use_mbd=use_mbd)

        generator_model.compile(loss='mse', optimizer=opt_discriminator)
        discriminator_model.trainable = False

        DCGAN_model = models.DCGAN(generator_model,
                                   discriminator_model,
                                   cat_dim,
                                   cont_dim,
                                   noise_dim)

        list_losses = ['binary_crossentropy', 'categorical_crossentropy', gaussian_loss]
        list_weights = [1, 1, 1]
        DCGAN_model.compile(loss=list_losses, loss_weights=list_weights, optimizer=opt_dcgan)

        # Multiple discriminator losses
        discriminator_model.trainable = True
        discriminator_model.compile(loss=list_losses, loss_weights=list_weights, optimizer=opt_discriminator)

        gen_loss = 100
        disc_loss = 100

        # Start training
        print("Start training")
        for e in range(nb_epoch):
            # Initialize progbar and batch counter
            progbar = generic_utils.Progbar(epoch_size)
            batch_counter = 1
            start = time.time()

            for X_real_batch in data_utils.gen_batch(X_real_train, batch_size):

                # Create a batch to feed the discriminator model
                X_disc, y_disc, y_cat, y_cont = data_utils.get_disc_batch(X_real_batch,
                                                                          generator_model,
                                                                          batch_counter,
                                                                          batch_size,
                                                                          cat_dim,
                                                                          cont_dim,
                                                                          noise_dim,
                                                                          noise_scale=noise_scale,
                                                                          label_smoothing=label_smoothing,
                                                                          label_flipping=label_flipping)

                # Update the discriminator
                disc_loss = discriminator_model.train_on_batch(X_disc, [y_disc, y_cat, y_cont])

                # Create a batch to feed the generator model
                X_gen, y_gen, y_cat, y_cont, y_cont_target = data_utils.get_gen_batch(batch_size,
                                                                                      cat_dim,
                                                                                      cont_dim,
                                                                                      noise_dim,
                                                                                      noise_scale=noise_scale)

                # Freeze the discriminator
                discriminator_model.trainable = False
                gen_loss = DCGAN_model.train_on_batch([y_cat, y_cont, X_gen], [y_gen, y_cat, y_cont_target])
                # Unfreeze the discriminator
                discriminator_model.trainable = True

                batch_counter += 1
                progbar.add(batch_size, values=[("D tot", disc_loss[0]),
                                                ("D log", disc_loss[1]),
                                                ("D cat", disc_loss[2]),
                                                ("D cont", disc_loss[3]),
                                                ("G tot", gen_loss[0]),
                                                ("G log", gen_loss[1]),
                                                ("G cat", gen_loss[2]),
                                                ("G cont", gen_loss[3])])

                # Save images for visualization
                if batch_counter % (n_batch_per_epoch / 2) == 0:
                    data_utils.plot_generated_batch(X_real_batch, generator_model,
                                                    batch_size, cat_dim, cont_dim, noise_dim, image_dim_ordering)

                if batch_counter >= n_batch_per_epoch:
                    break

            print("")
            print('Epoch %s/%s, Time: %s' % (e + 1, nb_epoch, time.time() - start))

            if e % 5 == 0:
                gen_weights_path = os.path.join('../../models/%s/gen_weights_epoch%s.h5' % (model_name, e))
                generator_model.save_weights(gen_weights_path, overwrite=True)

                disc_weights_path = os.path.join('../../models/%s/disc_weights_epoch%s.h5' % (model_name, e))
                discriminator_model.save_weights(disc_weights_path, overwrite=True)

                DCGAN_weights_path = os.path.join('../../models/%s/DCGAN_weights_epoch%s.h5' % (model_name, e))
                DCGAN_model.save_weights(DCGAN_weights_path, overwrite=True)

    except KeyboardInterrupt:
        pass
Beispiel #40
0
def cross_validate_inmemory(model_name, **kwargs):
    """
    StateFarm competition:
    Training set has 26 unique drivers. We do 26 fold CV where
    a driver is alternatively singled out to be the validation set

    Load the whole train data in memory for faster operations

    args: model (keras model)
          **kwargs (dict) keyword arguments that specify the model hyperparameters
    """

    # Roll out the parameters
    nb_classes = kwargs["nb_classes"]
    batch_size = kwargs["batch_size"]
    n_batch_per_epoch = kwargs["n_batch_per_epoch"]
    nb_epoch = kwargs["nb_epoch"]
    prob = kwargs["prob"]
    do_plot = kwargs["do_plot"]
    data_file = kwargs["data_file"]
    semi_super_file = kwargs["semi_super_file"]
    pretr_weights_file = kwargs["pretr_weights_file"]
    normalisation_style = kwargs["normalisation_style"]
    weak_labels = kwargs["weak_labels"]
    objective = kwargs["objective"]
    experiment = kwargs["experiment"]
    start_fold = kwargs["start_fold"]

    # Load env variables in (in .env file at the root of the project)
    load_dotenv(find_dotenv())

    # Load env variables
    model_dir = os.path.expanduser(os.environ.get("MODEL_DIR"))
    data_dir = os.path.expanduser(os.environ.get("DATA_DIR"))

    # Output path where we store experiment log and weights
    model_dir = os.path.join(model_dir, model_name)
    # Create if it does not exist
    general_utils.create_dir(model_dir)
    # Automatically determine experiment name
    list_exp = glob.glob(model_dir + "/*")
    # Create the experiment dir and weights dir
    if experiment:
        exp_dir = os.path.join(model_dir, experiment)
    else:
        exp_dir = os.path.join(model_dir, "Experiment_%s" % len(list_exp))
    general_utils.create_dir(exp_dir)

    # Compile model.
    # opt = RMSprop(lr=5E-6, rho=0.9, epsilon=1e-06)
    opt = SGD(lr=5e-4, decay=1e-6, momentum=0.9, nesterov=True)
    # opt = Adam(lr=1E-5, beta_1=0.9, beta_2=0.999, epsilon=1e-08)

    # Batch generator
    DataAug = batch_utils.AugDataGenerator(data_file,
                                           batch_size=batch_size,
                                           prob=prob,
                                           dset="train",
                                           maxproc=4,
                                           num_cached=60,
                                           random_augm=False,
                                           hdf5_file_semi=semi_super_file)
    DataAug.add_transform("h_flip")
    # DataAug.add_transform("v_flip")
    # DataAug.add_transform("fixed_rot", angle=40)
    DataAug.add_transform("random_rot", angle=40)
    # DataAug.add_transform("fixed_tr", tr_x=40, tr_y=40)
    DataAug.add_transform("random_tr", tr_x=40, tr_y=40)
    # DataAug.add_transform("fixed_blur", kernel_size=5)
    DataAug.add_transform("random_blur", kernel_size=5)
    # DataAug.add_transform("fixed_erode", kernel_size=4)
    DataAug.add_transform("random_erode", kernel_size=3)
    # DataAug.add_transform("fixed_dilate", kernel_size=4)
    DataAug.add_transform("random_dilate", kernel_size=3)
    # DataAug.add_transform("fixed_crop", pos_x=10, pos_y=10, crop_size_x=200, crop_size_y=200)
    DataAug.add_transform("random_crop", min_crop_size=140, max_crop_size=160)
    # DataAug.add_transform("hist_equal")
    # DataAug.add_transform("random_occlusion", occ_size_x=100, occ_size_y=100)

    epoch_size = n_batch_per_epoch * batch_size

    general_utils.pretty_print("Load all data...")

    with h5py.File(data_file, "r") as hf:
        X = hf["train_data"][:, :, :, :]
        y = hf["train_label"][:].astype(np.uint8)
        y = np_utils.to_categorical(y, nb_classes=nb_classes)  # Format for keras

        try:
            for fold in range(start_fold, 8):
                # for fold in np.random.permutation(26):

                min_valid_loss = 100

                # Save losses
                list_train_loss = []
                list_valid_loss = []

                # Load valid data in memory for fast error evaluation
                idx_valid = hf["valid_fold%s" % fold][:]
                idx_train = hf["train_fold%s" % fold][:]
                X_valid = X[idx_valid]
                y_valid = y[idx_valid]

                # Normalise
                X_valid = normalisation(X_valid, normalisation_style)

                # Compile model
                general_utils.pretty_print("Compiling...")
                model = models.load(model_name,
                                    nb_classes,
                                    X_valid.shape[-3:],
                                    pretr_weights_file=pretr_weights_file)
                model.compile(optimizer=opt, loss=objective)

                # Save architecture
                json_string = model.to_json()
                with open(os.path.join(data_dir, '%s_archi.json' % model.name), 'w') as f:
                    f.write(json_string)

                for e in range(nb_epoch):
                    # Initialize progbar and batch counter
                    progbar = generic_utils.Progbar(epoch_size)
                    batch_counter = 1
                    l_train_loss = []
                    start = time.time()

                    for X_train, y_train in DataAug.gen_batch_inmemory(X, y, idx_train=idx_train):
                        if do_plot:
                            general_utils.plot_batch(X_train, np.argmax(y_train, 1), batch_size)

                        # Normalise
                        X_train = normalisation(X_train, normalisation_style)

                        train_loss = model.train_on_batch(X_train, y_train)
                        l_train_loss.append(train_loss)
                        batch_counter += 1
                        progbar.add(batch_size, values=[("train loss", train_loss)])
                        if batch_counter >= n_batch_per_epoch:
                            break
                    print("")
                    print('Epoch %s/%s, Time: %s' % (e + 1, nb_epoch, time.time() - start))
                    y_valid_pred = model.predict(X_valid, verbose=0, batch_size=16)
                    train_loss = float(np.mean(l_train_loss))  # use float to make it json saveable
                    valid_loss = log_loss(y_valid, y_valid_pred)
                    print("Train loss:", train_loss, "valid loss:", valid_loss)
                    list_train_loss.append(train_loss)
                    list_valid_loss.append(valid_loss)

                    # Record experimental data in a dict
                    d_log = {}
                    d_log["fold"] = fold
                    d_log["nb_classes"] = nb_classes
                    d_log["batch_size"] = batch_size
                    d_log["n_batch_per_epoch"] = n_batch_per_epoch
                    d_log["nb_epoch"] = nb_epoch
                    d_log["epoch_size"] = epoch_size
                    d_log["prob"] = prob
                    d_log["optimizer"] = opt.get_config()
                    d_log["augmentator_config"] = DataAug.get_config()
                    d_log["train_loss"] = list_train_loss
                    d_log["valid_loss"] = list_valid_loss

                    json_file = os.path.join(exp_dir, 'experiment_log_fold%s.json' % fold)
                    general_utils.save_exp_log(json_file, d_log)

                    # Only save the best epoch
                    if valid_loss < min_valid_loss:
                        min_valid_loss = valid_loss
                        trained_weights_path = os.path.join(exp_dir, '%s_weights_fold%s.h5' % (model.name, fold))
                        model.save_weights(trained_weights_path, overwrite=True)

        except KeyboardInterrupt:
            pass
Beispiel #41
0
            cherrypy.log('Admin account already created, skipping.')
            os._exit(0)

        user = models.Users.new(username='******', user_group='admin', name='Administrator')
        user.password = models.Users.hash_password('puck')
        if models.Users.add(user):
            cherrypy.log('Admin account added.')
            cherrypy.log('\tUsername: admin')
            cherrypy.log('\tPassword: puck')
            os._exit(0)
        else:
            cherrypy.log('An error occured while creating the admin account.')
            os._exit(1)

    cherrypy.log("Loading models from persistent storage.")
    models.load()

    cherrypy.log("Loading virtualization plugin.")
    cherrypy.engine.virtualization = VirtualizationPlugin(cherrypy.engine)
    cherrypy.engine.virtualization.subscribe()
    models.Credential = cherrypy.engine.virtualization.get_credential_class()

    cherrypy.log("Loading controllers.")
    root = controllers.RootController(lookup)
    root.add('api', controllers.Api)
    root.add('environments', controllers.EnvironmentsController)
    root.add('firewalls', controllers.FirewallsController)
    root.add('images', controllers.ImagesController)
    root.add('jail_types', controllers.JailTypesController)
    root.add('jails', controllers.JailsController)
    root.add('keys', controllers.KeysController)
Beispiel #42
0
                                    device=device)

lr_scheduler = create_lr_scheduler(optimizer, **vars(args))

start_timestamp = int(time.time() * 1000)
start_epoch = 0
best_loss = 1e10
best_metric = 0
best_accuracy = 0
global_step = 0

if args.resume:
    print("resuming a checkpoint '%s'" % args.resume)
    if os.path.exists(args.resume):
        saved_checkpoint = torch.load(args.resume)
        old_model = models.load(saved_checkpoint['model_file'])
        model.module.load_state_dict(old_model.state_dict())
        model.float()

        if not args.resume_without_optimizer:
            optimizer.load_state_dict(saved_checkpoint['optimizer'])
            lr_scheduler.load_state_dict(saved_checkpoint['lr_scheduler'])
            best_loss = saved_checkpoint.get('best_loss', best_loss)
            best_metric = saved_checkpoint.get('best_metric', best_metric)
            best_accuracy = saved_checkpoint.get('best_accuracy', best_accuracy)
            start_epoch = saved_checkpoint.get('epoch', start_epoch)
            global_step = saved_checkpoint.get('step', global_step)

        del saved_checkpoint  # reduce memory
        del old_model
    else:
Beispiel #43
0
def train(**kwargs):
    """
    Train model

    Load the whole train data in memory for faster operations

    args: **kwargs (dict) keyword arguments that specify the model hyperparameters
    """

    # Roll out the parameters
    batch_size = kwargs["batch_size"]
    n_batch_per_epoch = kwargs["n_batch_per_epoch"]
    nb_epoch = kwargs["nb_epoch"]
    model_name = kwargs["model_name"]
    generator = kwargs["generator"]
    image_data_format = kwargs["image_data_format"]
    img_dim = kwargs["img_dim"]
    patch_size = kwargs["patch_size"]
    bn_mode = kwargs["bn_mode"]
    label_smoothing = kwargs["use_label_smoothing"]
    label_flipping = kwargs["label_flipping"]
    dset = kwargs["dset"]
    use_mbd = kwargs["use_mbd"]
    lastLayerActivation=kwargs["lastLayerActivation"]
    PercentageOfTrianable=kwargs["PercentageOfTrianable"]
    SpecificPathStr=kwargs["SpecificPathStr"]
    epoch_size = n_batch_per_epoch * batch_size

    # Setup environment (logging directory etc)
    #general_utils.setup_logging(model_name)

    # Load and rescale data
    #X_full_train, X_sketch_train, X_full_val, X_sketch_val = data_utils.load_data(dset, image_data_format)
    img_dim = (256,256,3) # Manual entry

    # Get the number of non overlapping patch and the size of input image to the discriminator
    nb_patch, img_dim_disc = data_utils.get_nb_patch(img_dim, patch_size, image_data_format)

    try:

        # Create optimizers
        opt_dcgan = Adam(lr=1E-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
        # opt_discriminator = SGD(lr=1E-3, momentum=0.9, nesterov=True)
        opt_discriminator = Adam(lr=1E-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)

        # Load generator model
        """
        generator_model = models.load("generator_unet_%s" % generator,
                                      img_dim,
                                      nb_patch,
                                      bn_mode,
                                      use_mbd,
                                      batch_size)
        """
        generator_model=CreatErrorMapModel(input_shape=img_dim,lastLayerActivation=lastLayerActivation, PercentageOfTrianable=PercentageOfTrianable)
        # Load discriminator model
        discriminator_model = models.load("DCGAN_discriminator",
                                          img_dim_disc,
                                          nb_patch,
                                          bn_mode,
                                          use_mbd,
                                          batch_size)

         generator_model.compile(loss='mae', optimizer=opt_discriminator)
#-------------------------------------------------------------------------------
         logpath=os.path.join('../../log','DepthMapWith'+lastLayerActivation+str(PercentageOfTrianable)+'UnTr'+SpecificPathStr)
         modelPath=os.path.join('../../models','DepthMapwith'+lastLayerActivation+str(PercentageOfTrianable)+'Untr'+SpecificPathStr)
         os.makedirs(logpath, exist_ok=True)
         os.makedirs(modelPath, exist_ok=True)os.makedirs(modelPath, exist_ok=True)

#-----------------------PreTraining Depth Map-------------------------------------
         nb_train_samples = 2000
         nb_validation_samples = 
         epochs = 20
         history=whole_model.fit_generator(data_utils.facades_generator(img_dim,batch_size=batch_size), samples_per_epoch=nb_train_samples,epochs=epochs,validation_data=data_utils.facades_generator(img_dim,batch_size=batch_size),nb_val_samples=nb_validation_    samples,       callbacks=[
         keras.callbacks.ModelCheckpoint(os.path.join(modelPath,'DepthMap_weightsBestLoss.h5'), monitor='val_loss', verbose=1, save_best_only=True),
         keras.callbacks.ModelCheckpoint(os.path.join(modelPath,'DepthMap_weightsBestAcc.h5'), monitor='acc', verbose=1, save_best_only=True),
         keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.1, patience=2, verbose=1, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0),
         keras.callbacks.TensorBoard(log_dir=logpath, histogram_freq=0, batch_size=batchSize, write_graph=True, write_grads=False, write_images=True, embeddin    gs_freq=0, embeddings_layer_names=None, embeddings_metadata=None)],)
#------------------------------------------------------------------------------------


        discriminator_model.trainable = False

        DCGAN_model = models.DCGAN(generator_model,
                                   discriminator_model,
                                   img_dim,
                                   patch_size,
                                   image_data_format)

        loss = [l1_loss, 'binary_crossentropy']
        loss_weights = [1E1, 1]
        DCGAN_model.compile(loss=loss, loss_weights=loss_weights, optimizer=opt_dcgan)

        discriminator_model.trainable = True
        discriminator_model.compile(loss='binary_crossentropy', optimizer=opt_discriminator)

        gen_loss = 100
        disc_loss = 100

        # Start training
        print("Start training")
        for e in range(nb_epoch):
            # Initialize progbar and batch counter
            progbar = generic_utils.Progbar(epoch_size)
            batch_counter = 1
            start = time.time()

            for X_full_batch, X_sketch_batch in data_utils.facades_generator(img_dim,batch_size=batch_size):

                # Create a batch to feed the discriminator model
                X_disc, y_disc = data_utils.get_disc_batch(X_full_batch,
                                                           X_sketch_batch,
                                                           generator_model,
                                                           batch_counter,
                                                           patch_size,
                                                           image_data_format,
                                                           label_smoothing=label_smoothing,
                                                           label_flipping=label_flipping)

                # Update the discriminator
                disc_loss = discriminator_model.train_on_batch(X_disc, y_disc) # X_disc, y_disc
                # Create a batch to feed the generator model
                X_gen_target, X_gen = next(data_utils.facades_generator(img_dim,batch_size=batch_size))
                y_gen = np.zeros((X_gen.shape[0], 2), dtype=np.uint8)
                y_gen[:, 1] = 1

                # Freeze the discriminator
                discriminator_model.trainable = False
                gen_loss = DCGAN_model.train_on_batch(X_gen, [X_gen_target, y_gen])
                # Unfreeze the discriminator
                discriminator_model.trainable = True

                batch_counter += 1
                progbar.add(batch_size, values=[("D logloss", disc_loss),
                                                ("G tot", gen_loss[0]),
                                                ("G L1", gen_loss[1]),
                                                ("G logloss", gen_loss[2])])

                # Save images for visualization
                if batch_counter % (n_batch_per_epoch / 2) == 0:
                    # Get new images from validation
                    figure_name = "training_"+str(e)
                    data_utils.plot_generated_batch(X_full_batch, X_sketch_batch, generator_model,
                                                    batch_size, image_data_format, figure_name)

                if batch_counter >= n_batch_per_epoch:
                    break

            print("")
            print('Epoch %s/%s, Time: %s' % (e + 1, nb_epoch, time.time() - start))

            if e % 5 == 0:
                gen_weights_path = os.path.join('../../models/%s/gen_weights_epoch%s.h5' % (model_name, e))
                generator_model.save_weights(gen_weights_path, overwrite=True)

                disc_weights_path = os.path.join('../../models/%s/disc_weights_epoch%s.h5' % (model_name, e))
                discriminator_model.save_weights(disc_weights_path, overwrite=True)

                DCGAN_weights_path = os.path.join('../../models/%s/DCGAN_weights_epoch%s.h5' % (model_name, e))
                DCGAN_model.save_weights(DCGAN_weights_path, overwrite=True)