コード例 #1
0
    def validation(iteration):
        model.eval()
        repeat_num = args.eval_repeat_num
        all_gt_fix_vectors = []
        all_predict_fix_vectors = []
        with tqdm(total=len(validation_loader) * repeat_num) as pbar_val:
            for i_batch, batch in enumerate(validation_loader):
                tmp = [batch["images"], batch["fix_vectors"]]
                tmp = [_ if not torch.is_tensor(_) else _.cuda() for _ in tmp]
                images, gt_fix_vectors = tmp
                N, C, H, W = images.shape

                with torch.no_grad():
                    predict = model(images)

                log_normal_mu = predict["log_normal_mu"]
                log_normal_sigma2 = predict["log_normal_sigma2"]
                all_actions_prob = predict["all_actions_prob"]

                for trial in range(repeat_num):
                    all_gt_fix_vectors.extend(gt_fix_vectors)

                    samples = sampling.random_sample(all_actions_prob, log_normal_mu, log_normal_sigma2)
                    prob_sample_actions = samples["selected_actions_probs"]
                    durations = samples["durations"]
                    sample_actions = samples["selected_actions"]
                    sampling_random_predict_fix_vectors, _, _ = sampling.generate_scanpath(
                        images, prob_sample_actions, durations, sample_actions)
                    all_predict_fix_vectors.extend(sampling_random_predict_fix_vectors)

                    pbar_val.update(1)

        cur_metrics, cur_metrics_std, _ = evaluation(all_gt_fix_vectors, all_predict_fix_vectors)

        # Print and log all evaluation metrics to tensorboard.
        logger.info("Evaluation metrics after iteration {iteration}:".format(iteration=iteration))
        for metrics_key in cur_metrics.keys():
            for (metric_name, metric_value) in cur_metrics[metrics_key].items():
                tensorboard_writer.add_scalar(
                    "metrics/{metrics_key}-{metric_name}".format(metrics_key=metrics_key, metric_name=metric_name),
                    metric_value, iteration
                )
                logger.info("{metrics_key:10}-{metric_name:15}: {metric_value:.4f} +- {std:.4f}".format
                            (metrics_key=metrics_key, metric_name=metric_name, metric_value=metric_value,
                             std=cur_metrics_std[metrics_key][metric_name]))

        return cur_metrics
コード例 #2
0
def train_reader(args):
    strategies = args.strategies
    seeds = args.seeds[:args.run_cnt]

    for idx, (seed, strategy) in enumerate(product(seeds, strategies)):
        wandb.init(project="p-stage-3", reinit=True)
        args = update_args(args,
                           strategy)  # auto add args.save_path, args.base_path
        args.strategy, args.seed = strategy, seed
        args.info = Namespace()
        set_seed(seed)

        checkpoint_dir = glob(p.join(args.path.checkpoint, f"{strategy}*"))
        if not checkpoint_dir:
            raise FileNotFoundError(
                f"{strategy} 전략에 대한 checkpoint가 존재하지 않습니다.")

        args.model.model_path = get_last_checkpoint(checkpoint_dir[0])
        if args.model.model_path is None:
            raise FileNotFoundError(
                f"{checkpoint_dir[0]} 경로에 체크포인트가 존재하지 않습니다.")

        # run_name: strategy + alias + seed
        args.train.run_name = "_".join([strategy, args.alias, str(seed)])
        args.train.output_dir = p.join(args.path.checkpoint,
                                       args.train.run_name)
        wandb.run.name = args.train.run_name

        print("checkpoint_dir: ", args.train.output_dir)

        datasets = get_dataset(args, is_train=True)
        reader = get_reader(args, eval_answers=datasets["validation"])
        retriever = get_retriever(args)

        datasets["validation"] = retriever.retrieve(
            datasets["validation"], topk=args.retriever.topk)["validation"]
        reader.set_dataset(eval_dataset=datasets["validation"])

        trainer = reader.get_trainer()

        if args.train.do_eval:
            metric_results = trainer.evaluate()
            results = evaluation(args)

            metric_results["predictions"]["exact_match"] = results["EM"][
                "value"]
            metric_results["predictions"]["f1"] = results["F1"]["value"]

            print("EVAL RESULT")
            print(metric_results["predictions"])

        if args.train.do_eval and args.train.pororo_prediction:
            assert metric_results is not None, "trainer.evaluate()가 None을 반환합니다."

            results = evaluation(args, prefix="pororo_")

            metric_results["pororo_predictions"]["exact_match"] = results[
                "EM"]["value"]
            metric_results["pororo_predictions"]["f1"] = results["F1"]["value"]

            print("PORORO EVAL RESULT")
            print(metric_results["pororo_predictions"])

        if args.train.do_eval and args.report:
            report_reader_to_slack(args,
                                   p.basename(__file__),
                                   metric_results["predictions"],
                                   use_pororo=False)

            if args.train.pororo_prediction:
                report_reader_to_slack(args,
                                       p.basename(__file__),
                                       metric_results["pororo_predictions"],
                                       use_pororo=True)
コード例 #3
0
def train_reader(args):
    strategies = args.strategies
    seeds = args.seeds[:args.run_cnt]

    for idx, (seed, strategy) in enumerate(product(seeds, strategies)):
        wandb.init(project="p-stage-3", reinit=True)
        args = update_args(args,
                           strategy)  # auto add args.save_path, args.base_path
        args.strategy, args.seed = strategy, seed
        args.info = Namespace()
        set_seed(seed)

        # below codes must run before 'reader.get_trainer()'
        # run_name: strategy + alias + seed
        args.train.run_name = "_".join([strategy, args.alias, str(seed)])
        args.train.output_dir = p.join(args.path.checkpoint,
                                       args.train.run_name)
        wandb.run.name = args.train.run_name

        print("checkpoint_dir: ", args.train.output_dir)

        # retrieve 과정이 없어 top-k를 반환할 수 없음. 무조건 top-1만 반환
        # run_mrc.py DOES NOT execute retrieve, so args.retriever.topk cannot be n(>1).
        # If topk > 1, post processing function returns mis-bundled predictions.
        args.retriever.topk = 1

        datasets = get_dataset(args, is_train=True)
        reader = get_reader(args, eval_answers=datasets["validation"])

        reader.set_dataset(train_dataset=datasets["train"],
                           eval_dataset=datasets["validation"])

        trainer = reader.get_trainer()

        if args.train.do_train:
            train_results = trainer.train()
            print(train_results)

        metric_results = None

        if args.train.do_eval:
            metric_results = trainer.evaluate()
            results = evaluation(args)

            metric_results["predictions"]["exact_match"] = results["EM"][
                "value"]
            metric_results["predictions"]["f1"] = results["F1"]["value"]

            print("EVAL RESULT")
            print(metric_results["predictions"])

        if args.train.do_eval and args.train.pororo_prediction:
            assert metric_results is not None, "trainer.evaluate()가 None을 반환합니다."

            results = evaluation(args, prefix="pororo_")

            metric_results["pororo_predictions"]["exact_match"] = results[
                "EM"]["value"]
            metric_results["pororo_predictions"]["f1"] = results["F1"]["value"]

            print("PORORO EVAL RESULT")
            print(metric_results["pororo_predictions"])

        if args.train.do_eval and args.report:
            report_reader_to_slack(args,
                                   p.basename(__file__),
                                   metric_results["predictions"],
                                   use_pororo=False)

            if args.train.pororo_prediction:
                report_reader_to_slack(args,
                                       p.basename(__file__),
                                       metric_results["pororo_predictions"],
                                       use_pororo=True)
コード例 #4
0
def main():

    # load logger
    log_dir = args.evaluation_dir
    checkpoints_dir = os.path.join(log_dir, "checkpoints")
    log_file = os.path.join(log_dir, "log_validation.txt")
    predicts_file = os.path.join(log_dir, "validation_predicts.json")
    logger = Logger(log_file)

    logger.info("The args corresponding to validation process are: ")
    for (key, value) in vars(args).items():
        logger.info("{key:20}: {value:}".format(key=key, value=value))

    validation_dataset = COCO_Search18_evaluation(
        args.img_dir,
        args.fix_dir,
        args.detector_dir,
        type="validation",
        split="split3",
        transform=transform,
        detector_threshold=args.detector_threshold)

    validation_loader = DataLoader(dataset=validation_dataset,
                                   batch_size=args.batch,
                                   shuffle=False,
                                   num_workers=4,
                                   collate_fn=validation_dataset.collate_func)

    object_name = [
        "bottle", "bowl", "car", "chair", "clock", "cup", "fork", "keyboard",
        "knife", "laptop", "microwave", "mouse", "oven", "potted plant",
        "sink", "stop sign", "toilet", "tv"
    ]

    model = baseline(embed_size=512,
                     convLSTM_length=args.max_length,
                     min_length=args.min_length).cuda()

    sampling = Sampling(convLSTM_length=args.max_length,
                        min_length=args.min_length)

    # Load checkpoint to start evaluation.
    # Infer iteration number through file name (it's hacky but very simple), so don't rename
    validation_checkpoint = torch.load(
        os.path.join(checkpoints_dir, "checkpoint_best.pth"))
    for key in validation_checkpoint:
        if key == "optimizer":
            continue
        else:
            model.load_state_dict(validation_checkpoint[key], strict=False)

    if len(args.gpu_ids) > 1:
        model = nn.DataParallel(model, args.gpu_ids)

    # get the human baseline score
    human_metrics, human_metrics_std, gt_scores_of_each_images = human_evaluation(
        validation_loader)
    logger.info("The metrics for human performance are: ")
    for metrics_key in human_metrics.keys():
        for (key, value) in human_metrics[metrics_key].items():
            logger.info(
                "{metrics_key:10}-{key:15}: {value:.4f} +- {std:.4f}".format(
                    metrics_key=metrics_key,
                    key=key,
                    value=value,
                    std=human_metrics_std[metrics_key][key]))

    model.eval()
    repeat_num = args.eval_repeat_num
    all_gt_fix_vectors = []
    all_predict_fix_vectors = []
    predict_results = []
    with tqdm(total=len(validation_loader) * repeat_num) as pbar_val:
        for i_batch, batch in enumerate(validation_loader):
            tmp = [
                batch["images"], batch["fix_vectors"], batch["attention_maps"],
                batch["tasks"], batch["img_names"]
            ]
            tmp = [_ if not torch.is_tensor(_) else _.cuda() for _ in tmp]
            images, gt_fix_vectors, attention_maps, tasks, img_names = tmp

            N, C, H, W = images.shape

            if args.ablate_attention_info:
                attention_maps *= 0

            with torch.no_grad():
                predict = model(images, attention_maps, tasks)

            log_normal_mu = predict["log_normal_mu"]
            log_normal_sigma2 = predict["log_normal_sigma2"]
            all_actions_prob = predict["all_actions_prob"]

            for trial in range(repeat_num):
                all_gt_fix_vectors.extend(gt_fix_vectors)

                samples = sampling.random_sample(all_actions_prob,
                                                 log_normal_mu,
                                                 log_normal_sigma2)
                prob_sample_actions = samples["selected_actions_probs"]
                durations = samples["durations"]
                sample_actions = samples["selected_actions"]
                sampling_random_predict_fix_vectors, _, _ = sampling.generate_scanpath(
                    images, prob_sample_actions, durations, sample_actions)
                all_predict_fix_vectors.extend(
                    sampling_random_predict_fix_vectors)

                for index in range(N):
                    predict_result = dict()
                    one_sampling_random_predict_fix_vectors = sampling_random_predict_fix_vectors[
                        index]
                    fix_vector_array = np.array(
                        one_sampling_random_predict_fix_vectors.tolist())
                    predict_result["img_names"] = img_names[index]
                    predict_result["task"] = object_name[tasks[index]]
                    predict_result["repeat_id"] = trial + 1
                    predict_result["X"] = list(fix_vector_array[:, 0])
                    predict_result["Y"] = list(fix_vector_array[:, 1])
                    predict_result["T"] = list(fix_vector_array[:, 2] * 1000)
                    predict_result["length"] = len(predict_result["X"])
                    predict_results.append(predict_result)

                pbar_val.update(1)

    cur_metrics, cur_metrics_std, _ = evaluation(all_gt_fix_vectors,
                                                 all_predict_fix_vectors)

    with open(predicts_file, 'w') as f:
        json.dump(predict_results, f, indent=2)

    logger.info("The metrics for best model performance are: ")
    for metrics_key in cur_metrics.keys():
        for (metric_name, metric_value) in cur_metrics[metrics_key].items():
            logger.info(
                "{metrics_key:10}-{metric_name:15}: {metric_value:.4f} +- {std:.4f}"
                .format(metrics_key=metrics_key,
                        metric_name=metric_name,
                        metric_value=metric_value,
                        std=cur_metrics_std[metrics_key][metric_name]))
コード例 #5
0
                flow = RealNVP(nets,
                               nett,
                               num_flows,
                               prior,
                               dequantization=True)

            # OPTIMIZER
            optimizer = torch.optim.Adamax(
                [p for p in flow.parameters() if p.requires_grad == True],
                lr=lr)

            # TRAINING
            nll_val = training(name=result_dir + name,
                               max_patience=max_patience,
                               num_epochs=num_epochs,
                               flow=flow,
                               optimizer=optimizer,
                               training_loader=training_loader,
                               val_loader=val_loader)

            # EVALUATION
            test_loss = evaluation(name=result_dir + name,
                                   test_loader=test_loader)
            f = open(result_dir + name + '_test_loss.txt', "w")
            f.write(str(test_loss))
            f.close()

            samples_real(result_dir + name, test_loader)

            plot_curve(result_dir + name, nll_val)
コード例 #6
0
ファイル: tansaku.py プロジェクト: 0z4ck/zeta_zero
def minimax(board, kmd, okmd, mv, depth, c_depth, sente):
    if depth == 0:
        return evaluation.evaluation(board, kmd, okmd, mv)
    movelist = []
    y = 0
    if c_depth % 2 == 0:
        for line in board:
            x = 0
            for square in line:
                if square == "P":
                    mlist = mv.pawn(x, y)

                    mlist2 = filter(
                        lambda xy: not board[xy[1]][xy[0]].isupper(), mlist)
                    for x2, y2 in mlist2:
                        if util.isThisMoveLegal(
                                copy.deepcopy(board),
                            (square.upper(), x, x2, y, y2)):
                            usimove = util.rawtousi(square.lower(), x, x2, y,
                                                    y2, sente)
                            if y2 < 3:
                                usimove += "+"
                            movelist.append(usimove)
                elif square == "L":
                    mlist = mv.lance(x, y)
                    mlist2 = []
                    for lx, ly in mlist:
                        if board[ly][lx] == "":
                            mlist2.append((lx, ly))
                        elif board[ly][lx].isupper():
                            break
                        else:
                            mlist2.append((lx, ly))
                            break

                    for x2, y2 in mlist2:
                        if util.isThisMoveLegal(
                                copy.deepcopy(board),
                            (square.upper(), x, x2, y, y2)):
                            usimove = util.rawtousi(square.lower(), x, x2, y,
                                                    y2, sente)
                            if y2 < 3:
                                usimove += "+"
                            movelist.append(usimove)
                elif square == "N":
                    mlist = mv.knight(x, y)

                    mlist2 = filter(
                        lambda xy: not board[xy[1]][xy[0]].isupper(), mlist)
                    for x2, y2 in mlist2:
                        if util.isThisMoveLegal(
                                copy.deepcopy(board),
                            (square.upper(), x, x2, y, y2)):
                            usimove = util.rawtousi(square.lower(), x, x2, y,
                                                    y2, sente)
                            if y2 < 3:
                                usimove += "+"
                            movelist.append(usimove)
                elif square == "S":
                    mlist = mv.silver(x, y)

                    mlist2 = filter(
                        lambda xy: not board[xy[1]][xy[0]].isupper(), mlist)
                    for x2, y2 in mlist2:
                        if util.isThisMoveLegal(
                                copy.deepcopy(board),
                            (square.upper(), x, x2, y, y2)):
                            usimove = util.rawtousi(square.lower(), x, x2, y,
                                                    y2, sente)
                            if y2 < 3:
                                usimove += "+"
                            movelist.append(usimove)
                elif square == "G":
                    mlist = mv.gold(x, y)

                    mlist2 = filter(
                        lambda xy: not board[xy[1]][xy[0]].isupper(), mlist)
                    for x2, y2 in mlist2:
                        if util.isThisMoveLegal(
                                copy.deepcopy(board),
                            (square.upper(), x, x2, y, y2)):
                            usimove = util.rawtousi(square.lower(), x, x2, y,
                                                    y2, sente)
                            movelist.append(usimove)
                elif square == "K":
                    mlist = mv.king(x, y)
                    mlist2 = filter(
                        lambda xy: not board[xy[1]][xy[0]].isupper(), mlist)

                    for x2, y2 in mlist2:
                        if util.isThisMoveLegal(
                                copy.deepcopy(board),
                            (square.upper(), x, x2, y, y2)):
                            usimove = util.rawtousi(square.lower(), x, x2, y,
                                                    y2, sente)
                            movelist.append(usimove)
                elif square == "R":
                    mlist = mv.rook(x, y)
                    mlist2 = []
                    #print(mlist)
                    for mlpart in mlist[:4]:
                        for lx, ly in mlpart:
                            if board[ly][lx] == "":
                                mlist2.append((lx, ly))
                            elif board[ly][lx].isupper():
                                break
                            else:
                                mlist2.append((lx, ly))
                                break
                    for x2, y2 in mlist2:
                        if util.isThisMoveLegal(
                                copy.deepcopy(board),
                            (square.upper(), x, x2, y, y2)):
                            usimove = util.rawtousi(square.lower(), x, x2, y,
                                                    y2, sente)
                            if y2 < 3:
                                usimove += "+"
                            movelist.append(usimove)
                elif square == "B":
                    mlist = mv.bishop(x, y)
                    mlist2 = []
                    #print(mlist)
                    for mlpart in mlist[:4]:
                        for lx, ly in mlpart:
                            if board[ly][lx] == "":
                                mlist2.append((lx, ly))
                            elif board[ly][lx].isupper():
                                break
                            else:
                                mlist2.append((lx, ly))
                                break

                    for x2, y2 in mlist2:
                        if util.isThisMoveLegal(
                                copy.deepcopy(board),
                            (square.upper(), x, x2, y, y2)):
                            usimove = util.rawtousi(square.lower(), x, x2, y,
                                                    y2, sente)
                            if y2 < 3:
                                usimove += "+"
                            movelist.append(usimove)
                elif square == "+P" or square == "+L" or square == "+N" or square == "+S":
                    mlist = mv.gold(x, y)

                    mlist2 = filter(
                        lambda xy: not board[xy[1]][xy[0]].isupper(), mlist)
                    for x2, y2 in mlist2:
                        if util.isThisMoveLegal(
                                copy.deepcopy(board),
                            (square.upper(), x, x2, y, y2)):
                            usimove = util.rawtousi(square.lower(), x, x2, y,
                                                    y2, sente)
                            movelist.append(usimove)
                elif square == "+R":
                    mlist = mv.rook(x, y, dragon=True)
                    mlist2 = []
                    for mlpart in mlist[:4]:
                        for lx, ly in mlpart:
                            if board[ly][lx] == "":
                                mlist2.append((lx, ly))
                            elif board[ly][lx].isupper():
                                break
                            else:
                                mlist2.append((lx, ly))
                                break
                    for lx, ly in mlist[4]:
                        mlist2.append((lx, ly))
                    for x2, y2 in mlist2:
                        if util.isThisMoveLegal(
                                copy.deepcopy(board),
                            (square.upper(), x, x2, y, y2)):
                            usimove = util.rawtousi(square.lower(), x, x2, y,
                                                    y2, sente)
                            movelist.append(usimove)
                elif square == "+B":
                    mlist = mv.bishop(x, y, horse=True)
                    mlist2 = []
                    for mlpart in mlist[:4]:
                        for lx, ly in mlpart:
                            if board[ly][lx] == "":
                                mlist2.append((lx, ly))
                            elif board[ly][lx].isupper():
                                break
                            else:
                                mlist2.append((lx, ly))
                                break
                    for lx, ly in mlist[4]:
                        mlist2.append((lx, ly))

                    for x2, y2 in mlist2:
                        if util.isThisMoveLegal(
                                copy.deepcopy(board),
                            (square.upper(), x, x2, y, y2)):
                            usimove = util.rawtousi(square.lower(), x, x2, y,
                                                    y2, sente)
                            movelist.append(usimove)
                x += 1
            y += 1
        sfeny = ["a", "b", "c", "d", "e", "f", "g", "h", "i"]
        for koma in kmd:
            for tempy in range(9):
                for tempx in range(9):
                    if board[tempy][tempx] == "":
                        if sente:
                            usimove = "{}*{}{}".format(koma.upper(), tempx + 1,
                                                       sfeny[tempy])
                        else:
                            usimove = "{}*{}{}".format(koma.upper(), 9 - tempx,
                                                       sfeny[8 - tempy])
                        if util.isThisDropLegal(copy.deepcopy(board),
                                                (koma.upper(), tempx, tempy)):
                            movelist.append(usimove)
    else:
        for line in board:
            x = 0
            for square in line:
                if square == "p":
                    mlist = mv.pawn(x, y, me=False)

                    mlist2 = filter(
                        lambda xy: not board[xy[1]][xy[0]].islower(), mlist)
                    for x2, y2 in mlist2:
                        if util.isThisMoveLegal(
                                copy.deepcopy(board),
                            (square.upper(), x, x2, y, y2)):
                            usimove = util.rawtousi(square.lower(), x, x2, y,
                                                    y2, sente)
                            if y2 < 3:
                                usimove += "+"
                            movelist.append(usimove)
                elif square == "l":
                    mlist = mv.lance(x, y, me=False)
                    mlist2 = []
                    for lx, ly in mlist:
                        if board[ly][lx] == "":
                            mlist2.append((lx, ly))
                        elif board[ly][lx].islower():
                            break
                        else:
                            mlist2.append((lx, ly))
                            break

                    for x2, y2 in mlist2:
                        if util.isThisMoveLegal(
                                copy.deepcopy(board),
                            (square.upper(), x, x2, y, y2)):
                            usimove = util.rawtousi(square.lower(), x, x2, y,
                                                    y2, sente)
                            if y2 < 3:
                                usimove += "+"
                            movelist.append(usimove)
                elif square == "n":
                    mlist = mv.knight(x, y, me=False)

                    mlist2 = filter(
                        lambda xy: not board[xy[1]][xy[0]].islower(), mlist)
                    for x2, y2 in mlist2:
                        if util.isThisMoveLegal(
                                copy.deepcopy(board),
                            (square.upper(), x, x2, y, y2)):
                            usimove = util.rawtousi(square.lower(), x, x2, y,
                                                    y2, sente)
                            if y2 < 3:
                                usimove += "+"
                            movelist.append(usimove)
                elif square == "s":
                    mlist = mv.silver(x, y, me=False)

                    mlist2 = filter(
                        lambda xy: not board[xy[1]][xy[0]].islower(), mlist)
                    for x2, y2 in mlist2:
                        if util.isThisMoveLegal(
                                copy.deepcopy(board),
                            (square.upper(), x, x2, y, y2)):
                            usimove = util.rawtousi(square.lower(), x, x2, y,
                                                    y2, sente)
                            if y2 < 3:
                                usimove += "+"
                            movelist.append(usimove)
                elif square == "g":
                    mlist = mv.gold(x, y, me=False)

                    mlist2 = filter(
                        lambda xy: not board[xy[1]][xy[0]].islower(), mlist)
                    for x2, y2 in mlist2:
                        if util.isThisMoveLegal(
                                copy.deepcopy(board),
                            (square.upper(), x, x2, y, y2)):
                            usimove = util.rawtousi(square.lower(), x, x2, y,
                                                    y2, sente)
                            movelist.append(usimove)
                elif square == "k":
                    mlist = mv.king(x, y)
                    mlist2 = filter(
                        lambda xy: not board[xy[1]][xy[0]].islower(), mlist)

                    for x2, y2 in mlist2:
                        if util.isThisMoveLegal(
                                copy.deepcopy(board),
                            (square.upper(), x, x2, y, y2)):
                            usimove = util.rawtousi(square.lower(), x, x2, y,
                                                    y2, sente)
                            movelist.append(usimove)
                elif square == "r":
                    mlist = mv.rook(x, y)
                    mlist2 = []
                    #print(mlist)
                    for mlpart in mlist[:4]:
                        for lx, ly in mlpart:
                            if board[ly][lx] == "":
                                mlist2.append((lx, ly))
                            elif board[ly][lx].islower():
                                break
                            else:
                                mlist2.append((lx, ly))
                                break
                    for x2, y2 in mlist2:
                        if util.isThisMoveLegal(
                                copy.deepcopy(board),
                            (square.upper(), x, x2, y, y2)):
                            usimove = util.rawtousi(square.lower(), x, x2, y,
                                                    y2, sente)
                            if y2 < 3:
                                usimove += "+"
                            movelist.append(usimove)
                elif square == "b":
                    mlist = mv.bishop(x, y)
                    mlist2 = []
                    #print(mlist)
                    for mlpart in mlist[:4]:
                        for lx, ly in mlpart:
                            if board[ly][lx] == "":
                                mlist2.append((lx, ly))
                            elif board[ly][lx].islower():
                                break
                            else:
                                mlist2.append((lx, ly))
                                break

                    for x2, y2 in mlist2:
                        if util.isThisMoveLegal(
                                copy.deepcopy(board),
                            (square.upper(), x, x2, y, y2)):
                            usimove = util.rawtousi(square.lower(), x, x2, y,
                                                    y2, sente)
                            if y2 < 3:
                                usimove += "+"
                            movelist.append(usimove)
                elif square == "+p" or square == "+l" or square == "+n" or square == "+s":
                    mlist = mv.gold(x, y, me=False)

                    mlist2 = filter(
                        lambda xy: not board[xy[1]][xy[0]].islower(), mlist)
                    for x2, y2 in mlist2:
                        if util.isThisMoveLegal(
                                copy.deepcopy(board),
                            (square.upper(), x, x2, y, y2)):
                            usimove = util.rawtousi(square.lower(), x, x2, y,
                                                    y2, sente)
                            movelist.append(usimove)
                elif square == "+r":
                    mlist = mv.rook(x, y, dragon=True)
                    mlist2 = []
                    for mlpart in mlist[:4]:
                        for lx, ly in mlpart:
                            if board[ly][lx] == "":
                                mlist2.append((lx, ly))
                            elif board[ly][lx].islower():
                                break
                            else:
                                mlist2.append((lx, ly))
                                break
                    for lx, ly in mlist[4]:
                        mlist2.append((lx, ly))
                    for x2, y2 in mlist2:
                        if util.isThisMoveLegal(
                                copy.deepcopy(board),
                            (square.upper(), x, x2, y, y2)):
                            usimove = util.rawtousi(square.lower(), x, x2, y,
                                                    y2, sente)
                            movelist.append(usimove)
                elif square == "+b":
                    mlist = mv.bishop(x, y, horse=True)
                    mlist2 = []
                    for mlpart in mlist[:4]:
                        for lx, ly in mlpart:
                            if board[ly][lx] == "":
                                mlist2.append((lx, ly))
                            elif board[ly][lx].islower():
                                break
                            else:
                                mlist2.append((lx, ly))
                                break
                    for lx, ly in mlist[4]:
                        mlist2.append((lx, ly))

                    for x2, y2 in mlist2:
                        if util.isThisMoveLegal(
                                copy.deepcopy(board),
                            (square.upper(), x, x2, y, y2)):
                            usimove = util.rawtousi(square.lower(), x, x2, y,
                                                    y2, sente)
                            movelist.append(usimove)
                x += 1
            y += 1
        sfeny = ["a", "b", "c", "d", "e", "f", "g", "h", "i"]
        for koma in okmd:
            for tempy in range(9):
                for tempx in range(9):
                    if board[tempy][tempx] == "":
                        if sente:
                            usimove = "{}*{}{}".format(koma.upper(), tempx + 1,
                                                       sfeny[tempy])
                        else:
                            usimove = "{}*{}{}".format(koma.upper(), 9 - tempx,
                                                       sfeny[8 - tempy])
                        if util.isThisDropLegal(copy.deepcopy(board),
                                                (koma.upper(), tempx, tempy)):
                            movelist.append(usimove)

    w = len(movelist)

    if w == 0:
        return evaluation.evaluation(board, kmd, okmd, mv, turn="opponent")

    if c_depth % 2 == 0:
        maxs = float("-inf")
        for i in range(w):
            usim = movelist[i]
            rawm = util.usitoraw(board, usim, sente)
            cboard = copy.deepcopy(board)
            legal = util.isThisMoveLegal(cboard, rawm, me=True)
            if not legal:
                continue
            cboard2 = copy.deepcopy(board)
            piece = rawm[0]
            x1 = rawm[1]
            x2 = rawm[2]
            y1 = rawm[3]
            y2 = rawm[4]
            if x1 and y1:
                cboard2[y1][x1] = ""
            cboard2[y2][x2] = piece
            score = minimax(cboard, kmd, okmd, mv, depth - 1, c_depth + 1,
                            sente)
            if score > maxs:
                maxs = score
        return maxs
    else:
        mins = float("inf")
        for i in range(w):
            usim = movelist[i]
            rawm = util.usitoraw(board, usim, sente)
            cboard = copy.deepcopy(board)
            legal = util.isThisMoveLegal(cboard, rawm, me=False)
            if not legal:
                continue
            cboard2 = copy.deepcopy(board)
            piece = rawm[0]
            x1 = rawm[1]
            x2 = rawm[2]
            y1 = rawm[3]
            y2 = rawm[4]
            if x1 and y1:
                cboard2[y1][x1] = ""
            cboard2[y2][x2] = piece
            score = minimax(cboard, kmd, okmd, mv, depth - 1, c_depth + 1,
                            sente)
            if score < mins:
                mins = score
        return mins