コード例 #1
0
def run(api_path, cloud_name, tags, output):
    if not os.path.isdir(output):
        os.makedirs(output)
    output = normal_dir(output)

    if api_path[-1] != "/":
        api_path += "/"

    product = read_yaml(api_path + "product.yaml")
    if not product:
        raise Exception("Read (%s) failed" % (api_path + "product.yaml"))

    api_yaml = read_yaml(api_path + "api.yaml")
    all_models = read_yaml(api_path + "models.yaml")

    all_tags = {i["name"]: i for i in product["tags"]}
    for tag in tags.split(","):
        tag = tag.strip().decode("utf8")
        if tag not in all_tags:
            raise Exception("Unknown tag(%s)" % tag)

        _, properties = generate_resource_properties(
            api_yaml, all_models, tag, read_yaml(api_path + tag + ".yaml"))

        write_file(output + tag + ".yaml", _generate_yaml(properties))
コード例 #2
0
def run(api_path, cloud_name, tags, output):
    if not os.path.isdir(output):
        os.makedirs(output)

    output = normal_dir(output)
    api_path = normal_dir(api_path)

    cloud = _get_cloud_info(cloud_name)

    product = read_yaml(api_path + "product.yaml")
    if not product:
        raise Exception("Read (%s) failed" % (api_path + "product.yaml"))

    product_info = {"service_type": product["service_type"]}
    product_info.update(cloud)

    all_tags = {i["name"]: i for i in product["tags"]}

    tag_info = {}
    for tag in tags.split(","):
        tag = tag.strip().decode("utf8")

        if tag not in all_tags:
            raise Exception("Unknown tag(%s)" % tag)

        tag_info[tag] = all_tags[tag]

    _generate_yaml(api_path, product_info, tag_info, output)
コード例 #3
0
ファイル: main.py プロジェクト: zengchen1024/developing_util
def _generate_platform_yaml(api_path, product_info, tag_info, output):
    prefix = "%s_%s" % (product_info["cloud_full_name"],
                        product_info["service_type"])

    config = {"ansible": {}, "terraform": {}}

    for tag, info in tag_info.items():
        custom_configs = read_yaml(api_path + tag + ".yaml")

        rn = get_resource_name(info, custom_configs)

        v = custom_configs.get("ansible")
        if v:
            config["ansible"][rn] = {
                "config": v,
            }

        v = custom_configs.get("terraform")
        if v:
            config["terraform"][rn] = {
                "config": v,
                "config_dir": api_path,
                "terraform_resource_name": "%s_%s" % (prefix, rn.lower())
            }

    m = {
        "ansible": build_ansible_yaml,
        "terraform": build_terraform_yaml
    }
    for k, v in config.items():
        m[k](v, output)
コード例 #4
0
def _get_cloud_info(cloud_name):
    cloud = None
    m = read_yaml("clouds.yaml")
    for i in m["clouds"]:
        if cloud_name == i["cloud_half_full_name"]:
            cloud = i
            break
    else:
        raise Exception("Unknown cloud(%s)" % cloud_name)

    return cloud
コード例 #5
0
ファイル: main.py プロジェクト: zengchen1024/developing_util
def _generate_api_yaml(api_path, product_info, tag_info, output):
    r = [_render_product(product_info)]

    api_yaml = read_yaml(api_path + "api.yaml")
    all_models = read_yaml(api_path + "models.yaml")

    for tag, v in tag_info.items():

        custom_configs = read_yaml(api_path + tag + ".yaml")

        api_info, properties = generate_resource_properties(
            api_yaml, all_models, tag, custom_configs
        )

        r.extend(
            build_resource_config(
                api_info, properties, v,
                custom_configs, product_info["service_type"])
        )

    write_file(output + "api.yaml", r)
コード例 #6
0
def run(config_file, cloud_name, tag, output):
    if not os.path.isdir(output):
        os.makedirs(output)
    output = normal_dir(output)
    api_path = os.path.dirname(config_file) + "/"

    product = read_yaml(api_path + "product.yaml")
    if not product:
        raise Exception("Read (%s) failed" % (api_path + "product.yaml"))

    api_yaml = read_yaml(api_path + "api.yaml")
    all_models = read_yaml(api_path + "models.yaml")

    all_tags = {i["name"]: i for i in product["tags"]}
    tag = tag.strip().decode("utf8")
    if tag not in all_tags:
        raise Exception("Unknown tag(%s)" % tag)

    _, properties = generate_resource_properties(api_yaml, all_models,
                                                 read_yaml(config_file))

    write_file(output + tag + "_desc.yaml", _generate_yaml(properties))
コード例 #7
0
def _generate_yaml(api_path, product_info, tag_info, output):
    r = [_render_product(product_info)]

    platform_config = []

    api_yaml = read_yaml(api_path + "api.yaml")
    all_models = read_yaml(api_path + "models.yaml")

    for tag, v in tag_info.items():

        custom_configs = read_yaml(api_path + tag + ".yaml")

        api_info, properties = generate_resource_properties(
            api_yaml, all_models, tag, custom_configs)

        argv = {
            "config_dir": api_path,
            "api_info": api_info,
            "all_models": all_models,
            "properties": properties,
            "service_type": product_info["service_type"],
            "resource_name": _get_resource_name(v, custom_configs),
            "version": _get_version(api_info),
            "resource_desc": v.get("description", ""),
            "custom_configs": custom_configs,
            "cloud_full_name": product_info["cloud_full_name"],
            "cloud_short_name": product_info["cloud_short_name"],
        }

        r.extend(build_resource_config(**argv))

        r.extend(build_resource_api_config(**argv))

        platform_config.append(argv)

    write_file(output + "api.yaml", r)

    _generate_platform_yaml(platform_config, all_models, output)
コード例 #8
0
def _build_example_render_info(f, module_name, cloud_short_name):
    tasks = None
    r = read_yaml(f)
    if len(r) == 1 and isinstance(r[0], dict) and "tasks" in r[0]:
        tasks = r[0].get("tasks")
    else:
        raise Exception("the format of example is not correct")

    if not tasks:
        raise Exception("no tasks in the example file")

    task = None
    for i in tasks:
        if module_name in i:
            task = i
            tasks.remove(i)
            break
    else:
        raise Exception("can't find the task(%s)" % module_name)

    v = {
        "example_description": r[0].get("name"),
        "task_name": module_name,
        "task_code": _build_module_params(task[module_name], 4),
        "task_description": task.get("name")
    }

    if tasks:
        d = []
        for t in tasks:
            module = ""
            for k in t:
                if k.startswith(cloud_short_name):
                    module = k
                    break
            else:
                continue

            d.append({
                "name": module,
                "register": t.get("register"),
                "description": t.get("name"),
                "code": _build_module_params(t[module], 6),
            })

        if d:
            v["depends"] = d
            v["has_depends"] = True

    return v
コード例 #9
0
ファイル: load.py プロジェクト: gueux/growbox420
def import_db_from_yaml():
    data_tree = []
    for index, tree in enumerate(os.walk(DATA_PATH)):
        if index > 0:
            data_tree.append(tree)

    for data in data_tree:

        database = data[0].split('/')[-1]

        connection = rdb_connector()
        try:
            RDB.db_create(database).run(connection)
            logger.info("Database `%s` setup completed", database)
        except RqlRuntimeError:
            logger.error("Database `%s` already exists", database)
        except RqlDriverError as e:
            logger.error(e)
            exit()
        finally:
            connection.close()

        tables = data[2]

        for table in tables:
            table_name = os.path.splitext(table)[0]
            table_data = read_yaml(DATA_PATH + '/' + database + '/' + table)
            connection = rdb_connector()
            try:
                RDB.db(database).table_create(table_name).run(connection)
                RDB.db(database).table(table_name).insert(table_data).run(
                    connection)
                logger.info("Table `%s` setup in database `%s` completed",
                            table_name, database)
            except RqlRuntimeError:
                logger.error("Table `%s` in database `%s` already exists",
                             table_name, database)
            except RqlDriverError as e:
                logger.error(e)
                exit()
            finally:
                connection.close()
コード例 #10
0
def test(model, conf_thres=0.001, nms_thres=0.5):
    device = "cuda:0"
    classes = ['person']
    if model is None:
        pass
    else:
        device = model.device  # get model device
        data_parameters = read_yaml(model.data_yaml)
        classes = data_parameters['classes']
        batch_size = model.batch_size
    # Dataset
    data_parameters[
        "data_path"] = '/home/lingc1/data/sports-training-data/player_detection/validate_dataset_5k_half_size'
    data_set = Yolov3Data(data_parameters, None, index_file='val_test')
    dataloader = DataLoader(data_set,
                            batch_size,
                            shuffle=False,
                            num_workers=0,
                            collate_fn=data_set.collate_fn)
    print(('%20s' + '%10s' * 6) %
          ('Class', 'Images', 'Targets', 'P', 'R', 'mAP', 'F1'))
    loss, p_80, r_80, f1_80, mp_80, mr_80, map_80, mf1_80 = 0., 0., 0., 0., 0., 0., 0., 0.
    seen = 0
    images_num = 0
    output_results = ""
    class_recs = {}
    nc = len(classes)
    names = classes
    det_lines = []
    imagenames = []
    npos_cls = {}
    for i in range(nc):
        npos_cls[i] = 0
        class_recs[i] = {}

    for i, (imgs, targets) in enumerate(dataloader):
        imgs = imgs.to(device)
        targets = targets.to(device)
        target_number = len(targets)

        preds = model.inference(imgs, None)

        output = non_max_suppression(preds,
                                     conf_thres=conf_thres,
                                     nms_thres=nms_thres)
        true_targets = targets[torch.sum(targets[:, 1:6], 1) != 0]

        # Statistics per image
        # remove the targets that fills 0 for data distribution.
        true_targets = targets[torch.sum(targets[:, 1:6], 1) != 0]
        # npos += len(true_targets)

        for si, pred in enumerate(output):
            images_num += 1
            if pred is not None and len(pred) > 0:
                # Rescale boxes from 416 to true image size
                # pred[:, :4] = scale_coords(imgs.shape[2:], pred[:, :4], im0_shape).round()
                for *xyxy, conf, cls_conf, cls in pred:
                    x = int(xyxy[0])
                    y = int(xyxy[1])
                    w = int((xyxy[2] - xyxy[0]).round())
                    h = int((xyxy[3] - xyxy[1]).round())
                    output_line = "{:s},{:d},{:d},{:d},{:d},{:f}\n".format(
                        Path(str(si)).name, x, y, w, h, conf)
                    det_lines.append(output_line)
                    output_results = output_results + output_line
            labels = true_targets[true_targets[:, 0] == si, 1:]
            nl = len(labels)
            tcls = labels[:, 4].tolist() if nl else []  # target class
            seen += 1

            imagename = os.path.splitext(Path(str(si)).name)[0]
            imagenames.append(imagename)
            if nl:

                unique_classes = np.unique(tcls).astype('int32')
                # detected = []
                for cls in unique_classes:
                    cls_idx = np.where(tcls == cls)[0]
                    tcls_tensor = labels[:, 4]
                    tcls_tensor = tcls_tensor[cls_idx]
                    npos_cls[cls] += len(tcls_tensor)
                    # target boxes
                    tbox = xywh2xyxy(labels[:, 0:4])
                    tbox = tbox[cls_idx]
                    tbox[:, [0, 2]] *= imgs.shape[3]
                    tbox[:, [1, 3]] *= imgs.shape[2]
                    bbox = np.array(tbox.cpu().numpy().round(), dtype=int)
                    det = [False] * len(tcls_tensor)
                    difficult = np.array(det)
                    class_recs[cls][imagename] = {
                        'bbox': bbox,
                        'difficult': difficult,
                        'det': det
                    }
    p_80, r_80, ap_80, f1_80 = [], [], [], []
    p_50, r_50, ap_50, f1_50 = [], [], [], []
    ap_80_iou = 0.8
    ap_50_iou = 0.5
    class_recs_80 = copy.deepcopy(class_recs)
    class_recs_50 = copy.deepcopy(class_recs)
    for cls in range(nc):
        rec_cls, prec_cls, ap_cls = voc_eval(det_lines,
                                             npos_cls[cls],
                                             imagenames,
                                             class_recs_80[cls],
                                             ovthresh=ap_80_iou,
                                             use_07_metric=True)
        f1_cls = 2 * prec_cls[-1] * rec_cls[-1] / (prec_cls[-1] + rec_cls[-1] +
                                                   1e-16)
        p_80.append(prec_cls[-1])
        r_80.append(rec_cls[-1])
        ap_80.append(ap_cls)
        f1_80.append(f1_cls)
        print("AP 80")
        print("person ap is: %.6f" % (ap_cls * 100))
        print("recall is %.6f" % (rec_cls[-1] * 100))
        print("precision is %.6f" % (prec_cls[-1] * 100))
        rec_cls, prec_cls, ap_cls = voc_eval(det_lines,
                                             npos_cls[cls],
                                             imagenames,
                                             class_recs_50[cls],
                                             ovthresh=ap_50_iou,
                                             use_07_metric=True)
        f1_cls = 2 * prec_cls[-1] * rec_cls[-1] / (prec_cls[-1] + rec_cls[-1] +
                                                   1e-16)
        p_50.append(prec_cls[-1])
        r_50.append(rec_cls[-1])
        ap_50.append(ap_cls)
        f1_50.append(f1_cls)
        print("AP 50")
        print("person ap is: %.6f" % (ap_cls * 100))
        print("recall is %.6f" % (rec_cls[-1] * 100))
        print("precision is %.6f" % (prec_cls[-1] * 100))
    mp_80, mr_80, map_80, mf1_80 = np.mean(p_80) * 100, np.mean(
        r_80) * 100, np.mean(ap_80) * 100, np.mean(f1_80) * 100
    mp_50, mr_50, map_50, mf1_50 = np.mean(p_50) * 100, np.mean(
        r_50) * 100, np.mean(ap_50) * 100, np.mean(f1_50) * 100
    # Print results
    all_target_sum = 0
    for _, cls_npos in npos_cls.items():
        all_target_sum += cls_npos
    pf = '%20s' + '%10.6g' * 6  # print format
    # print(pf % ('all', seen, nt.sum(), mp, mr, map, mf1), end='\n\n')
    print(pf % ('all', seen, all_target_sum, mp_80, mr_80, map_80, mf1_80),
          end='\n\n')
    print(pf % ('all', seen, all_target_sum, mp_50, mr_50, map_50, mf1_50),
          end='\n\n')

    # Print results per class
    # if nc > 1:
    #     for i, c in enumerate(ap_class):
    #         print(pf % (names[c], seen, npos_cls[c], p[i], r[i], ap[i], f1[i]))
    if nc > 1:
        for i in range(nc):
            print(pf % (names[i], seen, npos_cls[i], p_80[i], r_80[i],
                        ap_80[i], f1_80[i]))
            print(pf % (names[i], seen, npos_cls[i], p_50[i], r_50[i],
                        ap_50[i], f1_50[i]))
    # Return results
    maps = np.zeros(nc)
    # for i, c in enumerate(ap_class):
    #     maps[c] = ap[i]
    for i in range(nc):
        maps[i] = ap_80[i]
    return (mp_80, mr_80, map_80, mf1_80, loss / len(dataloader), mp_50, mr_50,
            map_50, mf1_50), maps