Пример #1
0
def print_triplet_info():
    m = h5py.File('/media/zawlin/ssd/Dropbox/proj/sg_vrd_meta.h5')
    train_triplets, train_triplets_sorted = get_triplet_stats('train')
    test_triplets, test_triplets_sorted = get_triplet_stats('test')
    triplets, triplets_sorted = test_triplets, test_triplets_sorted
    #triplets,triplets_sorted = train_triplets,train_triplets_sorted

    for i in xrange(len(triplets_sorted)):
        if triplets_sorted[i][0] in train_triplets: continue
        spo = triplets_sorted[i][0].split('_')
        s = zl.name2idx_cls(m, spo[0])
        p = zl.name2idx_pre(m, spo[1])
        o = zl.name2idx_cls(m, spo[2])
        spo_str = '%s_%s_%s' % (str(s), str(p), str(o))
        print '%s;%s;%s' % (triplets_sorted[i][0], spo_str,
                            triplets_sorted[i][1])
Пример #2
0
def save_zeroshots_to_meta():
    m = h5py.File('/media/zawlin/ssd/Dropbox/proj/sg_vrd_meta.h5')
    train_triplets, train_triplets_sorted = get_triplet_stats('train')
    test_triplets, test_triplets_sorted = get_triplet_stats('test')
    triplets, triplets_sorted = test_triplets, test_triplets_sorted
    #triplets,triplets_sorted = train_triplets,train_triplets_sorted
    zeroshots = []
    for i in xrange(len(triplets_sorted)):
        if triplets_sorted[i][0] in train_triplets: continue

        spo = triplets_sorted[i][0].split('_')
        s = zl.name2idx_cls(m, spo[0])
        p = zl.name2idx_pre(m, spo[1])
        o = zl.name2idx_cls(m, spo[2])
        zeroshots.append([s, p, o])
        spo_str = '%s_%s_%s' % (str(s), str(p), str(o))
        print '%s;%s;%s' % (triplets_sorted[i][0], spo_str,
                            triplets_sorted[i][1])
        # if triplets_sorted[i][0] not in train_triplets:
        # print triplets_sorted[i][0],triplets_idx_sorted[i][0],triplets_sorted[i][1]
        # if triplets_sorted[i][0] not in triplets:
        # print test_triplets_sorted[i][0],test_triplets_sorted[i][1]
    m.create_dataset('meta/zeroshots',
                     data=np.array(zeroshots).astype(np.int16))
Пример #3
0
def relation_make_meta():
    spo_info = zl.load('output/spo_info_vg.pkl')
    spo_list = []
    spo_dict = {}
    for k in spo_info.keys():
        if spo_info[k] >= 5:
            spo_list.append(k)
    for spo in spo_list:
        spo_dict[spo] = 0
    spo_list = spo_dict
    blacklist = [
        'VG_100K/2363098',
        'VG_100K_2/2402233',
        'VG_100K/2365839',
        'VG_100K_2/2398948',
        'VG_100K/2315697',
        'VG_100K_2/2403354',
    ]

    client = MongoClient("mongodb://localhost:27017")
    db = client.visual_genome_1_2
    imdatas = {}
    for imdata in db.image_data.find(no_cursor_timeout=True):
        imid = imdata['image_id']
        imdatas[imid] = imdata
    db_results_train = db.relationships_all_train.find(no_cursor_timeout=True)
    m = h5py.File('data/vg1_2_meta.h5')
    cnt = 0
    for doc in db_results_train:
        if cnt % 1000 == 0:
            print cnt
        cnt += 1
        imid = doc['image_id']
        im_data = imdatas[imid]
        if im_data['width'] < 100 or im_data['height'] < 100: continue
        im_path_full = im_data['url'].replace(
            'https://cs.stanford.edu/people/rak248/', '')
        im_path_folder = im_path_full.split('/')[0]
        im_path_file = im_path_full.split('/')[1]
        im_index = im_path_folder + '/' + im_path_file.replace('.jpg', '')
        if im_index in blacklist: continue
        obj_boxes = []
        sub_boxes = []
        rlp_labels = []
        imid = doc['image_id']
        imdata = imdatas[imid]
        for r in doc['relationships']:
            pre = r['predicate']
            sub_name = r['subject']['name']
            obj_name = r['object']['name']
            spo = sub_name + '_' + pre + '_' + obj_name
            if spo in spo_list:
                sidx = zl.name2idx_cls(m, sub_name)
                oidx = zl.name2idx_cls(m, obj_name)
                pidx = zl.name2idx_pre(m, pre)
                ox1, oy1, ow, oh = r['object']['x'], r['object']['y'], r[
                    'object']['w'], r['object']['h']
                sx1, sy1, sw, sh = r['subject']['x'], r['subject']['y'], r[
                    'subject']['w'], r['subject']['h']
                ox2, oy2 = ox1 + ow, oy1 + oh
                sx2, sy2 = sx1 + sw, sy1 + sh
                rlp_labels.append([sidx, pidx, oidx])
                sub_boxes.append([sx1, sy1, sx2, sy2])
                obj_boxes.append([ox1, oy1, ox2, oy2])
        m.create_dataset('gt/train/%s/rlp_labels' % imid,
                         data=np.array(rlp_labels).astype(np.int16))
        m.create_dataset('gt/train/%s/sub_boxes' % imid,
                         data=np.array(sub_boxes).astype(np.int16))
        m.create_dataset('gt/train/%s/obj_boxes' % imid,
                         data=np.array(obj_boxes).astype(np.int16))

    db_results_test = db.relationships_all_test.find(no_cursor_timeout=True)
    for doc in db_results_test:
        if cnt % 1000 == 0:
            print cnt
        cnt += 1
        imid = doc['image_id']
        im_data = imdatas[imid]
        if im_data['width'] < 100 or im_data['height'] < 100: continue
        im_path_full = im_data['url'].replace(
            'https://cs.stanford.edu/people/rak248/', '')
        im_path_folder = im_path_full.split('/')[0]
        im_path_file = im_path_full.split('/')[1]
        im_index = im_path_folder + '/' + im_path_file.replace('.jpg', '')
        if im_index in blacklist: continue
        obj_boxes = []
        sub_boxes = []
        rlp_labels = []
        imid = doc['image_id']
        imdata = imdatas[imid]
        for r in doc['relationships']:
            pre = r['predicate']
            sub_name = r['subject']['name']
            obj_name = r['object']['name']
            spo = sub_name + '_' + pre + '_' + obj_name
            if spo in spo_list:
                sidx = zl.name2idx_cls(m, sub_name)
                oidx = zl.name2idx_cls(m, obj_name)
                pidx = zl.name2idx_pre(m, pre)
                ox1, oy1, ow, oh = r['object']['x'], r['object']['y'], r[
                    'object']['w'], r['object']['h']
                sx1, sy1, sw, sh = r['subject']['x'], r['subject']['y'], r[
                    'subject']['w'], r['subject']['h']
                ox2, oy2 = ox1 + ow, oy1 + oh
                sx2, sy2 = sx1 + sw, sy1 + sh
                rlp_labels.append([sidx, pidx, oidx])
                sub_boxes.append([sx1, sy1, sx2, sy2])
                obj_boxes.append([ox1, oy1, ox2, oy2])
        m.create_dataset('gt/test/%s/rlp_labels' % imid,
                         data=np.array(rlp_labels).astype(np.int16))
        m.create_dataset('gt/test/%s/sub_boxes' % imid,
                         data=np.array(sub_boxes).astype(np.int16))
        m.create_dataset('gt/test/%s/obj_boxes' % imid,
                         data=np.array(obj_boxes).astype(np.int16))
Пример #4
0
def make_matlab_from_vp_nms2():
    import scipy.io as sio
    m = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_meta.h5')
    m_vp = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_vp_meta.h5')
    h5path = 'output/precalc/sg_vrd_2016_test_nms.7.hdf5'
    h5f_nms = h5py.File(h5path)

    imids = sorted(m['gt/test'].keys())
    boxes_ours= np.empty((len(imids)),dtype=np.object)
    rlp_labels_ours = np.empty((len(imids)),dtype=np.object)
    rlp_confs_ours = np.empty((len(imids)),dtype=np.object)
    for i in xrange(boxes_ours.shape[0]):
        boxes_ours[i]=[]
        rlp_labels_ours[i]=[]
        rlp_confs_ours[i]=[]
    cnt = 1
    idx = 0
    for imid in imids:
        if cnt %100==0:
            print cnt
        cnt+=1
        if imid not in h5f_nms:
            rlp_labels_ours[idx]=[]
            rlp_confs_ours[idx]=[]
            boxes_ours[idx]=[]
            idx += 1
            continue
        boxes = h5f_nms[imid]['boxes'][...]
        confs = h5f_nms[imid]['confs'][...]
        labels = h5f_nms[imid]['labels'][...]
        ind = np.argsort(confs)[::-1]

        if ind.shape[0]>100:
            ind = ind[:100]

        boxes_raw = boxes[ind]
        confs = confs[ind]
        labels = labels[ind]
        boxes = []
        rlp_confs = []
        rlp_labels = []
        for i in xrange(confs.shape[0]):
            lbl = zl.idx2name_tri(m_vp,labels[i])
            sub_lbl = lbl.split('_')[0]
            pre_lbl = lbl.split('_')[1]
            obj_lbl = lbl.split('_')[2]
            rlp_label = [zl.name2idx_cls(m,sub_lbl),zl.name2idx_pre(m,pre_lbl),zl.name2idx_cls(m,obj_lbl)]
            rlp_labels.append(np.array(rlp_label).astype(np.float64))
            rlp_confs.append(np.array(confs[i]).astype(np.float64))
            boxes.append(np.array(boxes_raw[i]))
        # boxes = np.array(boxes).reshape((-1,4))
        # rlp_labels = np.array(rlp_labels).reshape((-1,3))
        # rlp_confs= np.array(rlp_confs)[:,np.newaxis]
        rlp_confs_ours[idx] = rlp_confs
        rlp_labels_ours[idx] = rlp_labels
        boxes_ours[idx] = boxes
        idx += 1
    # boxes_ours.append([])
    # boxes_ours=boxes_ours[:-1]
    sio.savemat('output/sg_vrd_vp_results_.7.mat', {'bboxes_ours': boxes_ours,
        'rlp_labels_ours':rlp_labels_ours,'rlp_confs_ours':rlp_confs_ours})#'relation_vectors':relation_vectors})
Пример #5
0
def run_retrieval_vp_v2():
    # h5_path = 'output/sg_vrd_2016_result.classeme.hdf5'
    h5_path = 'output/precalc/vg1_2_vp2016_test_nms2_.4.hdf5'
    m = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_meta.h5', 'r', 'core')
    m_vp = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_vp_meta.h5', 'r', 'core')
    cache_path = 'output/cache/%s.pkl' % h5_path.split('/')[-1]
    data_root = '/home/zawlin/data/data_vrd/vg_1.2/'
    rlp_map = []
    for i in xrange(1, len(m_vp['meta/tri/idx2name'].keys())):
        tri = str(m_vp['meta/tri/idx2name'][str(i)][...])
        s_lbl = tri.split('_')[0]
        p_lbl = tri.split('_')[1]
        o_lbl = tri.split('_')[2]
        rlp_label = [
            zl.name2idx_cls(m, s_lbl),
            zl.name2idx_pre(m, p_lbl),
            zl.name2idx_cls(m, o_lbl)
        ]
        rlp_map.append(rlp_label)
    rlp_map = np.array(rlp_map)
    if os.path.exists(cache_path):
        print 'load from cache'
        h5f = zl.load(cache_path)
    else:
        h5_in = h5py.File(h5_path, 'r')
        h5f = {}
        print 'preloading data'
        for i in h5_in:
            h5f[i] = {}
            h5f[i]['labels'] = h5_in[i]['labels'][...]
            h5f[i]['confs'] = h5_in[i]['confs'][...]
            h5f[i]['boxes'] = h5_in[i]['boxes'][...]
        zl.save(cache_path, h5f)
        print 'preloading data done'
    retr_meta = zl.load('output/pkl/vg_retr_meta.pkl')
    thresh = 0.0
    images = {}
    imids = h5f.keys()
    results = {}
    cnt = 0
    r_acc_100 = 0
    r_acc_50 = 0

    tp_total = 0
    gt_total = 0
    median = []
    total = 0
    retr_meta = zl.load('output/pkl/vg_retr_meta.pkl')
    for k in xrange(len(retr_meta['rlp_labels'])):
        total += retr_meta['counts'][k]
        if k > 1000:
            print total / 1000.
            exit(0)
            break
        continue
        cnt += 1
        rlp_label = retr_meta['rlp_labels'][k]
        gt_files = retr_meta['files'][k]
        # print gt_files
        # exit(0)
        # for f in gt_files:
        # impath= zl.imid2path(m,f)
        # print impath
        # im= cv2.imread(data_root+impath)
        # cv2.imshow('im',im)
        # cv2.waitKey(0)
        results = {}
        zl.tick()
        ranks = []
        for imid in imids:
            labels = h5f[imid]['labels'] - 1
            rlp_confs = h5f[imid]['confs']
            rlp_labels = rlp_map[labels]
            if rlp_labels.shape[0] == 0:
                results[imid] = 0.0
                continue
            indexor = np.arange(rlp_labels.shape[0])
            ind = indexor[np.all(rlp_labels == rlp_label, axis=1)]
            if ind.shape[0] == 0:
                results[imid] = 0.0
                continue
            confs = rlp_confs[ind]
            results[imid] = np.average(confs)

        results_sorted = zl.sort_dict_by_val(results)
        total_gt = len(gt_files) + 0.0
        gt_total += total_gt + 0.0
        tp_50 = 0.
        tp_100 = 0.
        found = False
        s_lbl = zl.idx2name_cls(m, rlp_label[0])
        p_lbl = zl.idx2name_pre(m, rlp_label[1])
        o_lbl = zl.idx2name_cls(m, rlp_label[2])
        lbl_str = '%s_%s_%s' % (s_lbl, p_lbl, o_lbl)

        delay = 0
        for i in xrange(len(results_sorted)):
            imid, v = results_sorted[i]
            impath = zl.imid2path(m, imid)
            if found and i >= 5:
                break
            # print gt_files
            cor = imid in gt_files
            if cor:
                if not found:
                    found = True
                    median.append(i)
                if i < 5:
                    tp_100 += 1
                    tp_total += 1
                    if i < 50: tp_50 += 1
            # if True:
            # cor_or_not = str(cor)
            # if cor :delay=0
            # if delay ==0:
            # im = cv2.imread(data_root+impath)
            # cv2.putText(im, cor_or_not, (50, 50), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
            # cv2.putText(im, lbl_str, (50, 80), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
            # cv2.putText(im, str(i), (50, 100), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
            # cv2.imshow('im',im)
            # c = cv2.waitKey(delay)&0xFF
            # if c ==27:
            # exit(0)
            # if c == ord('s'):
            # delay = 1-delay

            # if c == ord('c'):
            # delay = 1

        r_50 = tp_50 / 5  #total_gt
        r_100 = tp_100 / 5  #total_gt
        r_acc_50 += r_50
        r_acc_100 += r_100
        med = np.median(median)
        print '%d %f %f %f %f %d %f' % (cnt, r_50, r_100, r_acc_50 / cnt,
                                        r_acc_100 / cnt, med, zl.tock())
Пример #6
0
def run_retrieval():
    h5f = h5py.File('output/vg_results/vg1_2_2016_result_all_100000.all.hdf5')
    #h5f = h5py.file('output/results/lu_method_results.hdf5')
    data_root = '/home/zawlin/g/py-faster-rcnn/data/vg1_2_2016/Data/test/'
    m = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_meta.h5', 'r', 'core')
    thresh = 0.0
    filters = []
    rois = []
    images = {}
    imids = h5f.keys()
    results = {}
    cnt = 0
    # pre = 'teddy bear_sit on_blanket'
    pre = 'cloth_next to_suitcase'
    # pre = 'paw_in front of_cat'
    # pre = 'racket_hold by_player'
    sub_lbl = pre.split('_')[0]
    pre_lbl = pre.split('_')[1]
    obj_lbl = pre.split('_')[2]
    sub_idx = zl.name2idx_cls(m, sub_lbl)
    pre_idx = zl.name2idx_pre(m, pre_lbl)
    obj_idx = zl.name2idx_cls(m, obj_lbl)
    rlp_label = np.array([sub_idx, pre_idx, obj_idx]).astype(np.int16)
    results = {}

    for imid in imids:
        if cnt % 100 == 0:
            print cnt, zl.tock()
            zl.tick()
        cnt += 1
        rlp_labels = h5f[imid + '/rlp_labels'][...]
        rlp_confs = h5f[imid + '/rlp_confs'][...]
        if rlp_confs.shape[0] == 0:
            results[imid] = 0.0
            continue
        zl.tick()
        indexor = np.arange(rlp_labels.shape[0])
        ind = indexor[np.all(rlp_labels == rlp_label, axis=1)]
        if ind.shape[0] == 0:
            results[imid] = 0.0
            continue
        confs = rlp_confs[ind]
        # print confs
        # print np.average(confs)
        results[imid] = np.average(confs)
    results_sorted = zl.sort_dict_by_val(results)
    example_folder = 'output/examples_retr_vg/%s/' % pre
    zl.make_dirs(example_folder)
    cnt = 0
    for imid, v in results_sorted[:20]:
        sub_boxes = h5f[imid + '/sub_boxes'][...]
        obj_boxes = h5f[imid + '/obj_boxes'][...]
        rlp_labels = h5f[imid + '/rlp_labels'][...]
        rlp_confs = h5f[imid + '/rlp_confs'][...]

        indexor = np.arange(rlp_labels.shape[0])
        ind = indexor[np.all(rlp_labels == rlp_label, axis=1)]
        if ind.shape[0] != 0:
            sub_boxes = sub_boxes[ind]
            obj_boxes = obj_boxes[ind]
            sb = sub_boxes[0]
            ob = obj_boxes[0]
            im = cv2.imread(data_root + zl.imid2path(m, imid))
            cv2.rectangle(im, (sb[0], sb[1]), (sb[2], sb[3]), (0, 200, 0), 1)
            cv2.rectangle(im, (ob[0], ob[1]), (ob[2], ob[3]), (0, 0, 200), 1)
            cv2.imwrite(example_folder + str(cnt) + '_%s.jpg' % imid, im)
            #cv2.imshow('im',im)
            cnt += 1