def test_findImage(self): db = vdms.vdms() db.connect(hostname, port) prefix_name = "fimg_brain_" for i in range(0, 2): props = {} props["name"] = prefix_name + str(i) self.insertImage(db, props=props) all_queries = [] for i in range(0, 2): constraints = {} constraints["name"] = ["==", prefix_name + str(i)] img_params = {} img_params["constraints"] = constraints query = {} query["FindImage"] = img_params all_queries.append(query) response, img_array = db.query(all_queries) self.assertEqual(response[0]["FindImage"]["status"], 0) self.assertEqual(response[1]["FindImage"]["status"], 0) self.assertEqual(len(img_array), 2)
def test_addEntityWithBlobNoBlob(self, thID=0): db = vdms.vdms() db.connect(hostname, port) props = {} props["name"] = "Luis" props["lastname"] = "Ferro" props["age"] = 27 props["threadid"] = thID addEntity = {} addEntity["properties"] = props addEntity["class"] = "AwesomePeople" addEntity["blob"] = True query = {} query["AddEntity"] = addEntity all_queries = [] all_queries.append(query) response, res_arr = db.query(all_queries) self.assertEqual(response[0]["status"], -1) self.assertEqual(response[0]["info"], "Expected blobs: 1. Received blobs: 0")
def find_frame(x, y, vname, isFull): db = vdms.vdms() db.connect("localhost") all_queries = [] findFrames = {} xToy = range(x, y + 1) xToylst = list(xToy) findFrames["frames"] = xToylst constrs = {} constrs["name"] = ["==", vname] if isFull: constrs["isFull"] = ["==", isFull] findFrames["constraints"] = constrs query = {} query["FindFrames"] = findFrames all_queries.append(query) #print("Issuing Query to find frames Between: " + str(x) + "," + str(y)) #print(all_queries) response, res_arr = db.query(all_queries) db.disconnect() return res_arr
def main(): db = vdms.vdms() db.connect("localhost") insert_descriptor_set(db) prefix = "/mnt/nvme1/features_1M/YFCC100M_hybridCNN_gmean_fc6_first_1M_" d_reader = d_io.descriptors_reader(prefix) total = int(1e6) batch_size = 1000 inserted = 0 total_errors = 0 while inserted < total: ids, desc = d_reader.get_next_n(batch_size) total_errors += insert_descriptors(batch_size, ids, desc, db) inserted += batch_size print("Elements Inserted: ", inserted, " - ", 100 * inserted / total, "%") print("total_errors:", total_errors, "Percentage:", 100 * total_errors / inserted, "%")
def test_findVideo(self): db = vdms.vdms() db.connect(hostname, port) prefix_name = "video_1_" number_of_inserts = 2 for i in range(0, number_of_inserts): props = {} props["name"] = prefix_name + str(i) self.insertVideo(db, props=props) all_queries = [] for i in range(0, number_of_inserts): constraints = {} constraints["name"] = ["==", prefix_name + str(i)] video_parms = {} video_parms["constraints"] = constraints query = {} query["FindVideo"] = video_parms all_queries.append(query) response, vid_array = db.query(all_queries) self.assertEqual(len(response), number_of_inserts) self.assertEqual(len(vid_array), number_of_inserts) for i in range(0, number_of_inserts): self.assertEqual(response[i]["FindVideo"]["status"], 0)
def test_findBoundingBoxByCoordinates(self): db = vdms.vdms() db.connect(hostname, port) all_queries = [] rect_coords = {} rect_coords["x"] = 0 rect_coords["y"] = 0 rect_coords["w"] = 500 rect_coords["h"] = 500 results = {} results["list"] = ["name"] bb_params = {} bb_params["rectangle"] = rect_coords bb_params["results"] = results query = {} query["FindBoundingBox"] = bb_params all_queries.append(query) response, img_array = db.query(all_queries) self.assertEqual(response[0]["FindBoundingBox"]["status"], 0)
def __init__(self, *args, **kwargs): super(TestCommand, self).__init__(*args, **kwargs) # VDMS Server Info self.hostname = "localhost" self.port = 55557 db_up = False attempts = 0 while (not db_up): try: db = vdms.vdms() db.connect(self.hostname, self.port) db.disconnect() db_up = True if (attempts > 0): print("Connection to VDMS successful.") except: print("Attempt", attempts, "to connect to VDMS failed, retying...") attempts += 1 time.sleep(1) # sleeps 1 second if attempts > 10: print("Failed to connect to VDMS after 10 attempts") exit()
def findEntity(self, thID, results): db = vdms.vdms() db.connect(hostname, port) constraints = {} constraints["threadid"] = ["==", thID] findEntity = {} findEntity["constraints"] = constraints findEntity["class"] = "AwesomePeople" results = {} results["list"] = ["name", "lastname", "threadid"] findEntity["results"] = results query = {} query["FindEntity"] = findEntity all_queries = [] all_queries.append(query) response, res_arr = db.query(all_queries) try: self.assertEqual(response[0]["FindEntity"]["status"], 0) self.assertEqual( response[0]["FindEntity"]["entities"][0]["lastname"], "Ferro") self.assertEqual( response[0]["FindEntity"]["entities"][0]["threadid"], thID) except: results[thID] = -1 results[thID] = 0
def test_findBoundingBox(self): db = vdms.vdms() db.connect(hostname, port) prefix_name = "find_my_bb_" for i in range(0, self.number_of_inserts): props = {} props["name"] = prefix_name + str(i) self.insertBoundingBox(db, props=props) all_queries = [] for i in range(0, self.number_of_inserts): constraints = {} constraints["name"] = ["==", prefix_name + str(i)] results = {} results["list"] = ["name"] bb_params = {} bb_params["constraints"] = constraints bb_params["results"] = results query = {} query["FindBoundingBox"] = bb_params all_queries.append(query) response, img_array = db.query(all_queries) self.assertEqual(response[0]["FindBoundingBox"]["status"], 0) self.assertEqual(response[1]["FindBoundingBox"]["status"], 0) self.assertEqual(response[0]["FindBoundingBox"]["entities"][0]["name"], prefix_name + "0") self.assertEqual(response[1]["FindBoundingBox"]["entities"][0]["name"], prefix_name + "1")
def addEntity(self, thID, results): db = vdms.vdms() db.connect(hostname, port) props = {} props["name"] = "Luis" props["lastname"] = "Ferro" props["age"] = 27 props["threadid"] = thID addEntity = {} addEntity["properties"] = props addEntity["class"] = "AwesomePeople" query = {} query["AddEntity"] = addEntity all_queries = [] all_queries.append(query) response, res_arr = db.query(all_queries) # print (db.get_last_response_str()) try: self.assertEqual(response[0]["AddEntity"]["status"], 0) except: results[thID] = -1 results[thID] = 0
def find_clip(vname, \ condition, \ size, \ headers, \ clip_no, \ isFull): db = vdms.vdms() db.connect("localhost") all_queries = [] findVideo = {} constrs = {} constrs["name"] = ["==", vname] if isFull: constrs["isFull"] = ["==", isFull] else: constrs["clipNo"] = ["==", clip_no] #add more filters based on the conditions findVideo["constraints"] = constrs findVideo["container"] = "mp4" findVideo["codec"] = "h264" query = {} query["FindVideo"] = findVideo all_queries.append(query) response, vid_arr = db.query(all_queries) #print(response) db.disconnect() return vid_arr
def test_concurrent(self): self.build_store() self.add_descriptor_set(name, dim) retries = 2 concurrency = 64 db_list = [] for i in range(0, concurrency): db = vdms.vdms() db.connect(hostname, port) db_list.append(db) results = [None] * concurrency * retries for ret in range(0, retries): thread_arr = [] for i in range(0, concurrency): idx = concurrency * ret + i thread_add = Thread(target=self.single, args=(idx, db_list[i], results)) thread_add.start() thread_arr.append(thread_add) idx = concurrency * ret error_counter = 0 for th in thread_arr: th.join() if (results[idx] == -1): error_counter += 1 idx += 1 self.assertEqual(error_counter, 0)
def test_addEntityWithBlob(self, thID=0): db = vdms.vdms() db.connect(hostname, port) props = {} props["name"] = "Luis" props["lastname"] = "Ferro" props["age"] = 27 props["threadid"] = thID addEntity = {} addEntity["properties"] = props addEntity["class"] = "AwesomePeople" addEntity["blob"] = True query = {} query["AddEntity"] = addEntity all_queries = [] all_queries.append(query) blob_arr = [] fd = open("../test_images/brain.png", 'rb') blob_arr.append(fd.read()) fd.close() response, res_arr = db.query(all_queries, [blob_arr]) self.assertEqual(response[0]["AddEntity"]["status"], 0)
def get_db_list(num): dbs = [] for i in range(0, num): db = vdms.vdms() db.connect("sky4.local") dbs.append(db) return dbs
def test_addBoundingBox(self): db = vdms.vdms() db.connect(hostname, port) all_queries = [] for i in range(0, self.number_of_inserts): bb_coords = {} bb_coords["x"] = i bb_coords["y"] = i bb_coords["h"] = 512 bb_coords["w"] = 512 props = {} props["name"] = "my_bb_" + str(i) bb = {} bb["properties"] = props bb["rectangle"] = bb_coords query = {} query["AddBoundingBox"] = bb all_queries.append(query) response, img_array = db.query(all_queries) self.assertEqual(len(response), self.number_of_inserts) for i in range(0, self.number_of_inserts): self.assertEqual(response[i]["AddBoundingBox"]["status"], 0)
def test_updateVideo(self): db = vdms.vdms() db.connect(hostname, port) prefix_name = "fvid_update_" for i in range(0, 2): props = {} props["name"] = prefix_name + str(i) self.insertVideo(db, props=props) all_queries = [] constraints = {} constraints["name"] = ["==", prefix_name + str(0)] props = {} props["name"] = "simg_update_0" img_params = {} img_params["constraints"] = constraints img_params["properties"] = props query = {} query["UpdateVideo"] = img_params all_queries.append(query) response, img_array = db.query(all_queries) self.assertEqual(response[0]["UpdateVideo"]["count"], 1) self.assertEqual(len(img_array), 0)
def test_findVideoNoBlob(self): db = vdms.vdms() db.connect(hostname, port) prefix_name = "fvid_no_blob_" for i in range(0, 2): props = {} props["name"] = prefix_name + str(i) self.insertVideo(db, props=props) all_queries = [] for i in range(0, 2): constraints = {} constraints["name"] = ["==", prefix_name + str(i)] results = {} results["blob"] = False results["list"] = ["name"] img_params = {} img_params["constraints"] = constraints img_params["results"] = results query = {} query["FindVideo"] = img_params all_queries.append(query) response, img_array = db.query(all_queries) self.assertEqual(response[0]["FindVideo"]["status"], 0) self.assertEqual(response[1]["FindVideo"]["status"], 0) self.assertEqual(len(img_array), 0)
def test_findVid_multiple_results(self): db = vdms.vdms() db.connect(hostname, port) prefix_name = "vid_multiple" number_of_inserts = 4 for i in range(0, number_of_inserts): props = {} props["name"] = prefix_name self.insertVideo(db, props=props) constraints = {} constraints["name"] = ["==", prefix_name] results = {} results["list"] = ["name"] img_params = {} img_params["constraints"] = constraints query = {} query["FindVideo"] = img_params all_queries = [] all_queries.append(query) response, vid_arr = db.query(all_queries) self.assertEqual(len(vid_arr), number_of_inserts) self.assertEqual(response[0]["FindVideo"]["status"], 0) self.assertEqual(response[0]["FindVideo"]["returned"], number_of_inserts)
def test_updateBoundingBoxCoords(self): db = vdms.vdms() db.connect(hostname, port) prefix_name = "update_bb_" for i in range(0, self.number_of_inserts): props = {} props["name"] = prefix_name + str(i) self.insertBoundingBox(db, props=props) all_queries = [] constraints = {} constraints["name"] = ["==", prefix_name + str(0)] rect_coords = {} rect_coords["x"] = 15 rect_coords["y"] = 15 rect_coords["w"] = 75 rect_coords["h"] = 75 bb_params = {} bb_params["constraints"] = constraints bb_params["rectangle"] = rect_coords query = {} query["UpdateBoundingBox"] = bb_params all_queries.append(query) response, img_array = db.query(all_queries) self.assertEqual(response[0]["UpdateBoundingBox"]["status"], 0) self.assertEqual(response[0]["UpdateBoundingBox"]["count"], 1) all_queries = [] constraints = {} constraints["name"] = ["==", prefix_name + str(0)] results = {} results["list"] = ["_coordinates"] bb_params = {} bb_params["constraints"] = constraints bb_params["results"] = results query = {} query["FindBoundingBox"] = bb_params all_queries.append(query) response, img_array = db.query(all_queries) self.assertEqual(response[0]["FindBoundingBox"]["status"], 0) self.assertEqual(response[0]["FindBoundingBox"]["entities"][0]["_coordinates"]["x"], 15) self.assertEqual(response[0]["FindBoundingBox"]["entities"][0]["_coordinates"]["y"], 15) self.assertEqual(response[0]["FindBoundingBox"]["entities"][0]["_coordinates"]["w"], 75) self.assertEqual(response[0]["FindBoundingBox"]["entities"][0]["_coordinates"]["h"], 75)
def test_addEntityWithBlobAndFind(self, thID=0): db = vdms.vdms() db.connect(hostname, port) props = {} props["name"] = "Tom" props["lastname"] = "Slash" props["age"] = 27 props["id"] = 45334 addEntity = {} addEntity["properties"] = props addEntity["class"] = "NotSoAwesome" addEntity["blob"] = True query = {} query["AddEntity"] = addEntity all_queries = [] all_queries.append(query) blob_arr = [] fd = open("../test_images/brain.png", 'rb') blob_arr.append(fd.read()) fd.close() response, res_arr = db.query(all_queries, [blob_arr]) self.assertEqual(response[0]["AddEntity"]["status"], 0) constraints = {} constraints["id"] = ["==", 45334] results = {} results["blob"] = True results["list"] = ["name"] FindEntity = {} FindEntity["constraints"] = constraints FindEntity["class"] = "NotSoAwesome" FindEntity["results"] = results query = {} query["FindEntity"] = FindEntity all_queries = [] all_queries.append(query) response, res_arr = db.query(all_queries) self.assertEqual(response[0]["FindEntity"]["entities"][0]["blob"], True) self.assertEqual(len(res_arr), len(blob_arr)) self.assertEqual(len(res_arr[0]), len(blob_arr[0])) self.assertEqual((res_arr[0]), (blob_arr[0]))
def test_findBoundingBoxBlobComplex(self): db = vdms.vdms() db.connect(hostname, port) prefix_name = "my_brain_complex_" all_queries = [] for i in range(0, self.number_of_inserts): db = vdms.vdms() db.connect(hostname, port) img_name = prefix_name + str(i) imgprops = {} imgprops["name"] = img_name self.addBoundingBoxwithImage(db, 1, imgprops) rect_coords = {} rect_coords["x"] = 0 rect_coords["y"] = 0 rect_coords["w"] = 500 rect_coords["h"] = 500 results = {} results["blob"] = True results["list"] = ["name"] bb_params = {} bb_params["rectangle"] = rect_coords bb_params["results"] = results query = {} query["FindBoundingBox"] = bb_params all_queries.append(query) response, img_array = db.query(all_queries) self.assertEqual(response[0]["FindBoundingBox"]["status"], 0) self.assertTrue(len(img_array) >= self.number_of_inserts) for i in range(0, self.number_of_inserts): test = {} test["name"] = prefix_name + str(i) + "_bb_0" self.assertIn(test, response[0]["FindBoundingBox"]["entities"])
def __init__(self, app, request, **kwargs): super(SearchHandler, self).__init__(app, request, **kwargs) self.executor = ThreadPoolExecutor(8) self._vdms = vdms.vdms() while True: try: self._vdms.connect(dbhost) break except Exception as e: print("Exception: " + str(e), flush=True) time.sleep(10)
def add_video(fname, \ vname, \ vstream, \ encoding, \ header): tags = [] totalFrames = 0 height = -1 width = -1 start = True for i, frame in enumerate(vstream): header.update(frame) if start == True: height = vstream.height width = vstream.width start = False tags.append(frame['tags']) totalFrames = i db = vdms.vdms() db.connect('localhost') fd = open(fname, 'rb') blob = fd.read() all_queries = [] addVideo = {} addVideo["container"] = "mp4" """ if encoding == H264: addVideo["codec"] = "h264" else: addVideo["codec"] = "xvid" Actually, I think you can only use h264 on VDMS """ addVideo["codec"] = "h264" header_dat = header.getHeader() props = {} props[0] = header_dat props[0]["isFull"] = True props[0]["name"] = vname props[0]["width"] = width props[0]["height"] = height #addVideo["properties"] = props vprops = {} vprops["name"] = vname vprops["isFull"] = True addVideo["properties"] = vprops query = {} query["AddVideo"] = addVideo all_queries.append(query) response, res_arr = db.query(all_queries, [[blob]]) #print(response) db.disconnect() return totalFrames, props
def test_addImageWithLink(self): db = vdms.vdms() db.connect(hostname, port) all_queries = [] props = {} props["name"] = "Luis" props["lastname"] = "Ferro" props["age"] = 27 addEntity = {} addEntity["_ref"] = 32 addEntity["properties"] = props addEntity["class"] = "AwesomePeople" query = {} query["AddEntity"] = addEntity all_queries.append(query) props = {} props["name"] = "Luis" props["lastname"] = "Malo" props["age"] = 27 link = {} link["ref"] = 32 link["direction"] = "in" link["class"] = "Friends" addImage = {} addImage["properties"] = props addImage["link"] = link addImage["format"] = "png" imgs_arr = [] fd = open("../test_images/brain.png", 'rb') imgs_arr.append(fd.read()) fd.close() img_params = {} query = {} query["AddImage"] = addImage all_queries.append(query) response, res_arr = db.query(all_queries, [imgs_arr]) self.assertEqual(response[0]["AddEntity"]["status"], 0) self.assertEqual(response[1]["AddImage"]["status"], 0)
def test_findBoundingBoxBlob(self): db = vdms.vdms() db.connect(hostname, port) prefix_name = "my_brain_return_" all_queries = [] for i in range(0, self.number_of_inserts): db = vdms.vdms() db.connect(hostname, port) img_name = prefix_name + str(i) imgprops = {} imgprops["name"] = img_name self.addBoundingBoxwithImage(db, 1, imgprops) for i in range(0, self.number_of_inserts): constraints = {} constraints["name"] = ["==", prefix_name + str(i) + "_bb_0"] results = {} results["blob"] = True results["list"] = ["name"] img_params = {} img_params["constraints"] = constraints img_params["results"] = results query = {} query["FindBoundingBox"] = img_params all_queries.append(query) response, img_array = db.query(all_queries) self.assertEqual(len(img_array), self.number_of_inserts) for i in range(0, self.number_of_inserts): coord = self.number_of_inserts - i - 1 self.assertEqual(response[i]["FindBoundingBox"]["status"], 0) self.assertEqual(response[i]["FindBoundingBox"]["entities"][0]["name"], prefix_name + str(i) + "_bb_0")
def test_addVideoWithLink(self): db = vdms.vdms() db.connect(hostname, port) all_queries = [] props = {} props["name"] = "Luis" props["lastname"] = "Ferro" props["age"] = 27 addEntity = {} addEntity["_ref"] = 32 addEntity["properties"] = props addEntity["class"] = "AwPeopleVid" query = {} query["AddEntity"] = addEntity all_queries.append(query) props = {} props["name"] = "Luis" props["lastname"] = "Malo" props["age"] = 27 link = {} link["ref"] = 32 link["direction"] = "in" link["class"] = "Friends" addVideo = {} addVideo["properties"] = props addVideo["link"] = link imgs_arr = [] fd = open("../test_videos/Megamind.avi", 'rb') imgs_arr.append(fd.read()) fd.close() img_params = {} query = {} query["AddVideo"] = addVideo all_queries.append(query) response, res_arr = db.query(all_queries, [imgs_arr]) self.assertEqual(response[0]["AddEntity"]["status"], 0) self.assertEqual(response[1]["AddVideo"]["status"], 0)
def test_findDescByBlob(self): # Add Set set_name = "findwith_blob" dims = 128 total = 100 self.create_set_and_insert(set_name, dims, total) db = vdms.vdms() db.connect(hostname, port) kn = 3 all_queries = [] finddescriptor = {} finddescriptor["set"] = set_name results = {} results["list"] = ["myid", "_id", "_distance"] results["blob"] = True finddescriptor["results"] = results finddescriptor["k_neighbors"] = kn query = {} query["FindDescriptor"] = finddescriptor all_queries = [] all_queries.append(query) descriptor_blob = [] x = np.ones(dims) x[2] = 2.34 + 30 * 20 x = x.astype('float32') descriptor_blob.append(x.tobytes()) response, blob_array = db.query(all_queries, [descriptor_blob]) self.assertEqual(len(blob_array), kn) self.assertEqual(descriptor_blob[0], blob_array[0]) # Check success self.assertEqual(response[0]["FindDescriptor"]["status"], 0) self.assertEqual(response[0]["FindDescriptor"]["returned"], kn) self.assertEqual( response[0]["FindDescriptor"]["entities"][0]["_distance"], 0) self.assertEqual( response[0]["FindDescriptor"]["entities"][1]["_distance"], 400) self.assertEqual( response[0]["FindDescriptor"]["entities"][2]["_distance"], 400)
def create_set_and_insert(self, set_name, dims, total): db = vdms.vdms() db.connect(hostname, port) all_queries = [] # Add Set descriptor_set = {} descriptor_set["name"] = set_name descriptor_set["dimensions"] = dims query = {} query["AddDescriptorSet"] = descriptor_set all_queries.append(query) response, img_array = db.query(all_queries) self.assertEqual(response[0]["AddDescriptorSet"]["status"], 0) all_queries = [] descriptor_blob = [] class_counter = -1 for i in range(0, total - 1): if ((i % 4) == 0): class_counter += 1 x = np.ones(dims) x[2] = 2.34 + i * 20 x = x.astype('float32') descriptor_blob.append(x.tobytes()) descriptor = {} descriptor["set"] = set_name descriptor["label"] = "class" + str(class_counter) props = {} props["myid"] = i + 200 descriptor["properties"] = props query = {} query["AddDescriptor"] = descriptor all_queries.append(query) response, img_array = db.query(all_queries, [descriptor_blob]) # Check success for x in range(0, total - 1): self.assertEqual(response[x]["AddDescriptor"]["status"], 0)
def test_FindWithSortKey(self): db = vdms.vdms() db.connect(hostname, port) all_queries = [] number_of_inserts = 10 for i in range(0, number_of_inserts): props = {} props["name"] = "entity_" + str(i) props["id"] = i entity = {} entity["properties"] = props entity["class"] = "Random" query = {} query["AddEntity"] = entity all_queries.append(query) response, blob_arr = db.query(all_queries) self.assertEqual(len(response), number_of_inserts) for i in range(0, number_of_inserts): self.assertEqual(response[i]["AddEntity"]["status"], 0) all_queries = [] results = {} results["list"] = ["name", "id"] results["sort"] = "id" entity = {} entity["results"] = results entity["class"] = "Random" query = {} query["FindEntity"] = entity all_queries.append(query) response, blob_arr = db.query(all_queries) self.assertEqual(response[0]["FindEntity"]["status"], 0) for i in range(0, number_of_inserts): self.assertEqual(response[0]["FindEntity"]["entities"][i]["id"], i)
def test_addEntityWithLink(self): db = vdms.vdms() db.connect(hostname, port) all_queries = [] props = {} props["name"] = "Luis" props["lastname"] = "Ferro" props["age"] = 27 addEntity = {} addEntity["_ref"] = 32 addEntity["properties"] = props addEntity["class"] = "AwesomePeople" query = {} query["AddEntity"] = addEntity all_queries.append(query) props = {} props["name"] = "Luis" props["lastname"] = "Bueno" props["age"] = 27 link = {} link["ref"] = 32 link["direction"] = "in" link["class"] = "Friends" addEntity = {} addEntity["properties"] = props addEntity["class"] = "AwesomePeople" addEntity["link"] = link img_params = {} query = {} query["AddEntity"] = addEntity all_queries.append(query) response, res_arr = db.query(all_queries) self.assertEqual(response[0]["AddEntity"]["status"], 0) self.assertEqual(response[1]["AddEntity"]["status"], 0)