def archive_search(): obj = {} obj["query"] = {} sessionstr = flask.request.args.get("session", None) statestr = flask.request.args.get("state", None) obj["query"]["session"] = sessionstr obj["query"]["state"] = statestr try: stateuid = uuid.UUID(statestr) sessionuid = uuid.UUID(sessionstr) ret = backend.archive_search(sessionuid, stateuid, medtest_kernal) obj["host"] = ret[0].host obj["port"] = ret[0].port obj["name"] = ret[0].name obj["collection"] = ret[0].collection return jsonify(obj) except Exception as e: obj["error"] = str(type(e)) + ": " + str(e) return jsonify(obj)
def refine_search(): """ Corresponds to def refine_search(self, search_uuid, refined_positive_ids, refined_negative_ids, removed_positive_ids, removed_negative_ids): Accepts the +ves -ves and @return: """ obj = {} # query : will be event kit in case of triage information uid = flask.request.args.get("uid", None) qpositive = flask.request.args.get("positive", "[]") # json array qnegative = flask.request.args.get("negative", "[]") # json array if uid == None: obj["error"] = "Missing search ID" positive = [] negative = [] # Convert from HVC to non HVC for apos in json.loads(qpositive): if len(apos) == 9: positive.append(int(apos[3:])) else: positive.append(int(apos)) for apos in json.loads(qnegative): if len(apos) == 9: negative.append(int(apos[3:])) else: negative.append(int(apos[3:])) obj["query"] = {} obj["query"]["uid"] = uid obj["query"]["positive"] = positive obj["query"]["negative"] = negative try: ret = backend.refine_iqr_search(uid, positive, negative, [], []) except Exception as e: obj["error"] = str(type(e)) + ": " + str(e) return jsonify(obj) obj["host"] = ret[0].host obj["port"] = ret[0].port obj["name"] = ret[0].name obj["collection"] = ret[0].collection obj["state"] = "http://localhost:5003/iqr/search_state?" + urllib.urlencode( {"uid": uid}) obj["results"] = "http://localhost:5003/iqr/search_results?" + urllib.urlencode( {"uid": uid}) return jsonify(obj)
def refine_search(): """ Corresponds to def refine_search(self, search_uuid, refined_positive_ids, refined_negative_ids, removed_positive_ids, removed_negative_ids): Accepts the +ves -ves and @return: """ obj = {} # query : will be event kit in case of triage information uid = flask.request.args.get("uid", None) qpositive = flask.request.args.get("positive", "[]") # json array qnegative = flask.request.args.get("negative", "[]") # json array if uid == None: obj["error"] = "Missing search ID" positive = [] negative = [] # Convert from HVC to non HVC for apos in json.loads(qpositive): if len(apos) == 9: positive.append(int(apos[3:])) else: positive.append(int(apos)) for apos in json.loads(qnegative): if len(apos) == 9: negative.append(int(apos[3:])) else: negative.append(int(apos[3:])) obj["query"] = {} obj["query"]["uid"] = uid obj["query"]["positive"] = positive obj["query"]["negative"] = negative try: ret = backend.refine_iqr_search(uid,positive,negative,[],[]) except Exception as e: obj["error"] = str(type(e)) + ": " + str(e) return jsonify(obj) obj["host"] = ret[0].host obj["port"] = ret[0].port obj["name"] = ret[0].name obj["collection"] = ret[0].collection obj["state"] = "http://localhost:5003/iqr/search_state?" + urllib.urlencode({"uid" : uid}) obj["results"] = "http://localhost:5003/iqr/search_results?" + urllib.urlencode({"uid" : uid}) return jsonify(obj)
def init_new_search_session(): """ Initialize the search results GET params query and feature both strings feature must be within known features @return: """ request = flask.request obj = {} # query : will be event kit in case of triage information query = request.args.get("query", "") feature_type = request.args.get("feature", "csift") # Prepare kernal if query: obj["query"] = urllib2.unquote("query") obj["feature"] = feature_type assert feature_type in known_features dkm = tc.get_dk(feature_type) uid = backend.init_new_search_session(None, obj["query"], dkm) obj["uid"] = str(uid) obj["spoe"] = "other" obj["next"] = "http://localhost:5003/iqr/refine_search?" + urllib.urlencode( {"uid": str(uid)}) return jsonify(obj)
def init_new_search_session(): """ Initialize the search results GET params query and feature both strings feature must be within known features @return: """ request = flask.request obj = {} # query : will be event kit in case of triage information query = request.args.get("query", "") feature_type = request.args.get("feature", "csift") # Prepare kernal if query: obj["query"] = urllib2.unquote("query") obj["feature"] = feature_type assert feature_type in known_features dkm = tc.get_dk(feature_type) uid = backend.init_new_search_session(None, obj["query"], dkm ) obj["uid"] = str(uid) obj["spoe"] = "other" obj["next"] = "http://localhost:5003/iqr/refine_search?" + urllib.urlencode({"uid" : str(uid)}) return jsonify(obj)
def random_results(): # skip # limit obj = {} obj["query"] = {} skip = int(flask.request.args.get('skip', "0")) limit = int(flask.request.args.get('limit', "20")) obj["query"]["skip"] = skip obj["query"]["limit"] = limit obj["clips"] = [] count = 0 for i in range(limit/2): # Get a random skip = int(random.random()*200) # Load the clip results clipres = clips.find({"dataset": "MEDTEST", "middle_preview" : {"$exists" : 1}},{"id" : 1, "duration" : 1, "_id" : 0}).skip(skip).limit(2) for aclip in clipres: count = count + 1 aclip["rank"] = count obj["clips"].append(aclip) # obj["clips"] = [aclip["rank"] = for aclip in clipresults] return jsonify(obj)
def archive_search_results(): skip = int(flask.request.args.get("skip", "0")) limit = int(flask.request.args.get("limit", "20")) query = flask.request.args.get("query", "") try: uidstr = json.loads(query) except: uidstr = query obj = {} if uidstr == None: obj["error"] = "Missing search ID" obj["query"] = {} obj["query"]["uid"] = uidstr obj["clips"] = [] states = backend.get_search_sessions() obj["sessions"] = [] for astate in states: obj["sessions"].append(str(astate)) try: uid = uuid.UUID(uidstr) allres = searchdb[uidstr].find({ "model_id": "FUSION" }).sort([("probability", pymongo.DESCENDING)]).skip(skip).limit(limit) rank = skip + 1 for one in allres: aclip = {} aclip["score"] = one["probability"] aclip["id"] = "HVC" + str(one["clip_id"]).zfill(6) clipobj = db["clips"].find_one( {"id": "HVC" + str(one["clip_id"]).zfill(6)}, {"duration": 1}) aclip["duration"] = clipobj["duration"] aclip["rank"] = rank rank = rank + 1 obj["clips"].append(aclip) obj["count"] = len(obj["clips"]) except Exception as e: obj["error"] = str(type(e)) + ": " + str(e) return jsonify(obj) return jsonify(obj)
def archive_search_results(): skip = int(flask.request.args.get("skip", "0")) limit = int(flask.request.args.get("limit", "20")) query = flask.request.args.get("query", "") try: uidstr = json.loads(query) except: uidstr = query obj = {} if uidstr == None: obj["error"] = "Missing search ID" obj["query"] = {} obj["query"]["uid"] = uidstr obj["clips"] = [] states = backend.get_search_sessions() obj["sessions"] = [] for astate in states: obj["sessions"].append(str(astate)) try: uid = uuid.UUID(uidstr) allres = searchdb[uidstr].find({"model_id" : "FUSION"}).sort([("probability", pymongo.DESCENDING)]).skip(skip).limit(limit) rank = skip + 1 for one in allres: aclip = {} aclip["score"] = one["probability"] aclip["id"] = "HVC" + str(one["clip_id"]).zfill(6) clipobj = db["clips"].find_one({"id" : "HVC" + str(one["clip_id"]).zfill(6)},{"duration" : 1}) aclip["duration"] = clipobj["duration"] aclip["rank"] = rank rank = rank + 1 obj["clips"].append(aclip) obj["count"] = len(obj["clips"]) except Exception as e: obj["error"] = str(type(e)) + ": " + str(e) return jsonify(obj) return jsonify(obj)
def search_state(): """ Accepts uuid @return: """ obj = {} # query : will be event kit in case of triage information uidstr = flask.request.args.get("uid", None) if uidstr == None: obj["error"] = "Missing search ID" obj["query"] = {} obj["query"]["uid"] = uidstr # obj["vids"] = [] states = backend.get_search_sessions() obj["sessions"] = [] for astate in states: obj["sessions"].append(str(astate)) try: uid = uuid.UUID(uidstr) state = backend.get_iqr_search_state(uid) # use the uid of the state and get the information from the database col = str(state.uuid) # obj["collection"] = col # all = searchdb[col].find() # for one in all: # obj["vids"].append(one) searchdb[col].ensure_index([("model_id", pymongo.ASCENDING)]) obj["count"] = searchdb[col].find({"model_id": "FUSION"}).count() # obj["stateobj"] = state except Exception as e: obj["error"] = str(type(e)) + ": " + str(e) return jsonify(obj) obj["state"] = "http://localhost:5003/iqr/search_state?" + urllib.urlencode( {"uid": str(uidstr)}) return jsonify(obj)
def search_state(): """ Accepts uuid @return: """ obj = {} # query : will be event kit in case of triage information uidstr = flask.request.args.get("uid", None) if uidstr == None: obj["error"] = "Missing search ID" obj["query"] = {} obj["query"]["uid"] = uidstr # obj["vids"] = [] states = backend.get_search_sessions() obj["sessions"] = [] for astate in states: obj["sessions"].append(str(astate)) try: uid = uuid.UUID(uidstr) state = backend.get_iqr_search_state(uid) # use the uid of the state and get the information from the database col = str(state.uuid) # obj["collection"] = col # all = searchdb[col].find() # for one in all: # obj["vids"].append(one) searchdb[col].ensure_index([("model_id", pymongo.ASCENDING)]) obj["count"] = searchdb[col].find({"model_id" : "FUSION"}).count() # obj["stateobj"] = state except Exception as e: obj["error"] = str(type(e)) + ": " + str(e) return jsonify(obj) obj["state"] = "http://localhost:5003/iqr/search_state?" + urllib.urlencode({"uid" : str(uidstr)}) return jsonify(obj)
def adjudication(): """ Api for adjudication @return: """ request = flask.request obj = {} # clip id clip = request.args.get("clip", None) if clip: obj["clip"] = clip # query : will be event kit in case of triage information query = request.args.get("query", None) if query: obj["query"] = query # If no label in the query then assume star, other options are yes and no obj["label"] = request.args.get("label", "star") # If no op in then assume add, other option is remove obj["op"] = request.args.get("op", "add") # Comment comment = request.args.get("comment", None) if comment: obj["comment"] = comment # Get ip address if not request.headers.getlist("X-Forwarded-For"): obj["ip"] = request.remote_addr else: obj["ip"] = request.headers.getlist("X-Forwarded-For")[0] # Get current time obj["time"] = datetime.datetime.now() obj["strtime"] = str(obj["time"]) # get user obj["user"] = "******" if flask.session.has_key("user"): # user logged in obj["user"] = flask.session["user"]["fullname"] db["adjudication"].insert(obj) return jsonify(obj)
def event_results(): # Event ID # DataSet # Now obsolete use /event_results_from_training obj = {} obj["query"] = {} kit = flask.request.args.get('kit', "1") obj["query"]["kit"] = kit training = flask.request.args.get('training', "0Ex") obj["query"]["training"] = training attribute = "scores.0.E%03d" % (int(kit)) obj["query"]["attribute"] = attribute skip = int(flask.request.args.get('skip', "0")) limit = int(flask.request.args.get('limit', "10")) obj["query"]["skip"] = skip obj["query"]["limit"] = limit # Load the clip results clipresults = clips.find({ attribute: { "$exists": 1 } }, { "_id": 1, "duration": 1, "scores": 1, "id": 1 }).sort(attribute, pymongo.DESCENDING).skip(skip).limit(limit) obj["clips"] = [] # Load duration count = 1 for aclip in clipresults: aclip["rank"] = skip + count aclip["score"] = aclip["scores"][0]["E%03d" % int(kit)] del aclip["scores"] obj["clips"].append(aclip) count = count + 1 # obj["clips"] = [aclip["rank"] = for aclip in clipresults] return jsonify(obj)
def event_results_from_training(): # Event ID # DataSet obj = {} obj["query"] = {} kit = flask.request.args.get('kit', "1") obj["query"]["kit"] = kit training = flask.request.args.get('training', "100Ex") obj["query"]["training"] = training attribute = "scores_eval.%s.E%03d." % (training, int(kit)) obj["query"]["attribute"] = attribute skip = int(flask.request.args.get('skip', "0")) limit = int(flask.request.args.get('limit', "10")) obj["query"]["skip"] = skip obj["query"]["limit"] = limit obj["query"]["attribute"] = attribute # Load the clip results clipresults = clipscores.find({}, { "_id": 1, attribute: 1, "id": 1 }).sort(attribute, pymongo.DESCENDING).skip(skip).limit(limit) obj["clips"] = [] # Load duration count = 1 for aclip in clipresults: aclip["rank"] = skip + count aclip["score"] = aclip["scores_eval"][training]["E%03d" % int(kit)] aclipobj = clips.find_one({"id": aclip["id"]}, {"duration": 1}) if "duration" in aclipobj: aclip["duration"] = aclipobj["duration"] del aclip["scores_eval"] obj["clips"].append(aclip) count = count + 1 return jsonify(obj)
def event_results_from_training(): # Event ID # DataSet obj = {} obj["query"] = {} kit = flask.request.args.get('kit',"1") obj["query"]["kit"] = kit training = flask.request.args.get('training',"100Ex") obj["query"]["training"] = training attribute = "scores_eval.%s.E%03d."%(training, int(kit)) obj["query"]["attribute"] = attribute skip = int(flask.request.args.get('skip', "0")) limit = int(flask.request.args.get('limit', "10")) obj["query"]["skip"] = skip obj["query"]["limit"] = limit obj["query"] ["attribute"]= attribute # Load the clip results clipresults = clipscores.find({}, {"_id" : 1, attribute: 1, "id" : 1}).sort(attribute,pymongo.DESCENDING).skip(skip).limit(limit) obj["clips"] = [] # Load duration count = 1 for aclip in clipresults: aclip["rank"] = skip + count aclip["score"] = aclip["scores_eval"][training]["E%03d"%int(kit)] aclipobj = clips.find_one({"id" : aclip["id"]},{"duration" : 1}) if "duration" in aclipobj: aclip["duration"] = aclipobj["duration"] del aclip["scores_eval"] obj["clips"].append(aclip) count = count + 1 return jsonify(obj)
def event_results(): # Event ID # DataSet # Now obsolete use /event_results_from_training obj = {} obj["query"] = {} kit = flask.request.args.get('kit',"1") obj["query"]["kit"] = kit training = flask.request.args.get('training',"0Ex") obj["query"]["training"] = training attribute = "scores.0.E%03d"%(int(kit)) obj["query"]["attribute"] = attribute skip = int(flask.request.args.get('skip', "0")) limit = int(flask.request.args.get('limit', "10")) obj["query"]["skip"] = skip obj["query"]["limit"] = limit # Load the clip results clipresults = clips.find({attribute : {"$exists" : 1}}, {"_id" : 1, "duration": 1, "scores": 1, "id" : 1}).sort(attribute,pymongo.DESCENDING).skip(skip).limit(limit) obj["clips"] = [] # Load duration count = 1 for aclip in clipresults: aclip["rank"] = skip + count aclip["score"] = aclip["scores"][0]["E%03d"%int(kit)] del aclip["scores"] obj["clips"].append(aclip) count = count + 1 # obj["clips"] = [aclip["rank"] = for aclip in clipresults] return jsonify(obj)
def random_results(): # skip # limit obj = {} obj["query"] = {} skip = int(flask.request.args.get('skip', "0")) limit = int(flask.request.args.get('limit', "20")) obj["query"]["skip"] = skip obj["query"]["limit"] = limit obj["clips"] = [] count = 0 for i in range(limit / 2): # Get a random skip = int(random.random() * 200) # Load the clip results clipres = clips.find( { "dataset": "MEDTEST", "middle_preview": { "$exists": 1 } }, { "id": 1, "duration": 1, "_id": 0 }).skip(skip).limit(2) for aclip in clipres: count = count + 1 aclip["rank"] = count obj["clips"].append(aclip) # obj["clips"] = [aclip["rank"] = for aclip in clipresults] return jsonify(obj)
def search_results(): """ Corresponds to Accepts uid and pool index @return: """ skip = int(flask.request.args.get("skip", "0")) limit = int(flask.request.args.get("limit", "20")) obj = {} # query : will be event kit in case of triage information uidstr = flask.request.args.get("query", None) if uidstr == None: obj["error"] = "Missing search ID" uidstr = json.loads(uidstr) obj["query"] = {} obj["query"]["uid"] = uidstr obj["clips"] = [] states = backend.get_search_sessions() obj["sessions"] = [] for astate in states: obj["sessions"].append(str(astate)) try: uid = uuid.UUID(uidstr) state = backend.get_iqr_search_state(uid) # use the uid of the state and get the information from the database col = str(state.uuid) obj["collection"] = col searchdb[col].ensure_index([("model_id", pymongo.ASCENDING),("probability", pymongo.DESCENDING) ]) # Force probabilities obj["positives"] = list(state.positives) obj["negatives"] = list(state.negatives) log = "" for id in state.positives: # log = log + "Found %d"%(searchdb[col].find({"model_id" : "FUSION", "clip_id" : id}).count()) + ", " # res = searchdb[col].update({"model_id" : "FUSION", "clip_id" : id}, {"$set" : { "probability" : 1.0}}) # log = log + "Done %d"%id + ", " news = searchdb[col].find_one({"model_id" : "FUSION", "clip_id" : id}) news["probability"] = 1.0001 searchdb[col].save(news) log = log + "Now : " + str(news) for id in state.negatives: # log = log + "Found %d"%(searchdb[col].find({"model_id" : "FUSION", "clip_id" : id}).count()) + ", " # res = searchdb[col].update({"model_id" : "FUSION", "clip_id" : id}, {"$set" : { "probability" : 0.0}}) # log = log + "Done %d"%id + ", " news = searchdb[col].find_one({"model_id" : "FUSION", "clip_id" : id}) news["probability"] = 0.0 searchdb[col].save(news) log = log + "Now : " + str(news) obj["log"] = log allres = searchdb[col].find({"model_id" : "FUSION"}).sort([("probability", pymongo.DESCENDING)]).skip(skip).limit(limit) rank = skip + 1 for one in allres: aclip = {} aclip["score"] = one["probability"] aclip["id"] = "HVC" + str(one["clip_id"]).zfill(6) try: clipobj = db["clips"].find_one({"id" : "HVC" + str(one["clip_id"]).zfill(6)},{"duration" : 1}) aclip["duration"] = clipobj["duration"] except: aclip["duration"] = "unknown" aclip["rank"] = rank rank = rank + 1 obj["clips"].append(aclip) obj["count"] = len(obj["clips"]) except Exception as e: obj["error"] = str(type(e)) + ": " + str(e) return jsonify(obj) obj["next"] = "http://localhost:5003/iqr/search_results?" + urllib.urlencode({"uid" : uid, "skip" : skip+limit } ) return jsonify(obj)
def search_results(): """ Corresponds to Accepts uid and pool index @return: """ skip = int(flask.request.args.get("skip", "0")) limit = int(flask.request.args.get("limit", "20")) obj = {} # query : will be event kit in case of triage information uidstr = flask.request.args.get("query", None) if uidstr == None: obj["error"] = "Missing search ID" uidstr = json.loads(uidstr) obj["query"] = {} obj["query"]["uid"] = uidstr obj["clips"] = [] states = backend.get_search_sessions() obj["sessions"] = [] for astate in states: obj["sessions"].append(str(astate)) try: uid = uuid.UUID(uidstr) state = backend.get_iqr_search_state(uid) # use the uid of the state and get the information from the database col = str(state.uuid) obj["collection"] = col searchdb[col].ensure_index([("model_id", pymongo.ASCENDING), ("probability", pymongo.DESCENDING)]) # Force probabilities obj["positives"] = list(state.positives) obj["negatives"] = list(state.negatives) log = "" for id in state.positives: # log = log + "Found %d"%(searchdb[col].find({"model_id" : "FUSION", "clip_id" : id}).count()) + ", " # res = searchdb[col].update({"model_id" : "FUSION", "clip_id" : id}, {"$set" : { "probability" : 1.0}}) # log = log + "Done %d"%id + ", " news = searchdb[col].find_one({ "model_id": "FUSION", "clip_id": id }) news["probability"] = 1.0001 searchdb[col].save(news) log = log + "Now : " + str(news) for id in state.negatives: # log = log + "Found %d"%(searchdb[col].find({"model_id" : "FUSION", "clip_id" : id}).count()) + ", " # res = searchdb[col].update({"model_id" : "FUSION", "clip_id" : id}, {"$set" : { "probability" : 0.0}}) # log = log + "Done %d"%id + ", " news = searchdb[col].find_one({ "model_id": "FUSION", "clip_id": id }) news["probability"] = 0.0 searchdb[col].save(news) log = log + "Now : " + str(news) obj["log"] = log allres = searchdb[col].find({ "model_id": "FUSION" }).sort([("probability", pymongo.DESCENDING)]).skip(skip).limit(limit) rank = skip + 1 for one in allres: aclip = {} aclip["score"] = one["probability"] aclip["id"] = "HVC" + str(one["clip_id"]).zfill(6) try: clipobj = db["clips"].find_one( {"id": "HVC" + str(one["clip_id"]).zfill(6)}, {"duration": 1}) aclip["duration"] = clipobj["duration"] except: aclip["duration"] = "unknown" aclip["rank"] = rank rank = rank + 1 obj["clips"].append(aclip) obj["count"] = len(obj["clips"]) except Exception as e: obj["error"] = str(type(e)) + ": " + str(e) return jsonify(obj) obj["next"] = "http://localhost:5003/iqr/search_results?" + urllib.urlencode( { "uid": uid, "skip": skip + limit }) return jsonify(obj)