def update_docs_by_ids(self, docs_matches, pred_labed, config_relative_path=None): if len(docs_matches)>0: if config_relative_path != None: my_connector = Es_connector(index=self.index, config_relative_path=config_relative_path) else: my_connector = Es_connector(index=self.index) query = { "query": { "bool": { "should": docs_matches, "minimum_should_match": 1 } } } original_docs = my_connector.search(query)["hits"]["hits"] if len(original_docs)>0: for doc in original_docs: # my_connector = Es_connector(index=self.index, config_relative_path=config_relative_path)\ my_connector.es.update( index=self.index, doc_type="tweet", id=doc["_id"], body={"doc": { self.session: pred_labed }}, retry_on_conflict=5 )
def get_answers(self, **kwargs): if "config_relative_path" in kwargs: my_connector = Es_connector( index=kwargs["index"], config_relative_path=kwargs["config_relative_path"]) else: my_connector = Es_connector(index=kwargs["index"]) wrong_labels = 0 all_ids = self.join_ids(kwargs["questions"]) res = my_connector.search({"query": {"match": {"id_str": all_ids}}}) for question in kwargs["questions"]: question_id = self.classifier.extract_filename_no_ext( question["filename"]) gt_tweet = [ tweet for tweet in res["hits"]["hits"] if tweet["_source"]["id_str"] == question_id ] question["label"] = gt_tweet[0]["_source"][kwargs["gt_session"]] if question["pred_label"] != question["label"]: wrong_labels += 1 # print(json.dumps(kwargs["questions"], indent=4, sort_keys=True)) return kwargs["questions"], wrong_labels
def add_session(self, name, index): my_connector = Es_connector(index=self.sessions_index, doc_type=self.sessions_doc_type) session = self.get_session_by_Name(name) if session['hits']['total'] == 0: res = my_connector.post({ "s_name": name, "s_index": index, "s_type": "tweet" }) tweets_connector = Es_connector(index=index, doc_type="tweet") tweets_connector.update_all('session_' + name, 'proposed') return res else: return False
def update_tweets_state_by_event_ngram(self, **kwargs): tweets_connector = Es_connector(index=kwargs["index"], doc_type="tweet") query = { "query": { "bool": { "should": kwargs["target_terms"], "minimum_should_match": 1, "must": [{ "match_phrase": { kwargs["ngramsPropName"]: kwargs["ngram"] } }, { "match": { kwargs["session"]: kwargs["query_label"] } }] } } } return tweets_connector.update_query(query, kwargs["session"], kwargs["new_label"])
def get_event_tweets(self, index="test3", main_term="", related_terms=""): my_connector = Es_connector(index=index) terms = [] words = main_term + ' ' for t in related_terms: terms.append( {"match": { "text": { "query": t['word'], "boost": t['value'] } }}) words += t['word'] + " " terms.append({"match": {"text": {"query": main_term, "boost": 2}}}) # res = my_connector.search({"query": {"term" : { "text" : word }}}) # query = { # "bool": { # "must": { # "match": { # "text": { # "query": main_term, # "operator": "or" # } # } # }, # "should": terms # } # } query = {"sort": ["_score"], "query": {"bool": {"should": terms}}} # print(query) # res = my_connector.search(query) res = my_connector.init_paginatedSearch(query) return res
def get_tweets_query_state(self, index="test3", word="", state="proposed", session=""): my_connector = Es_connector(index=index) query = { "query": { "bool": { "must": { "simple_query_string": { "fields": ["text"], "query": word } }, "filter": { "bool": { "should": [{ "match": { session: state } }] } } } } } res = my_connector.init_paginatedSearch(query) return res
def get_tweets(self, index="test3", word=""): my_connector = Es_connector(index=index) # res = my_connector.search({ # "query": { # "simple_query_string": { # "fields": [ # "text" # ], # "query": word # } # } # }) # res = my_connector.bigSearch( # { # "_source": ["text", "id_str", "extended_entities", "user", "created_at", "link"], # "query": { # "simple_query_string": { # "fields": [ # "text" # ], # "query": word # } # } # }) res = my_connector.init_paginatedSearch({ "query": { "simple_query_string": { "fields": ["text"], "query": word } } }) return res
def add_temporary_labels(self, index, session, duplicated_ids): parts = len(duplicated_ids) / 1000 if int(parts) != parts: parts = int(parts) + 1 else: parts = int(parts) paginated_ids = self.split_list(duplicated_ids, parts) for page_ids in paginated_ids: matching_ids = [] for id in page_ids: matching_ids.append({"match": {"id_str": id}}) Es_connector(index=index).update_by_query( { "query": { "bool": { "should": matching_ids, "minimum_should_match": 1 } } }, "ctx._source." + session + "_tmp = 'unlabeled'")
def get_ngrams_by_query(self, query="", **kwargs): try: my_connector = Es_connector( index=kwargs["index"], config_relative_path=self.config_relative_path) full_query = { "query": query, "size": 0, "aggs": { "ngrams_count": { "terms": { "field": kwargs["n_size"] + "grams.keyword", "size": kwargs["results_size"] }, "aggs": { "status": { "terms": { "field": kwargs["session"] + ".keyword" } } } } } } return my_connector.search(full_query) except Exception as e: print('Error: ' + str(e)) traceback.print_exc() return {}
def generate_text_images_prop(docs, langs=["en", "fr", "es"]): ngramsAnalizer = NgramBasedClasifier() for tweet in docs: image_clusters = tweet["_source"].get("imagesCluster", []) image_clusters_str = '' for cluster_id in image_clusters: image_clusters_str += ' ' + str(cluster_id) doc_text = tweet["_source"][input_field] clean_text = ngramsAnalizer.remove_stop_words(doc_text, langs=langs) # .split() clean_text = ngramsAnalizer.remove_urls(text=clean_text) lang = "en" if tweet["_source"]["lang"] is not None: lang = tweet["_source"]["lang"] clean_text = ngramsAnalizer.lemmatize(clean_text, lang) # "en", "fr" full_text = clean_text + image_clusters_str Es_connector(index=index).es.update( index=index, doc_type="tweet", id=tweet["_id"], body={"doc": { output_field: full_text }}) print("Languages for stopwords: ", ngramsAnalizer.retrievedLangs)
def get_sessions(self): my_connector = Es_connector(index=self.sessions_index, doc_type=self.sessions_doc_type) query = {"query": {"match_all": {}}} res = my_connector.search(query) return res
def get_clusters(self, index="test3", word=""): my_connector = Es_connector(index=index) res = my_connector.search({ "size": 1, "query": { "simple_query_string": { "fields": ["text"], "query": word } }, "aggs": { "group_by_cluster": { "terms": { "field": "imagesCluster", "size": 9999 } } } }) # print("Clusters") # print(res['aggregations']['group_by_cluster']['buckets']) clusters = res['aggregations']['group_by_cluster']['buckets'] with open(index + '.json') as f: data = json.load(f) for cluster in clusters: # print(cluster['key']) images = data['duplicates'][cluster['key']] # print(images[0]) cluster['image'] = images[0] cluster['size'] = len(images) # print(clusters) return clusters
def get_similar_docs(self, **kwargs): my_connector = Es_connector(index=kwargs["index"]) # , config_relative_path='../') duplicated_docs = [] # IT SHOULD BE BETTER TO TRY GETTING THE DUPLICATES FROM THE MATRIX splitted_questions = [] target_bigrams = [] # Process by 90 for question in kwargs["questions"]: splitted_questions.append(question) if (len(splitted_questions) > 99): matching_docs = self.process_duplicated_answers(my_connector, kwargs["session"], splitted_questions, kwargs["text_field"], kwargs["similarity_percentage"]) duplicated_docs += matching_docs splitted_questions = [] # re init if len(splitted_questions) > 0: matching_docs = self.process_duplicated_answers(my_connector, kwargs["session"], splitted_questions, kwargs["text_field"], kwargs["similarity_percentage"]) duplicated_docs += matching_docs return duplicated_docs
def get_tweets(self, index, doc_field): my_connector = Es_connector(index=index) all_tweets = [] query = { "_source": [doc_field, "timestamp_ms"], "query": { "exists": { "field": doc_field } } } res = my_connector.init_paginatedSearch(query) sid = res["sid"] scroll_size = res["scroll_size"] # Analyse and process page by page processed_tweets = 0 while scroll_size > 0: tweets = res["results"] all_tweets.extend([{ '_source': { doc_field: self.tknzr.tokenize(tweet["_source"][doc_field]), "timestamp_ms": tweet["_source"]["timestamp_ms"] } } for tweet in tweets]) processed_tweets += scroll_size res = my_connector.loop_paginatedSearch(sid, scroll_size) scroll_size = res["scroll_size"] return all_tweets
def updatePropertyValue(self, **kwargs): tweet = kwargs["tweet"] # cnn = Es_connector(index=kwargs["index"]); # # q = { # "script": { # "inline": "ctx._source." + kwargs["property_name"] + " = params.value", # "lang": "painless", # "params": { # "value": str(kwargs["property_value"]) # } # }, # "query": { # "match": { # "_id": tweet["_id"] # } # } # } # # cnn.es.update_by_query(body=q, doc_type='tweet', index=kwargs["index"]) Es_connector(index=kwargs["index"]).es.update( index=kwargs["index"], doc_type="tweet", id=tweet["_id"], body={"doc": { kwargs["property_name"]: kwargs["property_value"] }}, retry_on_conflict=5)
def set_tweet_state(self, index, session, tid, val): tweets_connector = Es_connector(index=index, doc_type="tweet") session = 'session_' + session query = {"doc": {session: val}} res = tweets_connector.update(tid, query) return res
def get_event_state_tweets_count(self, index="test3", session="", words="", state="confirmed"): my_connector = Es_connector(index=index) query = { "query": { "bool": { "must": [{ "match": { "text": { "query": words } } }], "filter": { "bool": { "should": [{ "match": { "session_" + session: state } }] } } } } } res = my_connector.count(query) return res['count']
def set_cluster_state(self, index, session, cid, state): tweets_connector = Es_connector(index=index, doc_type="tweet") # All tweets session = 'session_' + session query = {"query": {"term": {"imagesCluster": cid}}} res = tweets_connector.update_query(query, session, state) return res
def set_search_status(self, index, session, state, word): tweets_connector = Es_connector(index=index, doc_type="tweet") session = 'session_' + session query = { "query": { "bool": { "must": { "simple_query_string": { "fields": ["text"], "query": word } }, "filter": { "bool": { "should": [{ "match": { session: "proposed" } }] } } } } } res = tweets_connector.update_query(query, session, state) return res
def get_tweets(session, index, state='confirmed'): # Get all confirmed tweets connector = Es_connector(index=index, doc_type='tweet') query = {"query": {"term": {"session_" + session: state}}} res = connector.bigSearch(query) return res
def update_session_results(self, id, events, impact_data): my_connector = Es_connector(index=self.sessions_index, doc_type=self.sessions_doc_type) res = my_connector.update( id, {"doc": { "events": events, "impact_data": impact_data }}) return res
def get_tweets_state(self, index="test3", session="", state="proposed"): my_connector = Es_connector(index=index) res = my_connector.init_paginatedSearch( {"query": { "term": { "session_" + session: state } }}) return res
def remove_all_tmp_predictions_field(self, **kwargs): print("Removing all the predictions") Es_connector(index=kwargs["index"], doc_type="tweet").update_by_query( {"query": { "exists": { "field": kwargs["field"] } }}, "ctx._source.remove('" + kwargs["field"] + "')")
def get_similar_docs(self, **kwargs): if len(kwargs["questions"]) == 0: return [] my_connector = Es_connector(index=kwargs["index"]) # , config_relative_path='../') duplicated_docs = [] docs_ids_matches = [{"match": {"id_str": {"query": question["str_id"] }}} for question in kwargs["questions"]] docs_original_textual_content = my_connector.search({ "query": { "bool": { "should": docs_ids_matches, "minimum_should_match": 1, "must": [ { "match": { kwargs["session"]: "proposed" } } ] } } }) for doc in docs_original_textual_content["hits"]["hits"]: query = { "query": { "bool": { "must": [ { "term": { "text.keyword": { "value": doc["_source"][kwargs["text_field"]] } } } ] } } } matching_docs = my_connector.search(query) if matching_docs["hits"]["total"]>1: label = [question for question in kwargs["questions"] if question["str_id"] == doc["_source"]["id_str"]][0]["label"] for dup_doc in matching_docs["hits"]["hits"]: duplicated_docs.append({ "filename": dup_doc["_source"]["id_str"], "label": label, kwargs["text_field"]: dup_doc["_source"][kwargs["text_field"]] }) return duplicated_docs
def delete_session(self, id): session_connector = Es_connector(index=self.sessions_index, doc_type=self.sessions_doc_type) session = session_connector.get(id) if session: print("delete Session") # print(session) # 1. Delete session data from the tweets tweets_connector = Es_connector( index=session['_source']['s_index'], doc_type=session['_source']['s_type']) session_name = 'session_' + session['_source']['s_name'] print(session_name) tweets_connector.remove_field_all(session_name) # 2. Delete the session session_connector.delete(id) return True else: return False
def download_tweets_from_elastic(self, **kwargs): debug_limit = kwargs.get("debug_limit", False) log_enabled = kwargs.get("log_enabled", True) if "config_relative_path" in kwargs: my_connector = Es_connector( index=kwargs["index"], doc_type="tweet", config_relative_path=kwargs["config_relative_path"]) else: my_connector = Es_connector( index=kwargs["index"], doc_type="tweet") # config_relative_path='../') res = my_connector.init_paginatedSearch(kwargs["query"]) sid = res["sid"] scroll_size = res["scroll_size"] total = int(res["total"]) processed = len(res["results"]) self.write_data_in_folders(kwargs["field"], kwargs["folder"], res["results"]) while scroll_size > 0: res = my_connector.loop_paginatedSearch(sid, scroll_size) scroll_size = res["scroll_size"] processed += len(res["results"]) # Writing the retrieved files into the folders self.write_data_in_folders(kwargs["field"], kwargs["folder"], res["results"]) if log_enabled: print("Downloading: ", round(processed * 100 / total, 2), "%") if debug_limit: print("\nDEBUG LIMIT\n") res = my_connector.loop_paginatedSearch(sid, scroll_size) self.write_data_in_folders(kwargs["field"], kwargs["folder"], res["results"]) scroll_size = 0 return total
def restore_index(self, index, repo_name, snapshot_name, timeout="2m"): res = Es_connector(index=index, config_relative_path='../').es.snapshot.restore( repository=repo_name, snapshot=snapshot_name, wait_for_completion=True, master_timeout=timeout) print("The index ", index, " was restored") return res
def delete_index(self, index, timeout="2m"): try: res = Es_connector(index=index, config_relative_path='../').es.indices.delete( index=index, timeout=timeout) print("The index ", index, " was removed") return res except elasticsearch.exceptions.NotFoundError as e: return e
def get_event_tweets2(self, index="test3", main_term="", related_terms="", cid=0): my_connector = Es_connector(index=index) terms = [] words = main_term + ' ' for t in related_terms: terms.append( {"match": { "text": { "query": t['word'], "boost": t['value'] } }}) words += t['word'] + " " terms.append({"match": {"text": {"query": main_term, "boost": 2}}}) # terms.append({"match": { # "imagesCluster": { # "query": cid # } # }}) # query = { # "query": { # "bool": { # "must": { # "exists": { # "field": "imagesCluster" # } # }, # # "must": { "match": { "imagesCluster" : cid }}, # "should": terms # } # } # } query = { "sort": ["_score"], "query": { "bool": { "should": terms, "minimum_should_match": 1, "must": [{ "match": { "imagesCluster": cid } }] } } } # res = my_connector.bigSearch(query) res = my_connector.init_paginatedSearch(query) return res
def clear_temporary_labels(self, index, session): Es_connector(index=index).update_by_query( { "query": { "exists": { "field": session + "_tmp" } # e.g. session_lyon2015_test_01_tmp } }, "ctx._source.remove('" + session + "_tmp')")