def get_bad_headers(collection_obj): rv = [] for doc in collection_obj.find(): if doc['Header']['En'] and doc['Header']['He'] and ( is_hebrew(doc['Header']['En']) or is_hebrew(doc['Header']['He']) == False): rv.append({doc['UnitId']: doc['Header']}) return rv
def get_completion(collection, string, size=7): '''Search in the elastic search index for completion options. Returns tuple of (text_completion_results, phonetic_results) Where each array contains up to `size` results. ''' # currently we only do a simple starts with search, without contains or phonetics # TODO: fix phonetics search, some work was done for that # see https://github.com/Beit-Hatfutsot/dbs-back/blob/2e79c363e40472f28fd07f8a344fe55ab77198ee/bhs_api/v1_endpoints.py#L189 lang = "He" if phonetic.is_hebrew(string) else "En" q = { "_source": ["Slug", "Header"], "suggest": { "header" : { "prefix": string, "completion": { "field": "Header.{}.suggest".format(lang), "size": size, "contexts": { "collection": collection, } } }, } } q["suggest"]["header"]["completion"]["contexts"] = {"collection": collection} results = current_app.es.search(index=current_app.es_data_db_index_name, body=q, size=0) try: header_options = results['suggest']['header'][0]['options'] except KeyError: header_options = [] try: phonetic_options = results['suggest']['phonetic'][0]['options'] except KeyError: phonetic_options = [] return [i['_source']['Header'][lang] for i in header_options], [i['_source']['Header'][lang] for i in phonetic_options]
def search_by_header(string, collection, starts_with=True, db=None): if not db: db = current_app.data_db if not string: # Support empty strings return {} if phonetic.is_hebrew(string): lang = 'He' else: lang = 'En' string_re = re.escape(string) if starts_with: header_regex = re.compile(u'^'+string_re, re.IGNORECASE) else: header_regex = re.compile(u'^{}$'.format(string_re), re.IGNORECASE) lang_header = 'Header.{}'.format(lang) unit_text = 'UnitText1.{}'.format(lang) # Search only for non empty docs with right status show_filter = SHOW_FILTER.copy() show_filter[unit_text] = {"$nin": [None, '']} header_search_ex = {lang_header: header_regex} header_search_ex.update(show_filter) item = db[collection].find_one(header_search_ex) if item: item = enrich_item(item, db) return _make_serializable(item) else: return {}
def search_by_header(string, collection, starts_with=True, db=None): if not db: db = current_app.data_db if not string: # Support empty strings return {} if phonetic.is_hebrew(string): lang = 'He' else: lang = 'En' string_re = re.escape(string) if starts_with: header_regex = re.compile(u'^'+string_re, re.IGNORECASE) else: header_regex = re.compile(u'^{}$'.format(string_re), re.IGNORECASE) lang_header = 'Header.{}'.format(lang) unit_text = 'UnitText1.{}'.format(lang) # Search only for non empty docs with right status show_filter = SHOW_FILTER.copy() show_filter[unit_text] = {"$nin": [None, '']} header_search_ex = {lang_header: header_regex} header_search_ex.update(show_filter) item = db[collection].find_one(header_search_ex) if item: item = enrich_item(item, db) return _make_serializable(item) else: return {}
def es_search(q, size, collection=None, from_=0, sort=None, with_persons=False, **kwargs): if collection: # if user requested specific collections - we don't filter for persons (that's what user asked for!) collections = collection.split(",") else: # we consider the with_persons to decide whether to include persons collection or not collections = [collection for collection in SEARCHABLE_COLLECTIONS if with_persons or collection != "persons"] fields = ["Header.En^2", "Header.He^2", "UnitText1.En", "UnitText1.He"] default_query = { "query_string": { "fields": fields, "query": q, "default_operator": "and" } } for k, v in PERSONS_SEARCH_TEXT_PARAMS_LOWERCASE: if not isinstance(v, list): fields.append(v) must_queries = [] if q: must_queries.append(default_query) for year_param, year_attr in PERSONS_SEARCH_YEAR_PARAMS: if kwargs[year_param]: try: year_value = int(kwargs[year_param]) except Exception as e: raise Exception("invalid value for {} ({}): {}".format(year_param, year_attr, kwargs[year_param])) year_type_param = "{}_t".format(year_param) year_type = kwargs[year_type_param] if year_type == "pmyears": year_type_value_param = "{}_v".format(year_param) try: year_type_value = int(kwargs[year_type_value_param]) except Exception as e: raise Exception("invalid value for {} ({}): {}".format(year_type_value_param, year_attr, kwargs[year_type_value_param])) must_queries.append({"range": {year_attr: {"gte": year_value - year_type_value, "lte": year_value + year_type_value,}}}) elif year_type == "exact": must_queries.append({"term": {year_attr: year_value}}) else: raise Exception("invalid value for {} ({}): {}".format(year_type_param, year_attr, year_type)) for text_param, text_attr in PERSONS_SEARCH_TEXT_PARAMS_LOWERCASE: if kwargs[text_param]: text_value = kwargs[text_param].lower() text_type_param = "{}_t".format(text_param) text_type = kwargs[text_type_param] must_queries.append(get_person_elastic_search_text_param_query(text_type, text_attr, text_value)) for exact_param, exact_attr in PERSONS_SEARCH_EXACT_PARAMS: if kwargs[exact_param]: exact_value = kwargs[exact_param] if exact_param == "sex": exact_value = exact_value.upper() if exact_value not in ["F", "M", "U"]: raise Exception("invalid value for {} ({}): {}".format(exact_param, exact_attr, exact_value)) elif exact_param == "treenum": try: exact_value = int(exact_value) except Exception as e: raise Exception("invalid value for {} ({}): {}".format(exact_param, exact_attr, exact_value)) must_queries.append({"term": {exact_attr: exact_value}}) collection_boosts = { "places": 5 } for collection in SEARCHABLE_COLLECTIONS: if collection not in collection_boosts: collection_boosts[collection] = 1 must_queries.append({"bool": { "should": [{"match": {"_type": {"query": collection, "boost": boost}}} for collection, boost in collection_boosts.items()] }}) body = { "query": { "bool": { "must": must_queries } } } if sort == "abc": if phonetic.is_hebrew(q.strip()): # hebrew alphabetical sort body["sort"] = [{"Header.He_lc": "asc"}, "_score"] else: # english alphabetical sort body["sort"] = [{"Header.En_lc": "asc"}, "_score"] elif sort == "rel": # relevance sort body["sort"] = ["_score"] elif sort == "year" and collection == "photoUnits": body["sort"] = [{"UnitPeriod.PeriodStartDate.keyword": "asc"}, "_score"] try: current_app.logger.debug("es.search index={}, doc_type={} body={}".format(current_app.es_data_db_index_name, collections, json.dumps(body))) results = current_app.es.search(index=current_app.es_data_db_index_name, body=body, doc_type=collections, size=size, from_=from_) except elasticsearch.exceptions.ConnectionError as e: current_app.logger.error('Error connecting to Elasticsearch: {}'.format(e)) raise Exception("Error connecting to Elasticsearch: {}".format(e)) except Exception as e: raise Exception("Elasticsearch error: {}".format(e)) return results
def get_completion(collection, string, size=7): '''Search in the elastic search index for completion options. Returns to array each with up to `size` results. The first array contains the text completion and the second the phonetic suggestions. ''' if phonetic.is_hebrew(string): lang = 'He' else: lang = 'En' ''' TODO: fix pohonetics search or remove this code dms_soundex = phonetic.get_dms(string) if " " in dms_soundex: dms_completion = {"regex":"[{}]".format(dms_soundex.replace(' ', '|'))} dms_completion = {"prefix": dms_soundex.split(' ')[0]} else: dms_completion = {"prefix": dms_soundex} q = { "_source": ["Slug", "Header"], "suggest": { "header" : { "prefix" : string, "completion" : { "field" : "Header.{}.suggest".format(lang), "size": size, "contexts": { "collection": collection, } } }, "phonetic" : { "completion" : { "field" : "dm_soundex", "size": size, "contexts": { "collection": collection, } } } } } q["suggest"]["phonetic"].update(dms_completion) ''' # no phonetics query q = { "_source": ["Slug", "Header"], "suggest": { "header": { "prefix": string, "completion": { "field": "Header.{}.suggest".format(lang), "size": size, "contexts": { "collection": collection, } } }, } } results = current_app.es.search(index=current_app.data_db.name, body=q, size=0) try: header_options = results['suggest']['header'][0]['options'] except KeyError: header_options = [] try: phonetic_options = results['suggest']['phonetic'][0]['options'] except KeyError: phonetic_options = [] return [i['_source']['Header'][lang] for i in header_options],\ [i['_source']['Header'][lang] for i in phonetic_options]
def get_bad_headers(collection_obj): rv = [] for doc in collection_obj.find(): if doc['Header']['En'] and doc['Header']['He'] and (is_hebrew(doc['Header']['En']) or is_hebrew(doc['Header']['He']) == False): rv.append({doc['UnitId']: doc['Header']}) return rv