def _get_elasticsearch_item_key_query(self, collection_name, item_key): if collection_name == "persons": tree_num, tree_version, person_id = item_key return { "bool": { "must": [{ "term": { "tree_num": tree_num } }, { "term": { "tree_version": tree_version } }, { "term": { "person_id": person_id } }] } } else: return { "term": { get_collection_id_field(collection_name): item_key } }
def parse_n_update(row, collection_name): doc = parse_doc(row, collection_name) id_field = get_collection_id_field(collection_name) logger.info('{}:Updating {}: {}'.format( collection_name, id_field, doc[id_field])) update_row.delay(doc, collection_name) return doc
def _get_elasticsearch_item_key_query(self, collection_name, item_key): if collection_name == "persons": tree_num, tree_version, person_id = item_key return {"bool": {"must": [{"term": {"tree_num": tree_num}}, {"term": {"tree_version": tree_version}}, {"term": {"person_id": person_id}}]}} else: return {"term": {get_collection_id_field(collection_name): item_key}}
def parse_n_update(row, collection_name, dryrun=False): doc = parse_doc(row, collection_name) id_field = get_collection_id_field(collection_name) logger.info('{}:Updating {}: {}, updated {}'.format( collection_name, id_field, doc[id_field], doc.get('UpdateDate', '?'))) if not dryrun: update_row.delay(doc, collection_name) return doc
def _get_mongo_items(self, collection_name, key): if key: if collection_name == "persons": raise NotImplementedError("persons does not support updating by key yet") else: items = self.app.data_db[collection_name].find({get_collection_id_field(collection_name): key}) else: items = self.app.data_db[collection_name].find() items = self._limit(items) return items
def update_doc(collection, document): # update place items with geojson if collection.name == 'places': document['geometry'] = get_place_geo(document) # family trees get special treatment if collection.name == 'persons': tree_num = document['tree_num'] id = document['id'] tree_key = 'tree_vers_'+str(tree_num) query = {'tree_num': tree_num, 'id': id} tree_vers = current_app.redis.get(tree_key) if tree_vers: tree_vers = json.loads(tree_vers) i = find_version(tree_vers, document['tree_file_id']) else: tree = current_app.data_db['trees'].find_one({'num':tree_num}) if tree: tree_vers = tree['versions'] current_app.redis.set(tree_key, json.dumps(tree_vers), 300) i = find_version(tree_vers, document['tree_file_id']) else: current_app.logger.info("didn't find tree number {} using version 0 for {}" .format(tree_num, id)) i = 0 document['tree_version'] = i query['tree_version'] = i # we have to create it here as at the moment create_slug function requires Header to create slug # TODO: move this logic to create_slug function document['Slug'] = {'En': 'person_{};{}.{}'.format( tree_num, i, id)} created = update_collection(collection, query, document) if MIGRATE_ES == '1': is_ok, msg = update_es(collection.name, document, created) if not is_ok: current_app.logger.error(msg) current_app.logger.info('Updated person: {}.{}' .format(tree_num, id)) else: doc_id = get_doc_id(collection.name, document) if doc_id: query = {get_collection_id_field(collection): doc_id} created = update_collection(collection, query, document) if MIGRATE_ES == '1': is_ok, msg = update_es(collection.name, document, created) if not is_ok: current_app.logger.error(msg) slug = document.get("Slug", {}).get("En") current_app.logger.info('Updated {} {}, Slug: {}'.format(collection.name, doc_id, slug)) else: current_app.logger.error('update failed because of id {}'.format(collection.name))
def update_doc(collection, document): # update place items with geojson if collection.name == 'places': document['geometry'] = get_place_geo(document) # family trees get special treatment if collection.name == 'persons': tree_num = document['tree_num'] id = document['id'] tree_key = 'tree_vers_' + str(tree_num) query = {'tree_num': tree_num, 'id': id} tree_vers = current_app.redis.get(tree_key) if tree_vers: tree_vers = json.loads(tree_vers) i = find_version(tree_vers, document['tree_file_id']) else: tree = current_app.data_db['trees'].find_one({'num': tree_num}) if tree: tree_vers = tree['versions'] current_app.redis.set(tree_key, json.dumps(tree_vers), 300) i = find_version(tree_vers, document['tree_file_id']) else: current_app.logger.info( "didn't find tree number {} using version 0 for {}".format( tree_num, id)) i = 0 document['tree_version'] = i query['tree_version'] = i # we have to create it here as at the moment create_slug function requires Header to create slug # TODO: move this logic to create_slug function document['Slug'] = {'En': 'person_{};{}.{}'.format(tree_num, i, id)} created = update_collection(collection, query, document) if MIGRATE_ES == '1': is_ok, msg = update_es(collection.name, document, created) if not is_ok: current_app.logger.error(msg) current_app.logger.info('Updated person: {}.{}'.format(tree_num, id)) else: doc_id = get_doc_id(collection.name, document) if doc_id: query = {get_collection_id_field(collection): doc_id} created = update_collection(collection, query, document) if MIGRATE_ES == '1': is_ok, msg = update_es(collection.name, document, created) if not is_ok: current_app.logger.error(msg) slug = document.get("Slug", {}).get("En") current_app.logger.info('Updated {} {}, Slug: {}'.format( collection.name, doc_id, slug)) else: current_app.logger.error('update failed because of id {}'.format( collection.name))
def _get_mongo_items(self, collection_name, key): if key: if collection_name == "persons": raise NotImplementedError( "persons does not support updating by key yet") else: items = self.app.data_db[collection_name].find( {get_collection_id_field(collection_name): key}) else: items = self.app.data_db[collection_name].find() items = self._limit(items) return items
def _get_elasticsearch_item_key(self, collection_name, es_item): if collection_name == "persons": person_id = es_item.get("person_id", None) tree_num = es_item.get("tree_num", None) tree_version = es_item.get("tree_version", None) if person_id is not None and tree_num is not None and tree_version is not None: item_key = int(tree_num), int(tree_version), str(person_id) else: item_key = None else: id_field = get_collection_id_field(collection_name) item_key = es_item.get(id_field, None) return item_key
def _get_elasticsearch_item_key(self, collection_name, es_item): if collection_name == "persons": person_id = es_item.get("person_id", None) tree_num = es_item.get("tree_num", None) tree_version = es_item.get("tree_version", None) if person_id is not None and tree_num is not None and tree_version is not None: item_key = int(tree_num), int(tree_version), str(person_id) else: item_key = None else: id_field = get_collection_id_field(collection_name) item_key = es_item.get(id_field, None) return item_key
def _get_mongo_item_key(self, collection_name, mongo_item): if collection_name == "persons": person_id = mongo_item.get("id", None) if self.args.legacy and not person_id: person_id = mongo_item.get("ID", None) tree_num = mongo_item.get("tree_num", None) tree_version = mongo_item.get("tree_version", None) if person_id is not None and tree_num is not None and tree_version is not None: item_key = int(tree_num), int(tree_version), str(person_id) else: item_key = None else: id_field = get_collection_id_field(collection_name) item_key = mongo_item.get(id_field, None) return item_key
def _get_mongo_item_key(self, collection_name, mongo_item): if collection_name == "persons": person_id = mongo_item.get("id", None) if self.args.legacy and not person_id: person_id = mongo_item.get("ID", None) tree_num = mongo_item.get("tree_num", None) tree_version = mongo_item.get("tree_version", None) if person_id is not None and tree_num is not None and tree_version is not None: item_key = int(tree_num), int(tree_version), str(person_id) else: item_key = None else: id_field = get_collection_id_field(collection_name) item_key = mongo_item.get(id_field, None) return item_key
def _get_index_body(self): body = { "mappings": { collection: { "properties": {"Header": self.header_mapping,} } for collection in SEARCHABLE_COLLECTIONS } } body["mappings"]["familyNames"]["properties"]["dm_soundex"] = { "type": "completion", "max_input_length": 20, "contexts": [{ "name": "collection", "type": "CATEGORY", "path": "_type" }] } for collection_name, mapping in body["mappings"].items(): if collection_name == "persons": # persons specific mappings # ensure all fields relevant for search are properly indexed mapping["properties"].update({"tree_num": {"type": "integer"}, "tree_version": {"type": "integer"}, "person_id": {"type": "keyword"}, "birth_year": {"type": "integer"}, "death_year": {"type": "integer"}, "marriage_years": {"type": "integer"}, # these are updated in bhs_api.item.update_es functions "first_name_lc": {"type": "text"}, "last_name_lc": {"type": "text"}, "BIRT_PLAC_lc": {"type": "text"}, "MARR_PLAC_lc": {"type": "text"}, "DEAT_PLAC_lc": {"type": "text"}, "gender": {"type": "keyword"}}) else: mapping["properties"][get_collection_id_field(collection_name)] = {"type": "keyword"} return body
def _get_item_log_identifier(self, item_key, collection_name): if collection_name == "persons": return "(tree_num,version,id={},{},{})".format(*item_key) else: return "{}={}".format(get_collection_id_field(collection_name), item_key)
def _get_item_log_identifier(self, item_key, collection_name): if collection_name == "persons": return "(tree_num,version,id={},{},{})".format(*item_key) else: return "{}={}".format(get_collection_id_field(collection_name), item_key)
for c_name in SEARCHABLE_COLLECTIONS: if c_name != "persons": # TODO: add support for persons, at the moment it's not working due to the persons not having a single unique id field print("starting work on " + c_name) # in the process we might create duplicate index so remove them for now try: todb[c_name].drop_index('Slug.He_1') except pymongo.errors.OperationFailure: pass try: todb[c_name].drop_index('Slug.En_1') except pymongo.errors.OperationFailure: pass id_field = get_collection_id_field(c_name) # loop on all docs with a slug for from_doc in fromdb[c_name].find({'Slug': {'$exists': True, '$ne': {}}}): to_doc = app.data_db[c_name].find_one( {id_field: from_doc[id_field]}) if not to_doc: print("missing {}".format(get_item_slug(from_doc))) continue if from_doc['Slug'] != to_doc['Slug']: try: todb[c_name].update_one({'_id': to_doc['_id']}, {'$set': {'Slug': from_doc['Slug']} }) except pymongo.errors.DuplicateKeyError as e:
def reslugify(collection, document): ''' append the document id to the slug to ensure uniquness ''' for lang, val in document['Slug'].items(): if val: doc_id = get_collection_id_field(collection.name) document['Slug'][lang] += '-' + str(document[doc_id])
app, conf = create_app() fromdb = app.client_data_db[args.fromdb] todb = app.data_db for c_name in SEARCHABLE_COLLECTIONS: print("starting work on " + c_name) # in the process we might create duplicate index so remove them for now try: todb[c_name].drop_index('Slug.He_1') except pymongo.errors.OperationFailure: pass try: todb[c_name].drop_index('Slug.En_1') except pymongo.errors.OperationFailure: pass id_field = get_collection_id_field(c_name) # loop on all docs with a slug for from_doc in fromdb[c_name].find( {'Slug': { '$exists': True, '$ne': {} }}): to_doc = app.data_db[c_name].find_one( {id_field: from_doc[id_field]}) if not to_doc: print("missing {}".format(get_item_slug(from_doc))) continue if from_doc['Slug'] != to_doc['Slug']: try: todb[c_name].update_one( {'_id': to_doc['_id']},
def update_doc(collection, document): # update place items with geojson if collection.name == 'places': document['geometry'] = get_place_geo(document) # family trees get special treatment if collection.name == 'persons': tree_num = document['tree_num'] id = document['id'] tree_key = 'tree_vers_'+str(tree_num) query = {'tree_num': tree_num, 'id': id} tree_vers = current_app.redis.get(tree_key) if tree_vers: tree_vers = json.loads(tree_vers) i = find_version(tree_vers, document['tree_file_id']) else: tree = current_app.data_db['trees'].find_one({'num':tree_num}) if tree: tree_vers = tree['versions'] current_app.redis.set(tree_key, json.dumps(tree_vers), 300) i = find_version(tree_vers, document['tree_file_id']) else: current_app.logger.info("didn't find tree number {} using version 0 for {}" .format(tree_num, id)) i = 0; document['tree_version'] = i query['tree_version'] = i document['Slug'] = {'En': 'person_{};{}.{}'.format( tree_num, i, id)} update_collection(collection, query, document) current_app.logger.info('Updated person: {}.{}' .format(tree_num, id)) else: # post parsing: add _id and Slug doc_id_field = get_collection_id_field(collection.name) try: doc_id = document[doc_id_field] except KeyError: current_app.logger.error('update failed because of id {} {}' .format(collection.name, doc_id_field, )) if doc_id: document['_id'] = doc_id query = {doc_id_field: doc_id} result = update_collection(collection, query, document) update_es(collection.name, document, doc_id) try: slug = document['Slug']['En'] except KeyError: slug = 'None' current_app.logger.info('Updated {} {}: {}, Slug: {}'.format( collection.name, doc_id_field, doc_id, slug))
def reslugify(collection, document): ''' append the document id to the slug to ensure uniquness ''' for lang, val in document['Slug'].items(): if val: doc_id = get_collection_id_field(collection.name) document['Slug'][lang] += '-' + str(document[doc_id])