def extract_segments(limit: t.Optional[int] = None, log_level: str = "INFO"): """ Takes all conference talks and scripture chapters in the pages collection of the database and extracts all the segments from them, including cleaned text, meta-data, and the references of each segment. """ n_written = 0 logger.setLevel(log_level) # First delete all segments in the collection. logger.info("deleting all documents in the segments collection...") db.segments.delete_many({}) for page_dict in db.pages.find({}): if limit is not None and n_written >= limit: break page = Page(**page_dict) if page.doc_type == "scriptures": segmentable = Chapter(page) elif page.doc_type == "general-conference": segmentable = ConferenceTalk(page) else: raise ValueError(f"unsupported doc_type '{page.doc_type}'") logger.debug(segmentable) write_segments(segmentable.to_segments()) n_written += 1
def __init__(self, page: Page) -> None: logger.info(f"processing {page._id}...") self.url = page._id self.soup = BeautifulSoup(page.html, features="lxml") attrs = parse_scripture_chapter_url(self.url) self.id: str = attrs["id"] self.volume: str = attrs["volume"] self.book_id: str = attrs["book_id"] self.ch: int = attrs["ch"] self.book_name: str = book_map[self.book_id]["names"][0] self._set_segments()
def __init__(self, page: Page) -> None: logger.info(f"processing {page._id}...") self.url = page._id self.soup = BeautifulSoup(page.html, features="lxml") attrs = parse_conference_talk_url(self.url) self.id: str = attrs["id"] self.year: int = attrs["volume"] self.month: int = attrs["work"] self.url_name: str = attrs["parent_doc"] self.name = self.soup.find(**self.NAME_QUERY).string self._set_author() self._set_segments()
def __init__(self, _id: str, doc_type: str, html: str = None) -> None: """ `doc_type` is the document type, one of `["general-conference", "scriptures"]`. """ self._id = _id self.doc_type = doc_type if html is None: logger.info(f"pulling '{_id}'...") res = requests.get(self._id) # Raise exception for 4xx or 5xx HTTP codes. res.raise_for_status() res.encoding = "utf-8" self.html = res.text else: self.html = html
def pull_pages( overwrite: bool = False, limit: t.Optional[int] = None, log_level: str = "INFO", ): """ Writes the raw HTML content of all conference talks and scriptures to the database. Parameters ---------- overwrite If `True`, all pages will be removed, and a fresh write will take place. If `False`, pages will only be written to the db for documents that are not referenced yet in the db. limit If supplied, the number of documents written to the database will not exceed `limit`. Useful for testing and debugging. log_level The level to set logging to. One of `["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]`. """ logger.setLevel(log_level) if overwrite: # Delete all HTML pages in the collection. logger.info("deleting all documents in the pages collection...") db.pages.delete_many({}) sub_limit = math.ceil(limit / 2) if limit is not None else None existing_doc_ids = set(doc["_id"] for doc in db.pages.find({}, {"_id": 1})) pull_pages_for_type( sub_limit, "general-conference", get_all_conference_talk_urls(), existing_doc_ids, ) pull_pages_for_type( sub_limit, "scriptures", get_all_chapter_urls(), existing_doc_ids )
def compute_embeddings(overwrite: bool = False): """ Computes all the embeddings for all the segments in the database """ segments_collection = db.segments embedder = TextEmbedder() logger.info("computing an embedding vector for each segment...") # load them all into memory at once so our MongoDB connection doesn't # drop part way through. segments = list(segments_collection.find()) for segment in tqdm(segments): if not overwrite and "embedding" in segment: # This segment already has an embedding and we're not replacing it. continue embedding = embedder.embed_text(segment["text"]) segments_collection.find_one_and_update( # The list is stored in the database as a raw list, not a numpy array. {"_id": segment["_id"]}, {"$set": { "embedding": embedding.tolist() }}, ) logger.info("embedding complete")
def get_segments_by_document(*, include_embeddings: bool = False ) -> t.Dict[str, dict]: """ Collects all segments for each conference talk or scripture chapter into a single document, returning all documents as items in a dictionary, mapped from their document id. """ segments_collection = db.segments documents: t.Dict[str, dict] = defaultdict(lambda: {"segments": []}) logger.info("preprocessing segments into documents...") for segment in tqdm(segments_collection.find(), total=segments_collection.count_documents(filter={})): document = documents[segment["parent_id"]] if include_embeddings: # Mongodb stores the embedding as a raw list, not a numpy array. try: segment["embedding"] = np.fromiter(segment["embedding"], np.float) except Exception as e: logger.error(segment) raise e else: segment.pop("embedding") document["segments"].append(segment) for doc_id, document in documents.items(): # Store the document's id in the document itself so the # document can be fully self-contained. document["_id"] = doc_id # Sort the segments of each document to make sure they're in order. document["segments"].sort(key=lambda s: s["num"]) return documents
def import_docs(overwrite: bool = True, log_level: str = "INFO"): """ Imports all segments from the Mongo DB into ElasticSearch. If `overwrite == True`, the ES index will first be wiped out before indexing. """ logger.setLevel(log_level) logger.info( f"importing '{SEGMENTS}' index from MongoDB to ElasticSearch...") if overwrite and es_client.indices.exists(index=SEGMENTS): logger.info("deleting all documents in the segments index...") es_client.indices.delete(index=SEGMENTS) documents = get_segments_by_document() for doc in documents.values(): # Add the index name for Elastic Search. doc["_index"] = SEGMENTS logger.info("indexing documents...") bulk(es_client, tqdm(documents.values()))