def get_logged_in_browser(headless=True):
"""Sometimes headless browser fails with selenium.common.exceptions.ElementClickInterceptedException: Message: element click intercepted . Then, non-headless browser works fine! Or can try https://stackoverflow.com/questions/48665001/can-not-click-on-a-element-elementclickinterceptedexception-in-splinter-selen """
browser = scraping.get_selenium_browser(headless=headless)
browser.get("http://parankusan.cloudapp.net/Integrated/Texts.aspx")
username = browser.find_element_by_id("txtUserName")
username.send_keys(configuration_parankusha["user"])
browser.find_element_by_id("btnNext").click()
browser.find_element_by_id("txtPassword").send_keys(configuration_parankusha["pass"])
browser.find_element_by_id("btnLogin").click()
browser.get("http://parankusan.cloudapp.net/Integrated/Texts.aspx")
return browser
def get_logged_in_browser(headless=True):
"""Sometimes headless browser fails with selenium.common.exceptions.ElementClickInterceptedException: Message: element click intercepted . Then, non-headless browser works fine! Or can try https://stackoverflow.com/questions/48665001/can-not-click-on-a-element-elementclickinterceptedexception-in-splinter-selen """
browser = scraping.get_selenium_browser(headless=headless)
browser.get("https://vaakya.vedanidhi.in/login/")
username = browser.find_element_by_id("username")
username.send_keys(site_configuration["user"])
browser.find_element_by_id("password").send_keys(
site_configuration["pass"])
browser.find_element_by_id("submit_button").click()
browser.get("https://vaakya.vedanidhi.in/browse/?lang=En")
return browser
import os
from selenium.webdriver.remote.remote_connection import LOGGER
from curation_utils import scraping
LOGGER.setLevel(logging.WARNING)
from urllib3.connectionpool import log as urllibLogger
urllibLogger.setLevel(logging.WARNING)
logging.basicConfig(
level=logging.DEBUG,
format="%(levelname)s:%(asctime)s:%(module)s:%(lineno)d %(message)s")
browser = scraping.get_selenium_browser(headless=False)
def dump_text(url, out_path, overwrite=True):
if overwrite == False and os.path.exists(out_path):
logging.info("Not overwriting %s to %s", url, out_path)
return
logging.info("Dumping %s to %s", url, out_path)
browser.get(url)
text_elements = browser.find_elements_by_css_selector("div.sam")
if len(text_elements) == 0:
text_elements = [browser.find_elements_by_css_selector("table")[-1]]
os.makedirs(os.path.dirname(out_path), exist_ok=True)
with codecs.open(out_path, "w", 'utf-8') as file_out:
for text_element in text_elements:
text = text_element.text.replace("\n", " \n") + "\n"
import regex
import logging
from bs4 import BeautifulSoup, Tag
from indic_transliteration import sanscript
from curation_utils import scraping, file_helper
from doc_curation.md.file import MdFile
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(
level=logging.DEBUG,
format="%(levelname)s:%(asctime)s:%(module)s:%(lineno)d %(message)s")
browser = scraping.get_selenium_browser(headless=True)
def get_title(title_iast_in):
title_iast = title_iast_in.replace("Part ", "").replace(
"Volume ", "").replace("volume ", "").replace("Vol. ", "").replace(
"with ", "").replace("text ", "").replace("version ", "").replace(
"by ",
"").replace("commentary ",
"").replace("commentaries ",
"").replace("rescension ",
"").replace("ṣṃ",
"ṣm").strip()
return title_iast