def __init__(self, timeout: int = 10, selenium_debug: bool = False, headless: bool = False): """Creates new bot instance that interacts with gg.pl :Args: - timeout (int): number of seconds before timing out - selenium_debug (bool): whether the webdriver should output debug info - headless (bool): whether the webdriver should start in headless mode """ options = Options() if selenium_debug == False: seleniumLogger.setLevel(logging.WARNING) urllibLogger.setLevel(logging.WARNING) options.add_argument("--log-level=3") logging.debug("Running in debug mode") elif selenium_debug == True: seleniumLogger.setLevel(logging.DEBUG) urllibLogger.setLevel(logging.DEBUG) logging.debug("Running in selenium debug mode") if headless == False: options.add_argument("--headless") self.driver = webdriver.Chrome(options=options) self.wait = WebDriverWait(self.driver, timeout) logging.info("Succesfully initialized driver")
def main(): config.read('smule_properties.ini') LOGGER.setLevel(logging.WARNING) urllibLogger.setLevel(logging.WARNING) parser = argparse.ArgumentParser() parser.add_argument('--batch', type=str, help="batch of users to pick - morning or night") args = parser.parse_args() batch = args.batch t = time.localtime() current_time = time.strftime("%H:%M:%S", t) date_time = datetime.now().strftime("%m-%d-%Y-%H-%M-%S") log_name = "logs\\" + str(batch) + '-' + date_time + '.log' formatter = '%(asctime)s - %(name)s:%(lineno)d - %(levelname)s - %(message)s' logging.basicConfig(filename=log_name, level=logging.DEBUG, format=formatter, filemode='w+') logger.debug("Start Time- " + str(current_time)) user_lists = [] batch = 'night' if batch == 'morning': logger.info('assigning morning batch') user_lists = json.loads(config['DEFAULT']['MorningBatch']) elif batch == 'night': logger.info('assigning night batch') user_lists = json.loads(config['DEFAULT']['NightBatch']) logger.debug("user lists" + str(user_lists)) if not user_lists: logger.info('quitting as user list is empty') driver.quit() quit() logger.info('Signing in') site_login() for user in user_lists: logger.info('current user ' + str(user)) t = time.localtime() current_time = time.strftime("%H:%M:%S", t) logger.debug("current time " + current_time) if current_time.startswith('01') or current_time.startswith('10:15'): logger.info("times up - current time is " + current_time) break try: like_collabs(user) except Exception as e: logger.exception("like collab failed for user " + str(user), exc_info=True) logger.error(traceback.format_exc()) t = time.localtime() current_time = time.strftime("%H:%M:%S", t) logger.debug("End Time- " + current_time) driver.quit()
import logging import os.path from selenium import webdriver from selenium.webdriver.chrome import options from selenium.webdriver.remote.remote_connection import LOGGER LOGGER.setLevel(logging.WARNING) from urllib3.connectionpool import log as urllibLogger urllibLogger.setLevel(logging.WARNING) logging.basicConfig( level=logging.DEBUG, format="%(levelname)s:%(asctime)s:%(module)s:%(lineno)d %(message)s") class DVTTVedicConverter(object): def set_browser(self, debugger_address=None): opts = options.Options() opts.headless = True if debugger_address is not None: opts.add_experimental_option("debuggerAddress", debugger_address) self.browser = webdriver.Chrome(options=opts) def __init__(self, debugger_address=None): # We presume that you've installed chrome driver as per https://splinter.readthedocs.io/en/latest/drivers/chrome.html . self.set_browser(debugger_address=debugger_address) self.browser.get( 'file://' + os.path.join(os.path.dirname(__file__), "data", 'DV-TTVedicNormal ==_ यूनिकोड परिवर्तित्र.html'))
from selenium.webdriver.remote.remote_connection import LOGGER from urllib3.connectionpool import log as urllibLogger import concurrent.futures class Card: def __init__(self, question, result, resource, similarity): self.question = question self.result = result self.resource = resource self.similarity = similarity text = "Which area is most likely to contain brackish water?" LOGGER.setLevel(logging.NOTSET) urllibLogger.setLevel(logging.NOTSET) _sentinel = object() def search_quizlet_deck(link, text, data_queue): chrome_options = Options() chrome_options.add_argument('--headless') chrome_options.add_argument('--disable-gpu') chrome_options.add_argument('--ignore-certificate-errors') chrome_options.add_argument('--ignore-ssl-errors') driver = webdriver.Chrome(options=chrome_options) driver.get(link) soup = BeautifulSoup(driver.page_source, features="html.parser") cards = soup.find_all('div', class_="SetPageTerms-term") tidy = [] for card in cards:
except ImportError as e: HAS_PEXPECT = False if os.environ.get('RD_JOB_LOGLEVEL') == 'DEBUG': log_level = 'DEBUG' else: log_level = 'INFO' ##end console = logging.StreamHandler() console.setFormatter(ColoredFormatter(colored_formatter.format())) console.stream=sys.stdout log = logging.getLogger() log.addHandler(console) log.setLevel(log_level) parser = argparse.ArgumentParser(description='Run Bolt command.') parser.add_argument('hostname', help='the hostname') args = parser.parse_args() password=None authentication = "basic" transport = "http" port = "5985" nossl=False diabletls12=False debug=False shell = "cmd" certpath = None krb5config = None
from pageobjects import Cluster from pageobjects import Variables from pageobjects import Plan from pageobjects import Deploy from terraformCmd import TerraformCmd from selenium.webdriver.remote.remote_connection import LOGGER as seleniumLogger from urllib3.connectionpool import log as urllibLogger import logging import uuid import pytest import sys import time import pickle seleniumLogger.setLevel(logging.INFO) urllibLogger.setLevel(logging.INFO) @pytest.fixture def prepare_env(cmdopt, logger, ssh_key_file): logger.info("Prepare for AKS testing") variables_values = { 'subscription_id': os.environ.get('ARM_SUBSCRIPTION_ID'), 'location': os.environ.get('ARM_TEST_LOCATION'), 'client_id': os.environ.get('ARM_CLIENT_ID'), 'client_secret': os.environ.get('ARM_CLIENT_SECRET'), 'tenant_id': os.environ.get('ARM_TENANT_ID'), 'ssh_username': '******', 'dns_zone_name': 'anton.bear454.codes', 'email': '*****@*****.**', 'dns_zone_resource_group': 'AKS-testing-for-persistent-dns-zone',
import os import json import base64 import logging import shutil from selenium import webdriver from selenium.webdriver.remote.remote_connection import LOGGER LOGGER.setLevel(logging.WARNING) from urllib3.connectionpool import log log.setLevel(logging.WARNING) from selenium.webdriver.chrome.options import Options from selenium.common.exceptions import TimeoutException from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support.expected_conditions import staleness_of from webdriver_manager.chrome import ChromeDriverManager from .compressor import __compress def convert(driver_path: str, source: str, target: str, timeout: int = 2, compress: bool = False, power: int = 0, install_driver: bool = False): """ Convert a given html file or website into PDF
class MainSpider(scrapy.Spider, db.MydbOperator): LOGGER.setLevel(logging.WARNING) urllibLogger.setLevel(logging.WARNING) SITE_NAME = "ZhiLian" name = 'Zhilian Spider' def __init__(self, table_name='', webhook_url='', **kwargs): dispatcher.connect(self.spider_closed, signals.spider_closed) self.start_urls = [ f'https://fe-api.zhaopin.com/c/i/sou?pageSize=200&cityId=664&workExperience=-1&education=5&companyType=-1&employmentType=-1&jobWelfareTag=-1&kw=python&kt=3' ] # self.start_urls = url self.mydb = db.MydbOperator(table_name) option = ChromeOptions() tmp_path = './tem_path' prefs = { 'profile.default_content_settings.popups': 0, 'download.default_directory': tmp_path, "profile.default_content_setting_values.automatic_downloads": 1 } # 允许多个文件下载 option.add_experimental_option('prefs', prefs) option.add_experimental_option('excludeSwitches', ['enable-automation']) self.driver = webdriver.Chrome( options=option, executable_path="../bin/driver/chromedriver.exe") # self.driver = webdriver_chrome.gen_browser('../bin/driver/chromedriver.exe') self.webhook_service = webhook.WebHook(webhook_url) self.mydb.create_table() self.isInitialize = self.mydb.is_empty_table() self.page_limit = 5 # script = 'Object.defineProperty(navigator,"webdriver",{get:() => false,});' # # # 运行Javascript # # self.driver.execute_script(script) self.driver.execute_cdp_cmd( "Page.addScriptToEvaluateOnNewDocument", { "source": """ Object.defineProperty(navigator, 'webdriver', { get: () => undefined }) """ }) super().__init__(**kwargs) def parse(self, response): print("#########################") print(response.url) self.driver.get(response.url) element = self.driver.find_element_by_css_selector( 'body > div.a-modal.risk-warning > div > div > button') element.click() time.sleep(50) print(response.url) try: WebDriverWait(self.driver, 10).until( EC.presence_of_element_located( (By.CLASS_NAME, "contentpile__content__wrapper clearfix"))) time.sleep(50) container_elements = self.driver.find_elements_by_xpath( "//div[@class='contentpile__content__wrapper__item clearfix']") for container_element in container_elements: job_title = container_element.find_element_by_xpath( "//span[@class='contentpile__content__wrapper__item__info__box__jobname__title']//@title" ) print(job_title) location = container_element.find_element_by_xpath( "//li[1][@class='contentpile__content__wrapper__item__info__box__job__demand__item']" ) print(location) company_name = container_element.find_element_by_xpath( "//a[@class='contentpile__content__wrapper__item__info__box__cname__title company_title']//@title" ) company_url = container_element.find_element_by_xpath( "//a[@class='contentpile__content__wrapper__item__info__box__cname__title company_title']//@href" ) company_in_db = self.mydb.getByCompanyName(company_name) if company_in_db is None: company_obj = company.company(job_title, company_name, company_url, location) self.email_content = self.email_content + company_name + " " + company_url + '\r\n' self.mydb.save_company(company_obj) else: logging.info("Found existing record, hence quite.") raise CloseSpider("There's no new record yet.") except: print("进入网站失败!") def spider_closed(self, spider): self.mydb.close() self.driver.close()
from selenium.webdriver.common.action_chains import ActionChains from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as ec from selenium.webdriver.support.wait import WebDriverWait from selenium.webdriver.remote.remote_connection import LOGGER as selenium_logger from urllib3.connectionpool import log as urllib_logger from logging import WARNING as logging_WARNING options = Options() options.add_experimental_option("excludeSwitches", ["enable-automation", "enable-logging"]) options.add_experimental_option("useAutomationExtension", False) # CHROME ONLY option to prevent Restore Session popup options.add_argument("--disable-session-crashed-bubble") selenium_logger.setLevel(logging_WARNING) urllib_logger.setLevel(logging_WARNING) class AnyEc: """Use with WebDriverWait to combine expected_conditions in an OR. """ def __init__(self, *args): self.ecs = args def __call__(self, driver): for fn in self.ecs: try: if fn(driver): return True except:
def main(): """ Main Startet das ganze Programm """ # --- Setup Chrome Optionen --- options = webdriver.ChromeOptions() # Browser wird nach beenden des Programmes nicht beendet options.add_experimental_option("detach", True) # Falsche Warnemldung unterdruecken (Bluetooth error) options.add_experimental_option("excludeSwitches", ["enable-logging"]) # --- Setup Argumente --- parser = init_argument_parser() arguments = parser.parse_args() if not arguments.NOTuserinteractive: arguments = parser.parse_args(get_arguments_from_user()) code = arguments.code plz = arguments.plz bundesland = arguments.bundesland driver_path = arguments.driver minimized = arguments.minimized sound = arguments.soundpath pushsafer_code = arguments.pushsaferCode wait = arguments.wait zyklus = arguments.zyklus debug = arguments.debug url = "https://www.impfterminservice.de/impftermine" # --- Setup Logger --- seleniumLogger.setLevel(logging.WARNING) urllibLogger.setLevel(logging.WARNING) file_handler = logging.FileHandler(f"debug-{plz}.log") file_handler.setLevel(logging.DEBUG) stream_handler = logging.StreamHandler(sys.stdout) stream_handler.setLevel(logging.INFO) logging.basicConfig( format='%(asctime)s\t%(levelname)s\t%(lineno)d: %(message)s', handlers=[file_handler, stream_handler], level=logging.DEBUG) # --- Start Nachricht --- logging.info("=== START ===") logging.info("Code: %s", code) logging.info("PLZ: %s", plz) logging.info("Bundesland: %s", bundesland) logging.info("Treiber: %s", driver_path) logging.info("Minimiert: %s", minimized) logging.info("Sound Pfad: %s", sound) logging.info("Warte auf Seite: %s sek.", wait) logging.info("Zyklus: %s sek.", zyklus) logging.info("url: %s", url) logging.info("debug: %s", debug) logging.info("=============") while True: # --- Erstelle Driver --- driver = webdriver.Chrome(driver_path, options=options) if minimized: logging.debug("Minimize Funktion ausser Betrieb") # driver.minimize_window() # --- Starte Aufrufe der Webseiten try: # Startseite driver.get(url) print_countdown(wait, "Warte auf Seite... ") # Impfzentrum waehlen impfzentrum_waehlen(bundesland, plz, driver) print_countdown(wait, "Warte auf Seite... ") # evtl. Warteraum skippen check_queue(driver) print_countdown(wait, "Warte auf Seite... ") # Code eingeben vermittlungscode_eingeben(code, driver, wait) print_countdown(wait, "Warte auf Seite... ") # Termin suchen termin_suchen(driver) print_countdown(wait + 15, "Warte auf Seite... ") if driver.page_source.find("Fehler") != -1: raise Exception # --- Fehlerbehandlung wenn Element nicht gefunden worden ist --- except NoSuchElementException as error: if driver.page_source.find("Warteraum") != -1: logging.warning( "Seite befindet sich im Warteraum - Pause von %s sek wird eingelegt", zyklus) else: if debug: create_screenshot(driver, "debug_NoSuchElementException") logging.warning( "Element zum klicken konnte nicht gefunden werden, bitte prüfen - Pause von %s sek wird eingelegt", zyklus) logging.debug("Error message: ", exc_info=error) driver.quit() print_countdown(zyklus) # --- Neustart wenn das Programm zu viele Anfragen hat --- except StuckedException as error: logging.info( "Programm wird aufgrund von vielen Fehlveruchen neu gestartet - evtl. Zyklus hochsetzten" ) logging.info( "Längere Pause von 5min wird eingelegt um das Problem zu beheben" ) driver.quit() print_countdown(60 * 5) # --- Fehler werden von WebDriverWait erzeugt --- except (ElementClickInterceptedException, TimeoutException) as error: if debug: create_screenshot(driver, "debug_ElementClickInterceptedException") logging.info("Element zum klicken konnte nicht gefunden werden") logging.debug("Fehler: ", exc_info=error) driver.quit() print_countdown(zyklus) except KeyboardInterrupt as error: logging.info("Programm beenden", exc_info=error) driver.quit() sys.exit(0) # --- Beenden wenn es ein unbekannter Fehler ist --- except Exception as error: if debug: create_screenshot(driver, "debug_allgemeine_Exception") logging.critical("Unerwarteter Fehler!", exc_info=error) driver.quit() print_countdown(60 * 5, "Starte neu in...") # --- Seiten konnten erfolgreich aufgerufen werden. Checken ob Termin verfügbar --- else: if driver.page_source.find("keine Termine") == -1: if debug: create_screenshot(driver, "debug_termin_gefunden") logging.info("Termin gefunden! - %s", plz) if pushsafer_code: send_push_nachricht(f"Termine verfügbar!!! - {plz}", pushsafer_code) logging.info("Pushbenachrichtigung gesendet") play_sound(sound) logging.info("Sound abgespielt") input("Zum Beenden Enter drücken...") sys.exit(0) else: driver.close() logging.info( "Kein Termin gefunden - Browser geschlossen - erneuter Versuch in %s Sekunden", zyklus) print_countdown(zyklus)
def main(ctx, driver, profile, headless, delay, variance, wait, testrun, sound): """A bot that purchases items online, rendering them bought. By default, bought uses the parameters specified in config.ini for configuration but these can also be specified via CLI. Made with <3 by jsonV """ ctx.ensure_object(dict) ctx.obj["main"] = ctx.params drive = None fp = None no_head = False eager = False invoke_newegg = False wait = 0 logging.basicConfig( filemode="a", filename="bought.log", format="[%(asctime)s] [%(levelname)s] %(message)s", ) log = logging.getLogger("bought") log.setLevel(logging.DEBUG) selenium_logger.setLevel(logging.WARNING) urllibLogger.setLevel(logging.WARNING) sh = logging.StreamHandler() sh.setLevel(logging.DEBUG) formatter = logging.Formatter("[%(asctime)s] [%(levelname)s] %(message)s", "%Y-%m-%d %H:%M:%S") sh.setFormatter(formatter) log.addHandler(sh) latest, current = check() if latest > current: log.warn( f"You are running bought v{current}. Please upgrade to v{latest} with: 'pip install --upgrade bought'" ) # Override defaults if config.ini variable specified if ctx.config: driver = ctx.config["Main"]["Driver"].lower() if ctx.config["Main"]["Headless"].lower() == "true": no_head = True if ctx.config["Newegg"]["Enabled"].lower() == "true": items = ctx.config["Newegg"]["Items"] invoke_newegg = True if ctx.config["Main"]["ImplicitWait"]: wait = int(ctx.config["Main"]["ImplicitWait"]) # Override if CLI argument specified if driver: drive = driver if headless: no_head = headless if drive == "firefox": log.info(f"Using FireFox webdriver.") if ctx.config: if os.path.isdir(ctx.config["Main"]["BrowserProfile"]): log.info( f'Logging profile: {ctx.config["Main"]["BrowserProfile"]}') fp = webdriver.FirefoxProfile( ctx.config["Main"]["BrowserProfile"]) log.info("Finished loading profile.") else: log.info("Using default blank profile.") options = webdriver.FirefoxOptions() elif drive == "chrome" or drive == "chromium": log.info(f"Using Chrome webdriver.") options = webdriver.ChromeOptions() if ctx.config: if ctx.config["Main"]["BrowserProfile"]: log.info( f'Logging profile: {ctx.config["Main"]["BrowserProfile"]}') options.add_argument( f'--user-data-dir={ctx.config["Main"]["BrowserProfile"]}') else: log.info("Using default blank profile.") if no_head: options.set_headless() if eager: options.page_load_strategy = "eager" if drive == "firefox": log.info("Starting webdriver.") driver = webdriver.Firefox(fp, firefox_options=options) log.info("Successfully created webdriver.") elif drive == "chrome" or drive == "chromium": log.info("Starting webdriver.") driver = webdriver.Chrome(chrome_options=options) log.info("Successfully created webdriver.") else: log.warn("No webdriver specified via config.ini or CLI!") log.warn( "Ensure config.ini specifies [Main][Driver] or CLI specifies driver via `bought -d firefox|chrome|chromium!" ) if wait: log.info("Setting implicit wait time.") driver.implicitly_wait(wait) if drive: ctx.obj["driver"] = driver if invoke_newegg: log.info("Starting Newegg script.") ctx.obj["config"] = ctx.config ctx.invoke(newegg, items=items)
def set_level_logging(level): urllibLogger.setLevel(level) LOGGER.setLevel(level)