def __init__(self,
timeout: int = 10,
selenium_debug: bool = False,
headless: bool = False):
"""Creates new bot instance that interacts with gg.pl
:Args:
- timeout (int): number of seconds before timing out
- selenium_debug (bool): whether the webdriver should output debug info
- headless (bool): whether the webdriver should start in headless mode
"""
options = Options()
if selenium_debug == False:
seleniumLogger.setLevel(logging.WARNING)
urllibLogger.setLevel(logging.WARNING)
options.add_argument("--log-level=3")
logging.debug("Running in debug mode")
elif selenium_debug == True:
seleniumLogger.setLevel(logging.DEBUG)
urllibLogger.setLevel(logging.DEBUG)
logging.debug("Running in selenium debug mode")
if headless == False:
options.add_argument("--headless")
self.driver = webdriver.Chrome(options=options)
self.wait = WebDriverWait(self.driver, timeout)
logging.info("Succesfully initialized driver")
def main():
config.read('smule_properties.ini')
LOGGER.setLevel(logging.WARNING)
urllibLogger.setLevel(logging.WARNING)
parser = argparse.ArgumentParser()
parser.add_argument('--batch',
type=str,
help="batch of users to pick - morning or night")
args = parser.parse_args()
batch = args.batch
t = time.localtime()
current_time = time.strftime("%H:%M:%S", t)
date_time = datetime.now().strftime("%m-%d-%Y-%H-%M-%S")
log_name = "logs\\" + str(batch) + '-' + date_time + '.log'
formatter = '%(asctime)s - %(name)s:%(lineno)d - %(levelname)s - %(message)s'
logging.basicConfig(filename=log_name,
level=logging.DEBUG,
format=formatter,
filemode='w+')
logger.debug("Start Time- " + str(current_time))
user_lists = []
batch = 'night'
if batch == 'morning':
logger.info('assigning morning batch')
user_lists = json.loads(config['DEFAULT']['MorningBatch'])
elif batch == 'night':
logger.info('assigning night batch')
user_lists = json.loads(config['DEFAULT']['NightBatch'])
logger.debug("user lists" + str(user_lists))
if not user_lists:
logger.info('quitting as user list is empty')
driver.quit()
quit()
logger.info('Signing in')
site_login()
for user in user_lists:
logger.info('current user ' + str(user))
t = time.localtime()
current_time = time.strftime("%H:%M:%S", t)
logger.debug("current time " + current_time)
if current_time.startswith('01') or current_time.startswith('10:15'):
logger.info("times up - current time is " + current_time)
break
try:
like_collabs(user)
except Exception as e:
logger.exception("like collab failed for user " + str(user),
exc_info=True)
logger.error(traceback.format_exc())
t = time.localtime()
current_time = time.strftime("%H:%M:%S", t)
logger.debug("End Time- " + current_time)
driver.quit()
import logging
import os.path
from selenium import webdriver
from selenium.webdriver.chrome import options
from selenium.webdriver.remote.remote_connection import LOGGER
LOGGER.setLevel(logging.WARNING)
from urllib3.connectionpool import log as urllibLogger
urllibLogger.setLevel(logging.WARNING)
logging.basicConfig(
level=logging.DEBUG,
format="%(levelname)s:%(asctime)s:%(module)s:%(lineno)d %(message)s")
class DVTTVedicConverter(object):
def set_browser(self, debugger_address=None):
opts = options.Options()
opts.headless = True
if debugger_address is not None:
opts.add_experimental_option("debuggerAddress", debugger_address)
self.browser = webdriver.Chrome(options=opts)
def __init__(self, debugger_address=None):
# We presume that you've installed chrome driver as per https://splinter.readthedocs.io/en/latest/drivers/chrome.html .
self.set_browser(debugger_address=debugger_address)
self.browser.get(
'file://' +
os.path.join(os.path.dirname(__file__), "data",
'DV-TTVedicNormal ==_ यूनिकोड परिवर्तित्र.html'))
from selenium.webdriver.remote.remote_connection import LOGGER
from urllib3.connectionpool import log as urllibLogger
import concurrent.futures
class Card:
def __init__(self, question, result, resource, similarity):
self.question = question
self.result = result
self.resource = resource
self.similarity = similarity
text = "Which area is most likely to contain brackish water?"
LOGGER.setLevel(logging.NOTSET)
urllibLogger.setLevel(logging.NOTSET)
_sentinel = object()
def search_quizlet_deck(link, text, data_queue):
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
chrome_options.add_argument('--ignore-certificate-errors')
chrome_options.add_argument('--ignore-ssl-errors')
driver = webdriver.Chrome(options=chrome_options)
driver.get(link)
soup = BeautifulSoup(driver.page_source, features="html.parser")
cards = soup.find_all('div', class_="SetPageTerms-term")
tidy = []
for card in cards:
except ImportError as e:
HAS_PEXPECT = False
if os.environ.get('RD_JOB_LOGLEVEL') == 'DEBUG':
log_level = 'DEBUG'
else:
log_level = 'INFO'
##end
console = logging.StreamHandler()
console.setFormatter(ColoredFormatter(colored_formatter.format()))
console.stream=sys.stdout
log = logging.getLogger()
log.addHandler(console)
log.setLevel(log_level)
parser = argparse.ArgumentParser(description='Run Bolt command.')
parser.add_argument('hostname', help='the hostname')
args = parser.parse_args()
password=None
authentication = "basic"
transport = "http"
port = "5985"
nossl=False
diabletls12=False
debug=False
shell = "cmd"
certpath = None
krb5config = None
from pageobjects import Cluster
from pageobjects import Variables
from pageobjects import Plan
from pageobjects import Deploy
from terraformCmd import TerraformCmd
from selenium.webdriver.remote.remote_connection import LOGGER as seleniumLogger
from urllib3.connectionpool import log as urllibLogger
import logging
import uuid
import pytest
import sys
import time
import pickle
seleniumLogger.setLevel(logging.INFO)
urllibLogger.setLevel(logging.INFO)
@pytest.fixture
def prepare_env(cmdopt, logger, ssh_key_file):
logger.info("Prepare for AKS testing")
variables_values = {
'subscription_id': os.environ.get('ARM_SUBSCRIPTION_ID'),
'location': os.environ.get('ARM_TEST_LOCATION'),
'client_id': os.environ.get('ARM_CLIENT_ID'),
'client_secret': os.environ.get('ARM_CLIENT_SECRET'),
'tenant_id': os.environ.get('ARM_TENANT_ID'),
'ssh_username': '******',
'dns_zone_name': 'anton.bear454.codes',
'email': '*****@*****.**',
'dns_zone_resource_group': 'AKS-testing-for-persistent-dns-zone',
import os
import json
import base64
import logging
import shutil
from selenium import webdriver
from selenium.webdriver.remote.remote_connection import LOGGER
LOGGER.setLevel(logging.WARNING)
from urllib3.connectionpool import log
log.setLevel(logging.WARNING)
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.expected_conditions import staleness_of
from webdriver_manager.chrome import ChromeDriverManager
from .compressor import __compress
def convert(driver_path: str,
source: str,
target: str,
timeout: int = 2,
compress: bool = False,
power: int = 0,
install_driver: bool = False):
"""
Convert a given html file or website into PDF
class MainSpider(scrapy.Spider, db.MydbOperator):
LOGGER.setLevel(logging.WARNING)
urllibLogger.setLevel(logging.WARNING)
SITE_NAME = "ZhiLian"
name = 'Zhilian Spider'
def __init__(self, table_name='', webhook_url='', **kwargs):
dispatcher.connect(self.spider_closed, signals.spider_closed)
self.start_urls = [
f'https://fe-api.zhaopin.com/c/i/sou?pageSize=200&cityId=664&workExperience=-1&education=5&companyType=-1&employmentType=-1&jobWelfareTag=-1&kw=python&kt=3'
]
# self.start_urls = url
self.mydb = db.MydbOperator(table_name)
option = ChromeOptions()
tmp_path = './tem_path'
prefs = {
'profile.default_content_settings.popups': 0,
'download.default_directory': tmp_path,
"profile.default_content_setting_values.automatic_downloads": 1
} # 允许多个文件下载
option.add_experimental_option('prefs', prefs)
option.add_experimental_option('excludeSwitches',
['enable-automation'])
self.driver = webdriver.Chrome(
options=option, executable_path="../bin/driver/chromedriver.exe")
# self.driver = webdriver_chrome.gen_browser('../bin/driver/chromedriver.exe')
self.webhook_service = webhook.WebHook(webhook_url)
self.mydb.create_table()
self.isInitialize = self.mydb.is_empty_table()
self.page_limit = 5
# script = 'Object.defineProperty(navigator,"webdriver",{get:() => false,});'
# # # 运行Javascript
# # self.driver.execute_script(script)
self.driver.execute_cdp_cmd(
"Page.addScriptToEvaluateOnNewDocument", {
"source":
"""
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined
})
"""
})
super().__init__(**kwargs)
def parse(self, response):
print("#########################")
print(response.url)
self.driver.get(response.url)
element = self.driver.find_element_by_css_selector(
'body > div.a-modal.risk-warning > div > div > button')
element.click()
time.sleep(50)
print(response.url)
try:
WebDriverWait(self.driver, 10).until(
EC.presence_of_element_located(
(By.CLASS_NAME, "contentpile__content__wrapper clearfix")))
time.sleep(50)
container_elements = self.driver.find_elements_by_xpath(
"//div[@class='contentpile__content__wrapper__item clearfix']")
for container_element in container_elements:
job_title = container_element.find_element_by_xpath(
"//span[@class='contentpile__content__wrapper__item__info__box__jobname__title']//@title"
)
print(job_title)
location = container_element.find_element_by_xpath(
"//li[1][@class='contentpile__content__wrapper__item__info__box__job__demand__item']"
)
print(location)
company_name = container_element.find_element_by_xpath(
"//a[@class='contentpile__content__wrapper__item__info__box__cname__title company_title']//@title"
)
company_url = container_element.find_element_by_xpath(
"//a[@class='contentpile__content__wrapper__item__info__box__cname__title company_title']//@href"
)
company_in_db = self.mydb.getByCompanyName(company_name)
if company_in_db is None:
company_obj = company.company(job_title, company_name,
company_url, location)
self.email_content = self.email_content + company_name + " " + company_url + '\r\n'
self.mydb.save_company(company_obj)
else:
logging.info("Found existing record, hence quite.")
raise CloseSpider("There's no new record yet.")
except:
print("进入网站失败!")
def spider_closed(self, spider):
self.mydb.close()
self.driver.close()
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as ec
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.remote.remote_connection import LOGGER as selenium_logger
from urllib3.connectionpool import log as urllib_logger
from logging import WARNING as logging_WARNING
options = Options()
options.add_experimental_option("excludeSwitches",
["enable-automation", "enable-logging"])
options.add_experimental_option("useAutomationExtension", False)
# CHROME ONLY option to prevent Restore Session popup
options.add_argument("--disable-session-crashed-bubble")
selenium_logger.setLevel(logging_WARNING)
urllib_logger.setLevel(logging_WARNING)
class AnyEc:
"""Use with WebDriverWait to combine expected_conditions
in an OR.
"""
def __init__(self, *args):
self.ecs = args
def __call__(self, driver):
for fn in self.ecs:
try:
if fn(driver):
return True
except:
def main():
"""
Main
Startet das ganze Programm
"""
# --- Setup Chrome Optionen ---
options = webdriver.ChromeOptions()
# Browser wird nach beenden des Programmes nicht beendet
options.add_experimental_option("detach", True)
# Falsche Warnemldung unterdruecken (Bluetooth error)
options.add_experimental_option("excludeSwitches", ["enable-logging"])
# --- Setup Argumente ---
parser = init_argument_parser()
arguments = parser.parse_args()
if not arguments.NOTuserinteractive:
arguments = parser.parse_args(get_arguments_from_user())
code = arguments.code
plz = arguments.plz
bundesland = arguments.bundesland
driver_path = arguments.driver
minimized = arguments.minimized
sound = arguments.soundpath
pushsafer_code = arguments.pushsaferCode
wait = arguments.wait
zyklus = arguments.zyklus
debug = arguments.debug
url = "https://www.impfterminservice.de/impftermine"
# --- Setup Logger ---
seleniumLogger.setLevel(logging.WARNING)
urllibLogger.setLevel(logging.WARNING)
file_handler = logging.FileHandler(f"debug-{plz}.log")
file_handler.setLevel(logging.DEBUG)
stream_handler = logging.StreamHandler(sys.stdout)
stream_handler.setLevel(logging.INFO)
logging.basicConfig(
format='%(asctime)s\t%(levelname)s\t%(lineno)d: %(message)s',
handlers=[file_handler, stream_handler],
level=logging.DEBUG)
# --- Start Nachricht ---
logging.info("=== START ===")
logging.info("Code: %s", code)
logging.info("PLZ: %s", plz)
logging.info("Bundesland: %s", bundesland)
logging.info("Treiber: %s", driver_path)
logging.info("Minimiert: %s", minimized)
logging.info("Sound Pfad: %s", sound)
logging.info("Warte auf Seite: %s sek.", wait)
logging.info("Zyklus: %s sek.", zyklus)
logging.info("url: %s", url)
logging.info("debug: %s", debug)
logging.info("=============")
while True:
# --- Erstelle Driver ---
driver = webdriver.Chrome(driver_path, options=options)
if minimized:
logging.debug("Minimize Funktion ausser Betrieb")
# driver.minimize_window()
# --- Starte Aufrufe der Webseiten
try:
# Startseite
driver.get(url)
print_countdown(wait, "Warte auf Seite... ")
# Impfzentrum waehlen
impfzentrum_waehlen(bundesland, plz, driver)
print_countdown(wait, "Warte auf Seite... ")
# evtl. Warteraum skippen
check_queue(driver)
print_countdown(wait, "Warte auf Seite... ")
# Code eingeben
vermittlungscode_eingeben(code, driver, wait)
print_countdown(wait, "Warte auf Seite... ")
# Termin suchen
termin_suchen(driver)
print_countdown(wait + 15, "Warte auf Seite... ")
if driver.page_source.find("Fehler") != -1:
raise Exception
# --- Fehlerbehandlung wenn Element nicht gefunden worden ist ---
except NoSuchElementException as error:
if driver.page_source.find("Warteraum") != -1:
logging.warning(
"Seite befindet sich im Warteraum - Pause von %s sek wird eingelegt",
zyklus)
else:
if debug:
create_screenshot(driver, "debug_NoSuchElementException")
logging.warning(
"Element zum klicken konnte nicht gefunden werden, bitte prüfen - Pause von %s sek wird eingelegt",
zyklus)
logging.debug("Error message: ", exc_info=error)
driver.quit()
print_countdown(zyklus)
# --- Neustart wenn das Programm zu viele Anfragen hat ---
except StuckedException as error:
logging.info(
"Programm wird aufgrund von vielen Fehlveruchen neu gestartet - evtl. Zyklus hochsetzten"
)
logging.info(
"Längere Pause von 5min wird eingelegt um das Problem zu beheben"
)
driver.quit()
print_countdown(60 * 5)
# --- Fehler werden von WebDriverWait erzeugt ---
except (ElementClickInterceptedException, TimeoutException) as error:
if debug:
create_screenshot(driver,
"debug_ElementClickInterceptedException")
logging.info("Element zum klicken konnte nicht gefunden werden")
logging.debug("Fehler: ", exc_info=error)
driver.quit()
print_countdown(zyklus)
except KeyboardInterrupt as error:
logging.info("Programm beenden", exc_info=error)
driver.quit()
sys.exit(0)
# --- Beenden wenn es ein unbekannter Fehler ist ---
except Exception as error:
if debug:
create_screenshot(driver, "debug_allgemeine_Exception")
logging.critical("Unerwarteter Fehler!", exc_info=error)
driver.quit()
print_countdown(60 * 5, "Starte neu in...")
# --- Seiten konnten erfolgreich aufgerufen werden. Checken ob Termin verfügbar ---
else:
if driver.page_source.find("keine Termine") == -1:
if debug:
create_screenshot(driver, "debug_termin_gefunden")
logging.info("Termin gefunden! - %s", plz)
if pushsafer_code:
send_push_nachricht(f"Termine verfügbar!!! - {plz}",
pushsafer_code)
logging.info("Pushbenachrichtigung gesendet")
play_sound(sound)
logging.info("Sound abgespielt")
input("Zum Beenden Enter drücken...")
sys.exit(0)
else:
driver.close()
logging.info(
"Kein Termin gefunden - Browser geschlossen - erneuter Versuch in %s Sekunden",
zyklus)
print_countdown(zyklus)
def main(ctx, driver, profile, headless, delay, variance, wait, testrun,
sound):
"""A bot that purchases items online, rendering them bought.
By default, bought uses the parameters specified in config.ini for
configuration but these can also be specified via CLI.
Made with <3 by jsonV
"""
ctx.ensure_object(dict)
ctx.obj["main"] = ctx.params
drive = None
fp = None
no_head = False
eager = False
invoke_newegg = False
wait = 0
logging.basicConfig(
filemode="a",
filename="bought.log",
format="[%(asctime)s] [%(levelname)s] %(message)s",
)
log = logging.getLogger("bought")
log.setLevel(logging.DEBUG)
selenium_logger.setLevel(logging.WARNING)
urllibLogger.setLevel(logging.WARNING)
sh = logging.StreamHandler()
sh.setLevel(logging.DEBUG)
formatter = logging.Formatter("[%(asctime)s] [%(levelname)s] %(message)s",
"%Y-%m-%d %H:%M:%S")
sh.setFormatter(formatter)
log.addHandler(sh)
latest, current = check()
if latest > current:
log.warn(
f"You are running bought v{current}. Please upgrade to v{latest} with: 'pip install --upgrade bought'"
)
# Override defaults if config.ini variable specified
if ctx.config:
driver = ctx.config["Main"]["Driver"].lower()
if ctx.config["Main"]["Headless"].lower() == "true":
no_head = True
if ctx.config["Newegg"]["Enabled"].lower() == "true":
items = ctx.config["Newegg"]["Items"]
invoke_newegg = True
if ctx.config["Main"]["ImplicitWait"]:
wait = int(ctx.config["Main"]["ImplicitWait"])
# Override if CLI argument specified
if driver:
drive = driver
if headless:
no_head = headless
if drive == "firefox":
log.info(f"Using FireFox webdriver.")
if ctx.config:
if os.path.isdir(ctx.config["Main"]["BrowserProfile"]):
log.info(
f'Logging profile: {ctx.config["Main"]["BrowserProfile"]}')
fp = webdriver.FirefoxProfile(
ctx.config["Main"]["BrowserProfile"])
log.info("Finished loading profile.")
else:
log.info("Using default blank profile.")
options = webdriver.FirefoxOptions()
elif drive == "chrome" or drive == "chromium":
log.info(f"Using Chrome webdriver.")
options = webdriver.ChromeOptions()
if ctx.config:
if ctx.config["Main"]["BrowserProfile"]:
log.info(
f'Logging profile: {ctx.config["Main"]["BrowserProfile"]}')
options.add_argument(
f'--user-data-dir={ctx.config["Main"]["BrowserProfile"]}')
else:
log.info("Using default blank profile.")
if no_head:
options.set_headless()
if eager:
options.page_load_strategy = "eager"
if drive == "firefox":
log.info("Starting webdriver.")
driver = webdriver.Firefox(fp, firefox_options=options)
log.info("Successfully created webdriver.")
elif drive == "chrome" or drive == "chromium":
log.info("Starting webdriver.")
driver = webdriver.Chrome(chrome_options=options)
log.info("Successfully created webdriver.")
else:
log.warn("No webdriver specified via config.ini or CLI!")
log.warn(
"Ensure config.ini specifies [Main][Driver] or CLI specifies driver via `bought -d firefox|chrome|chromium!"
)
if wait:
log.info("Setting implicit wait time.")
driver.implicitly_wait(wait)
if drive:
ctx.obj["driver"] = driver
if invoke_newegg:
log.info("Starting Newegg script.")
ctx.obj["config"] = ctx.config
ctx.invoke(newegg, items=items)
def set_level_logging(level):
urllibLogger.setLevel(level)
LOGGER.setLevel(level)
