def setUpClass(cls):
super(LiveTestCase, cls).setUpClass()
options = Options()
options.headless = True
cls.selenium = WebDriver(options=options)
cls.selenium.implicitly_wait(10)
def get_firefox_headless():
# https://selenium-python.readthedocs.io/faq.html#how-to-auto-save-files-using-custom-firefox-profile
fp = webdriver.FirefoxProfile()
content_types = "text/plain,text/csv,application/csv,application/pdf"
fp.set_preference("browser.download.folderList", 2)
fp.set_preference("browser.download.manager.showWhenStarting", False)
fp.set_preference("browser.download.dir", datadir)
fp.set_preference("browser.helperApps.neverAsk.saveToDisk", content_types)
fp.set_preference("browser.helperApps.neverAsk.openFile", content_types)
fp.set_preference("plugin.scan.plid.all", False)
fp.set_preference("plugin.scan.Acrobat", "99.0")
fp.set_preference("pdfjs.disabled", True)
options = Options()
options.headless = True
# options.log.level = "trace"
driver = webdriver.Firefox(
options=options,
firefox_profile=fp,
executable_path="/usr/local/bin/geckodriver",
service_log_path=os.path.devnull,
# service_log_path="/code/firefox.log",
)
return driver
def _local_browser_class(browser_name):
"""
Returns class, kwargs, and args needed to instantiate the local browser.
"""
# Log name of local browser
LOGGER.info(u"Using local browser: %s [Default is firefox]", browser_name)
# Get class of local browser based on name
browser_class = BROWSERS.get(browser_name)
headless = os.environ.get('BOKCHOY_HEADLESS', 'false').lower() == 'true'
if browser_class is None:
raise BrowserConfigError(
u"Invalid browser name {name}. Options are: {options}".format(
name=browser_name, options=", ".join(list(BROWSERS.keys()))))
else:
if browser_name == 'firefox':
# Remove geckodriver log data from previous test cases
log_path = os.path.join(os.getcwd(), 'geckodriver.log')
if os.path.exists(log_path):
os.remove(log_path)
firefox_options = FirefoxOptions()
firefox_options.log.level = 'trace'
if headless:
firefox_options.headless = True
browser_args = []
browser_kwargs = {
'firefox_profile': _firefox_profile(),
'options': firefox_options,
}
firefox_path = os.environ.get('SELENIUM_FIREFOX_PATH')
firefox_log = os.environ.get('SELENIUM_FIREFOX_LOG')
if firefox_path and firefox_log:
browser_kwargs.update({
'firefox_binary': FirefoxBinary(
firefox_path=firefox_path, log_file=firefox_log)
})
elif firefox_path:
browser_kwargs.update({
'firefox_binary': FirefoxBinary(firefox_path=firefox_path)
})
elif firefox_log:
browser_kwargs.update({
'firefox_binary': FirefoxBinary(log_file=firefox_log)
})
elif browser_name == 'chrome':
chrome_options = ChromeOptions()
if headless:
chrome_options.headless = True
# Emulate webcam and microphone for testing purposes
chrome_options.add_argument('--use-fake-device-for-media-stream')
# Bypasses the security prompt displayed by the browser when it attempts to
# access a media device (e.g., a webcam)
chrome_options.add_argument('--use-fake-ui-for-media-stream')
browser_args = []
browser_kwargs = {
'options': chrome_options,
}
else:
browser_args, browser_kwargs = [], {}
return browser_class, browser_args, browser_kwargs
def Get_HTML(URL, mode=1, IP_proxy='', flag_return_driver=0, driver=False):
'''
Функция должна:
получать страницу по переданому URL
Пременяет один из указанных способов, режимов, :
Возможные способы, режимы, получения:
1 - Библиотека requests:
контроль заголовков
передача строки юзерагента
КУКИ
2 - Селениум
Очень медлено!
3 - Библиотека Splinter
????????????
4 - Библиотека MechanicalSoup
????????????
5 - Библиотека RoboBrowser
????????????
6 - Библиотека Mechanize
- не выполняет Javascript на страницах, которые он просматривает (проверить)
7 - Библиотека Scrapy
??????????????
Возврт значений:
если mode=1:
пользователь имеет возможность получить экземпляр браузера для дальнейшего использования
вне этой ф-ции, но тогда он отвечает за закрытие этого экземпляра браузера.
Есть возможность вернут в ф-цию экземпляра браузера
'''
# Проверка на корректность полученных данных
if type(URL) != str:
print('You must input only str')
return False
if mode == 1:
# print('You choso Selenium:')
if flag_return_driver == 0 or driver == False:
r = tkinter.Tk() # получаем объект для доступа к параметрам экрана
# выбираю версию geckodriver в зависимости от разрядности Windows
name_file_geckodriver = general_functions.choose_geckodriver_file()
pathDriver = os.path.dirname(
os.path.abspath(__file__)) + "\\" + name_file_geckodriver
opts = Options()
opts.set_preference(
"dom.webdriver.enabled",
False) # скрывает то что браузер управляеться автоматически
opts.headless = False
opts.add_argument(
'-width=' +
str(r.winfo_screenwidth() / 2)) # Устанавливаем ширину окна
opts.add_argument(
'-height=' +
str(r.winfo_screenheight() / 1.3)) # Устанавливаем высоту окна
driver = webdriver.Firefox(executable_path=pathDriver,
options=opts)
driver.set_window_position(r.winfo_screenwidth() / 2, 0)
if IP_proxy:
webdriver.DesiredCapabilities.FIREFOX['proxy'] = {
"httpProxy": IP_proxy,
"ftpProxy": IP_proxy,
"sslProxy": IP_proxy,
"proxyType": "MANUAL",
}
try:
driver.get(URL)
try:
WebDriverWait(
driver,
5).until(lambda driver: driver.find_elements_by_xpath(
"//*[.='IP адрес']"))
except Exception as errMess:
pass
# print('Элемент не найден')
# print(errMess)
html = driver.page_source
# Оброботка исключний:
except Exception as errMess:
print('Текущий URL недоступен')
html = False
# Вывод результатов в зависимости от значения flag_return_driver
if flag_return_driver and html:
arr_result = [html, driver]
return arr_result
# driver.close() # закрываю браузер
driver.quit() # закрываю браузер
return html
elif mode == 2:
print('You choso Requests lib')
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0'
}
# response = requests.get(url,headers=headers,proxies=proxies,timeout=_timeout,verify=False)
response = requests.get(URL, headers=headers)
response.encoding = 'utf-8'
html = response.text
elif mode == 3:
print('You choso Splinter')
elif mode == 4:
print('You choso MechanicalSoup')
elif mode == 5:
print('You choso RoboBrowser')
elif mode == 6:
print('You choso Mechanize')
elif mode == 7:
print('You choso Scrapy')
def main():
print("Main Executed")
#%%
site = 'https://www.wsj.com/market-data/stocks/marketsdiary'
# options = webdriver.ChromeOptions()
# options.add_argument('headless')
# options.add_argument('--disable-gpu')
# options.add_experimental_option('excludeSwitches', ['enable-logging'])
# driver=webdriver.Chrome(executable_path='E:\Trading\Quant\chromedriver.exe', options=options)
display = Display(visible=0, size=(1366, 768))
options = Options()
fire_profile = webdriver.FirefoxProfile()
fire_profile.set_preference("browser.cache.disk.enable", False)
fire_profile.set_preference("browser.cache.memory.enable", False)
fire_profile.set_preference("browser.cache.offline.enable", False)
fire_profile.set_preference("network.http.use-cache", False)
options.headless = True
display.start()
driver = webdriver.Firefox(options=options, firefox_profile=fire_profile)
driver.get(site)
delay = 30
html = driver.page_source
soup = BeautifulSoup(html, "lxml")
#table=soup.find_all('tbody')
#table=table[0]
div = soup.find('div', id='root')
df = pd.read_html(str(soup))[0]
driver.close()
driver.quit()
display.stop()
df_NYSE = df.iloc[:14, :2]
df_NYSE.columns = ['Data', 'Value']
df_NASDAQ = df.iloc[14:25, :2]
df_NASDAQ.columns = ['Data', 'Value']
#%%
#------------get data from AAII sentiment site
url = 'https://www.aaii.com/sentimentsurvey/sent_results?'
page = requests.get(url)
soup = BeautifulSoup(page.content, 'html.parser')
for tr in soup.find_all('tr')[1:2]:
tds = tr.find_all('td')
bulls = tds[1].text
neutral = tds[2].text
bears = tds[3].text
bulls = float(bulls.replace('%', ''))
neutral = float(neutral.replace('%', ''))
bears = float(bears.replace('%', ''))
AAII_NET_BULLS = bulls - bears - neutral
NYSE_UPV = df_NYSE['Value'][6] / 1000000
NYSE_DNV = df_NYSE['Value'][7] / 1000000
NYSE_ADV = df_NYSE['Value'][1]
NYSE_DEC = df_NYSE['Value'][2]
ticker = wb.DataReader('^GSPC', data_source='yahoo')
SPX = ticker[-1:]['Close'][0]
NYSE_TOTAL = df_NYSE['Value'][0]
NYSE_LOWS = 100 * df_NYSE['Value'][5] / NYSE_TOTAL
NYSE_HIGHS = 100 * df_NYSE['Value'][4] / NYSE_TOTAL
ticker = wb.DataReader('^VIX', data_source='yahoo')
VIX = ticker[-1:]['Close'][0]
today_data = {
'UPV': NYSE_UPV,
'DNV': NYSE_DNV,
'ADV': NYSE_ADV,
'DEC': NYSE_DEC,
'SPX': SPX,
'Lows': NYSE_LOWS,
'Highs': NYSE_HIGHS,
'VIX': VIX,
'AAII': AAII_NET_BULLS
}
today = pd.to_datetime('today').tz_localize('Asia/Singapore').tz_convert(
'US/Eastern')
today = today.replace(tzinfo=None,
hour=0,
minute=0,
second=0,
microsecond=0)
nyse = mcal.get_calendar('NYSE')
is_holiday = nyse.valid_days(start_date=today, end_date=today)
if len(is_holiday) > 0:
#take yesterday's CSV and add to it
df_raw_data = pd.read_csv('raw data.csv', index_col=0)
#append today's data to yesterday's
df_raw_data = df_raw_data.append(
pd.DataFrame(today_data, index=[today]))
df_raw_data = df_raw_data.reset_index()
df_raw_data = df_raw_data.rename(columns={'index': 'Date'})
#save a copy of today's and update the main csv to be added to tomorrow
df_raw_data.to_csv(str(today.date()) + ' raw data.csv', index=False)
df_raw_data.to_csv('raw data.csv', index=False)
#update SPX close
spx_price = {'Last Price': SPX}
df_spx = pd.read_csv('SPX 1930 Daily.csv', index_col=0)
df_spx = df_spx.append(pd.DataFrame(spx_price, index=[today]))
df_spx = df_spx.reset_index()
df_spx = df_spx.rename(columns={'index': 'Date'})
df_spx.to_csv('SPX 1930 Daily.csv', index=False)
print("Main Complete")
# -*- coding: utf-8 -*-
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.firefox.options import Options
import time, csv
from selenium.webdriver.common.by import By
options = Options()
options.headless = True
driver = webdriver.Firefox(options=options)
# driver = webdriver.Firefox ()
driver.implicitly_wait(10)
driver.get("https://www.ydxxt.com/")
elem_user = driver.find_element_by_name("userid")
elem_user.clear()
elem_user.send_keys("150881732061")
elem_pwd = driver.find_element_by_name("psw")
elem_pwd.send_keys("1y20050917@1")
time.sleep(5)
elem_pwd.send_keys(Keys.RETURN)
time.sleep(3)
driver.find_element_by_css_selector('input[value="进入系统"]').click()
time.sleep(3)
driver.find_element_by_css_selector("#A2").click()
driver.find_element_by_css_selector(
".WEBItc_list_con1 > ul:nth-child(1) > li:nth-child(9) > a:nth-child(1)"
).click()
time.sleep(2)
def __init__(self,
client="firefox",
username="******",
proxy=None,
command_executor=None,
loadstyles=False,
profile=None,
headless=False,
autoconnect=True,
logger=None,
extra_params=None,
chrome_options=None,
executable_path=None,
script_timeout=60,
element_timeout=30,
license_key=None):
"""Initialises the webdriver"""
self.logger = logger or self.logger
self.license_key = license_key
extra_params = extra_params or {}
if profile is not None:
self._profile_path = profile
self.logger.info("Checking for profile at %s" % self._profile_path)
if not os.path.exists(self._profile_path):
self.logger.critical("Could not find profile at %s" % profile)
raise WhatsAPIException("Could not find profile at %s" %
profile)
else:
self._profile_path = None
self.client = client.lower()
if self.client == "firefox":
if self._profile_path is not None:
self._profile = webdriver.FirefoxProfile(self._profile_path)
else:
self._profile = webdriver.FirefoxProfile()
if not loadstyles:
# Disable CSS
self._profile.set_preference('permissions.default.stylesheet',
2)
# Disable images
self._profile.set_preference('permissions.default.image', 2)
# Disable Flash
self._profile.set_preference(
'dom.ipc.plugins.enabled.libflashplayer.so', 'false')
if proxy is not None:
self.set_proxy(proxy)
options = Options()
if headless:
options.headless = True
options.profile = self._profile
capabilities = DesiredCapabilities.FIREFOX.copy()
capabilities['webStorageEnabled'] = True
self.logger.info("Starting webdriver")
if executable_path is not None:
executable_path = os.path.abspath(executable_path)
self.logger.info("Starting webdriver")
self.driver = webdriver.Firefox(
capabilities=capabilities,
options=options,
executable_path=executable_path,
**extra_params)
else:
self.logger.info("Starting webdriver")
self.driver = webdriver.Firefox(capabilities=capabilities,
options=options,
**extra_params)
elif self.client == "chrome":
self._profile = webdriver.ChromeOptions()
if self._profile_path is not None:
self._profile.add_argument("--user-data-dir=%s" %
self._profile_path)
if proxy is not None:
self._profile.add_argument('--proxy-server=%s' % proxy)
if headless:
self._profile.add_argument('--headless')
self._profile.add_argument('--disable-gpu')
self._profile.add_argument('--remote-debugging-port=9222')
self._profile.add_argument(
'--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36'
)
if chrome_options is not None:
for option in chrome_options:
self._profile.add_argument(option)
if executable_path is not None:
self.logger.info("Starting webdriver")
self.driver = webdriver.Chrome(executable_path=executable_path,
chrome_options=self._profile,
**extra_params)
else:
self.logger.info("Starting webdriver")
self.driver = webdriver.Chrome(chrome_options=self._profile,
**extra_params)
elif client == 'remote':
if self._profile_path is not None:
self._profile = webdriver.FirefoxProfile(self._profile_path)
else:
self._profile = webdriver.FirefoxProfile()
capabilities = DesiredCapabilities.FIREFOX.copy()
self.driver = webdriver.Remote(command_executor=command_executor,
desired_capabilities=capabilities,
**extra_params)
else:
self.logger.error("Invalid client: %s" % client)
self.username = username
self.wapi_functions = WapiJsWrapper(self.driver, self)
self.driver.set_script_timeout(script_timeout)
self.element_timeout = element_timeout
if autoconnect:
self.connect()
def search_by_hashtag(self, hashtag, count=10):
import requests
from browsermobproxy import Server
import psutil
import json
import time
import json
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
import string
import random
for proc in psutil.process_iter():
# check whether the process name matches
if proc.name() == "browsermob-proxy":
proc.kill()
dict = {'port': 8090}
server = Server(path=self.browsermobDirectory, options=dict)
# "browsermob-proxy/bin/browsermob-proxy"
server.start()
time.sleep(1)
proxy = server.create_proxy()
time.sleep(1)
# Firefox selenium stuff
profile = webdriver.FirefoxProfile()
selenium_proxy = proxy.selenium_proxy()
profile.set_proxy(selenium_proxy)
options = Options()
if self.headless == True:
options.headless = True
driver = webdriver.Firefox(firefox_profile=profile, options=options)
# Browsermob-capture
proxy.new_har("list")
driver.get("https://www.tiktok.com/tag/" + hashtag + "?langCountry=en")
data = proxy.har
hashtagId = None
# Assigns signature and hashtagID
for element in data['log']['entries']:
if "https://m.tiktok.com/share/item/list?" in element['request'][
'url'] or "https://www.tiktok.com/share/item/list?" in element[
'request']['url']:
hashtagId = element['request']['queryString'][1]['value']
self.signature = element['request']['queryString'][6]['value']
driver.quit()
response = []
if hashtagId != None:
while True:
try:
var = data['body']['hasMore']
hasMore = True
except:
hasMore = False
cookie = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(40))
url = "https://www.tiktok.com/share/item/list?secUid=&id=" + hashtagId + "&type=3&count=" + \
str(count - len(response)) + "&minCursor=-1&maxCursor=0&_signature=" + \
self.signature + "&shareUid="
headers = {
"authority":
"www.tiktok.com",
"method":
"GET",
"path":
url.split("https://www.tiktok.com")[1],
"scheme":
"https",
"accept":
"application/json, text/plain, */*",
"accept-encoding":
"gzip, deflate, br",
"accept-language":
"en-US,en;q=0.9",
"cache-control":
"no-cache",
"cookie":
cookie,
"referer":
"https://www.tiktok.com/tag/" + hashtag +
"?langCountry=en",
"sec-fetch-mode":
"cors",
"sec-fetch-site":
"same-origin",
"user-agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36"
}
r = requests.get(url, headers=headers)
data = r.json()
if data["statusCode"] == 0:
for tiktok in data["body"]["itemListData"]:
response.append(tiktok)
if hasMore == True:
if count > len(response) and str(
data['body']['hasMore']) == "True":
cookie = ''.join(
random.choice(string.ascii_uppercase +
string.digits) for _ in range(40))
url = "https://www.tiktok.com/share/item/list?secUid=&id=" + hashtagId + "&type=3&count=" + \
str(count - len(response)) + "&minCursor=-1&maxCursor=" + data['body']['maxCursor'] + "&_signature=" + \
self.signature + "&shareUid="
headers = {
"authority":
"www.tiktok.com",
"method":
"GET",
"path":
url.split("https://www.tiktok.com")[1],
"scheme":
"https",
"accept":
"application/json, text/plain, */*",
"accept-encoding":
"gzip, deflate, br",
"accept-language":
"en-US,en;q=0.9",
"cache-control":
"no-cache",
"cookie":
cookie,
"referer":
"https://www.tiktok.com/tag/" + hashtag +
"?langCountry=en",
"sec-fetch-mode":
"cors",
"sec-fetch-site":
"same-origin",
"user-agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36"
}
r = requests.get(url, headers=headers)
data = r.json()
if data["statusCode"] == 0:
for tiktok in data["body"]["itemListData"]:
response.append(tiktok)
else:
return response
else:
raise Exception('Unable to locate the hashtag ID')
import requests
import time
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
ioc = input("informe o código ICAO que deseja buscar informações: ")
url = url = f"https://www.aisweb.aer.mil.br/?i=aerodromos&codigo={ioc}"
print("Aguarde, informações sendo coletadas!")
option = Options()
option.headless = True
driver = webdriver.Firefox(options=option)
driver.get(url)
carts_elements = list()
try:
element = driver.find_elements_by_class_name('list')
size_element = len(element)
start = 0
for c in range(start, size_element):
element = driver.find_elements_by_class_name(
'list')[start].find_elements_by_tag_name('li')
amount_element = len(element)
start_elementos = 0
if (amount_element > 1):
for d in range(start_elementos, amount_element):
elementary = driver.find_elements_by_class_name(
'list')[start].find_elements_by_tag_name(
'li')[start_elementos].text
def __init__(self, path_of_task, php_proxy = 'CORP', s3_proxy = 'CORP', slack_proxy = 'CORP', slack_channel = 'log-test'):
'''
param path_of_task : path of task file for execution
param php_proxy : proxy setting for logging in PhpMyAdmin chosen from 'AWS' / 'GCP' / 'CORP' / 'LOCAL' (use local settnig) or None (No Proxy) with default = 'CORP'
param s3_proxy : S3 proxy chosen from AWS /GCP / CORP or None (not setting proxy) with default = 'CORP'
param slack_proxy : Slack proxy chosen from AWS /GCP / CORP or None (not setting proxy) with default = 'CORP'
param slack_channel : slack channel for recording log with default = 'log-test'
'''
starting_time = datetime.now()
self.is_driver_launch_succeed = False
try:
# change initial slack proxy setting of SlackHandler & s3 proxy setting of S3Handler
super().__init__(s3_proxy, slack_proxy, slack_channel)
# read task file
with open(path_of_task,'r') as file:
self.task_information_dictionary = json.load(file)
# modify option
options = Options()
# set headless mode
options.headless = True
# modify profile
profile = webdriver.FirefoxProfile()
# not use proxy
if not php_proxy:
profile.set_preference("network.proxy.type", 0)
elif php_proxy != 'LOCAL':
proxy = self.key_dict['proxy'][php_proxy]
profile.set_preference('network.proxy.type', 1)
profile.set_preference('network.proxy.http', proxy['host'])
profile.set_preference('network.proxy.http_port', proxy['port'])
profile.set_preference('network.proxy.ssl',proxy['host'])
profile.set_preference('network.proxy.ssl_port', proxy['port'])
# modify download setting
profile.set_preference("browser.download.folderList", 2)
profile.set_preference("browser.download.manager.showWhenStarting", False)
# set download directory
profile.set_preference("browser.download.dir", self.task_information_dictionary['download_directory'])
# types of file for directly executing downloading task without asking
profile.set_preference("browser.helperApps.neverAsk.saveToDisk",
'''
text/plain,text/x-csv,text/csv,
application/vnd.ms-excel,
application/csv,application/x-csv,
text/csv,text/comma-separated-values,
text/x-comma-separated-values,
text/tab-separated-values,
application/pdf
''')
# evoke driver with geckodriver path and options
self.driver = webdriver.Firefox(options = options,firefox_profile = profile, \
executable_path = self.task_information_dictionary['geckodriver_path'])
self.driver.maximize_window()
# setting static waiting time
self.driver.implicitly_wait(20)
self.is_driver_launch_succeed = True
except Exception as E:
error_log = str(E)
ending_time = datetime.now()
message = '{} to launch driver using time: {}{}'.format('Succeeded' if self.is_driver_launch_succeed else 'Failed',
ending_time - starting_time,
'' if self.is_driver_launch_succeed else ' due to {}'.format(error_log))
print(message + '\n\n')
self.PostMessage(self.slack_channel, message, '{}_Log_PhpMyAdmin'.format('Correct' if self.is_driver_launch_succeed else 'Error'))
import shutil
from time import sleep
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.common.proxy import Proxy, ProxyType
from stem import Signal
from stem.control import Controller
import random
import requests
import json
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
elevate()
proxy = 0
options = Options()
options.headless = True #запуск браузера в headless режиме
hidemy = True #Сбор прокси с hidemy.name
freeproxy = False #Сбор прокси с free-proxy.cz
using_proxy = False
if (os.stat("proxy_0.txt").st_size > 0 and using_proxy == True):
try:
if (os.stat("proxy/proxy_0.txt").st_size > 0):
src_dir = os.path.join(os.curdir, "proxy")
dst_dir = os.curdir
src_file = os.path.join(src_dir, "proxy_0.txt")
shutil.copy(src_file, dst_dir)
with open('proxy_0.txt') as f:
content = f.readlines()
content = [x.strip() for x in content]
myproxy = random.choice(content)
async def on_message(message):
if message.author == client.user:
return
if message.content.startswith('$s'):
await message.channel.send('Bot Nike pronto pra missao')
if message.content.startswith('$getcalendario'):
db_conn = mysql.connector.connect(host="129.213.131.233",
port="600",
user="******",
passwd="root",
database="snkrs")
cursor = db_conn.cursor()
binary = FirefoxBinary(
'C:\\Program Files\\Firefox Developer Edition\\firefox.exe')
url = 'https://www.nike.com.br/Snkrs#calendario'
opts = Options()
opts.headless = True
driver = webdriver.Firefox(firefox_binary=binary,
executable_path='C:\\geckodriver.exe',
options=opts)
driver.get(url)
time.sleep(9)
driver.execute_script("window.scrollTo(0,document.body.scrollHeight)")
time.sleep(3)
driver.execute_script("window.scrollTo(0,document.body.scrollHeight)")
time.sleep(9)
#element = driver.find_element_by_xpath("/html/body/main/div/div[3]/section/div/div/div/div[@class='box-resultados vitrine-content--feed grid-produtos grid-produtos--3-col snkr-container']")
element = driver.find_element_by_xpath(
"//*[@id='DadosPaginacaoCalendario']")
html_content = element.get_attribute("outerHTML")
soup = BeautifulSoup(html_content, 'lxml')
a_href = soup.find_all('a', href=True)
hreflist = []
for a in a_href:
href = str(a['href'])
hreflist.append(href)
for i in range(len(hreflist)):
if hreflist[i - 1] != hreflist[i]:
url = hreflist[i - 1]
datetimedb = datetime.datetime.now()
print(url)
await message.channel.send(url)
driver.get(url)
time.sleep(1.2)
element = driver.find_element_by_xpath(
"/html/body/main/div/div[1]/div[3]/div/div[2]/div[@class='nome-preco-produto']"
)
html_content = element.get_attribute("outerHTML")
soup = BeautifulSoup(html_content, 'lxml')
modelo = soup.find_all('span')
cw = soup.find_all('a')
for node in modelo:
print(''.join(node.findAll(text=True)))
modelodb = str(node.findAll(text=True))
#await message.channel.send(modelodb.replace("['", "").replace("']", "").replace("'", ""))
modelolist.append(
modelodb.replace("['",
"").replace("']",
"").replace("'", ""))
for node in cw:
print(' '.join(node.findAll(text=True)))
cwdb = str(node.findAll(text=True))
#await message.channel.send(cwdb.replace("['", "").replace("']", "").replace("'", ""))
cwlist.append(
cwdb.replace("['", "").replace("']",
"").replace("'", ""))
element = driver.find_element_by_xpath(
"/html/body/main/div/div[1]/div[3]/div/div[2]/div[2]/span/span/span[@class='js-valor-por']"
)
html_content = element.get_attribute("outerHTML")
soup = BeautifulSoup(html_content, 'lxml')
preco = soup.find_all('span')
for node in preco:
print(''.join(node.findAll(text=True)))
precodb = str(node.findAll(text=True))
#await message.channel.send(precodb.replace("['", "").replace("']", "").replace("'", ""))
precolist.append(
float(
precodb.replace("['", "").replace(
"']",
"").replace("'",
"").replace("R$",
"").replace(",", ".")))
try:
element = driver.find_element_by_xpath(
"/html/body/main/div/div[1]/div[3]/div/div[2]/h3")
html_content = element.get_attribute("outerHTML")
soup = BeautifulSoup(html_content, 'lxml')
data = soup.find_all('h3')
for node in data:
print(''.join(node.findAll(text=True)))
datadb = str(node.findAll(text=True))
#await message.channel.send(datadb.replace("['", "").replace("']", "").replace("'", ""))
datalist.append(
datadb.replace("['",
"").replace("']",
"").replace("'", ""))
except NoSuchElementException:
print('data não encontrado.')
df = pd.DataFrame({
'Link': [url],
'Modelo': [modelolist],
'CW': [cwlist],
'Preço': [precolist],
'Data': [datalist]
})
await message.channel.send(df.sort_values(by='Preço'))
#sql = """insert into site_refer (refer_href, refer_data, refer_preco, refer_cw, refer_modelo) values(%s, %s, %s, %s, %s)"""
#val = (str(url), datadb, precodb, cwdb, modelodb)
#cursor.execute(sql, val)
#sql = """insert into logs(html_logs, data_hora_logs, tipo_site) values(%s, %s, 1)"""
#val = (str(url), datetimedb)
#cursor.execute(sql, val)
#db_conn.commit()
driver.quit()
db_conn.close()
def download_gisaid_EpiCoV(
uname, # username
upass, # password
normal, # normal mode (quite)
wd, # output dir
loc, # location
host, # host
cs, # collection start date
ce, # collection end date
ss, # submission start date
se, # submission end date
cg, # complete genome only
hc, # high coverage only
le, # low coverage excluding
to, # timeout in sec
rt, # num of retry
iv, # interval in sec
nnd, # do not download nextstrain data
ffbin # firefox binary path
):
"""Download sequences and metadata from EpiCoV GISAID"""
# when user doesn't download nextstrain data, it's essential to enter time range/location
if not (cs or ce or ss or se or loc) and nnd:
logging.error("No time range or location entered.")
sys.exit(1)
# output directory
if not os.path.exists(wd):
os.makedirs(wd, exist_ok=True)
wd = os.path.abspath(wd)
GISAID_DTL_JASON = f'{wd}/gisaid_detail_metadata.json'
metadata = []
# MIME types
mime_types = "application/octet-stream"
mime_types += ",application/excel,application/vnd.ms-excel"
mime_types += ",application/pdf,application/x-pdf"
mime_types += ",application/x-bzip2"
mime_types += ",application/x-gzip,application/gzip"
# start fresh
try:
os.remove(GISAID_DTL_JASON)
except OSError:
pass
logging.info("Opening browser...")
profile = webdriver.FirefoxProfile()
profile.set_preference("browser.download.folderList", 2)
profile.set_preference("browser.download.manager.showWhenStarting", False)
profile.set_preference("browser.download.dir", wd)
profile.set_preference("browser.helperApps.neverAsk.saveToDisk",
mime_types)
profile.set_preference("plugin.disable_full_page_plugin_for_types",
mime_types)
profile.set_preference("pdfjs.disabled", True)
profile.update_preferences()
options = Options()
if not normal:
options.headless = True
driver = webdriver.Firefox(firefox_profile=profile,
options=options,
firefox_binary=ffbin)
# driverwait
driver.implicitly_wait(30)
wait = WebDriverWait(driver, to)
# open GISAID
logging.info("Opening website GISAID...")
driver.get('https://www.epicov.org/epi3/frontend')
waiting_sys_timer(wait)
logging.info(driver.title)
assert 'GISAID' in driver.title
# login
logging.info("Logining to GISAID...")
username = driver.find_element_by_name('login')
username.send_keys(uname)
password = driver.find_element_by_name('password')
password.send_keys(upass)
driver.execute_script("return doLogin();")
waiting_sys_timer(wait)
# navigate to EpiFlu
logging.info("Navigating to EpiCoV...")
epicov_tab = driver.find_element_by_xpath("//div[@id='main_nav']//li[3]/a")
epicov_tab.click()
waiting_sys_timer(wait)
# download nextstrain data
if not nnd:
# download from downloads section
logging.info("Clicking downloads...")
pd_button = wait.until(
EC.element_to_be_clickable(
(By.XPATH, "//div[@class='sys-actionbar-bar']//div[3]")))
pd_button.click()
waiting_sys_timer(wait)
# have to click the first row twice to start the iframe
iframe_dl = waiting_for_iframe(wait, driver, rt, iv)
logging.info("Downloading metadata...")
driver.switch_to.frame(iframe_dl)
waiting_sys_timer(wait)
dl_button = driver.find_element_by_xpath(
'//div[contains(text(), "metadata")]')
dl_button.click()
waiting_sys_timer(wait)
# waiting for REMINDER
iframe = waiting_for_iframe(wait, driver, rt, iv)
driver.switch_to.frame(iframe)
waiting_sys_timer(wait)
# agree terms and conditions
logging.info(" -- agreeing terms and conditions")
checkbox = driver.find_element_by_xpath(
'//input[@class="sys-event-hook"]')
checkbox.click()
waiting_sys_timer(wait)
# click download button
dl_button = wait.until(
EC.element_to_be_clickable(
(By.XPATH, '//button[contains(text(), "Download")]')))
dl_button.click()
waiting_sys_timer(wait)
logging.info(" -- downloading")
# Opening Firefox downloading window
driver.switch_to.default_content()
fn = wait_downloaded_filename(wait, driver, 600)
logging.info(f" -- downloaded to {fn}.")
waiting_sys_timer(wait)
logging.info("Downloading FASTA...")
driver.switch_to.frame(iframe_dl)
waiting_sys_timer(wait)
# click nextfasta button
dl_button = wait.until(
EC.element_to_be_clickable((By.XPATH, '//div[text()="FASTA"]')))
dl_button.click()
waiting_sys_timer(wait)
# waiting for REMINDER
iframe = waiting_for_iframe(wait, driver, rt, iv)
driver.switch_to.frame(iframe)
waiting_sys_timer(wait)
# agree terms and conditions
logging.info(" -- agreeing terms and conditions")
checkbox = driver.find_element_by_xpath(
'//input[@class="sys-event-hook"]')
checkbox.click()
waiting_sys_timer(wait)
# click download button
dl_button = wait.until(
EC.element_to_be_clickable(
(By.XPATH, '//button[contains(text(), "Download")]')))
dl_button.click()
waiting_sys_timer(wait)
logging.info(" -- downloading")
# Opening Firefox downloading window
driver.switch_to.default_content()
fn = wait_downloaded_filename(wait, driver, 600)
logging.info(f" -- downloaded to {fn}.")
waiting_sys_timer(wait)
# go back to main frame
driver.switch_to.frame(iframe_dl)
back_button = driver.find_element_by_xpath(
'//button[contains(text(), "Back")]')
back_button.click()
driver.switch_to.default_content()
waiting_sys_timer(wait)
if cs or ce or ss or se or loc:
logging.info("Browsing EpiCoV...")
browse_tab = wait.until(
EC.element_to_be_clickable(
(By.XPATH, '//*[contains(text(), "Browse")]')))
browse_tab.click()
waiting_sys_timer(wait)
waiting_table_to_get_ready(wait)
# set location
if loc:
logging.info("Setting location...")
loc_input = driver.find_element_by_xpath(
"//td/div[contains(text(), 'Location')]/../following-sibling::td/div/div/input"
)
loc_input.send_keys(loc)
waiting_sys_timer(wait, 7)
# set host
if host:
logging.info("Setting host...")
host_input = driver.find_element_by_xpath(
"//td/div[contains(text(), 'Host')]/../following-sibling::td/div/div/input"
)
host_input.send_keys(host)
waiting_sys_timer(wait, 7)
# set dates
date_inputs = driver.find_elements_by_css_selector(
"div.sys-form-fi-date input")
dates = (cs, ce, ss, se)
for dinput, date in zip(date_inputs, dates):
if date:
logging.info("Setting date...")
dinput.send_keys(date)
ActionChains(driver).send_keys(Keys.ESCAPE).perform()
waiting_sys_timer(wait, 7)
# complete genome only
if cg:
logging.info("complete genome only...")
checkbox = driver.find_element_by_xpath(
'//input[@value="complete"]')
checkbox.click()
waiting_sys_timer(wait)
# high coverage only
if hc:
logging.info("high coverage only...")
checkbox = driver.find_element_by_xpath('//input[@value="highq"]')
checkbox.click()
waiting_sys_timer(wait)
# excluding low coverage
if le:
logging.info("low coverage excluding...")
checkbox = driver.find_element_by_xpath('//input[@value="lowco"]')
checkbox.click()
waiting_sys_timer(wait)
# check if any genomes pass filters
warning_message = None
try:
warning_message = driver.find_element_by_xpath(
"//div[contains(text(), 'No data found.')]")
except:
pass
if warning_message:
logging.info("No data found.")
sys.exit(1)
# select all genomes
logging.info("Selecting all genomes...")
button_sa = driver.find_element_by_css_selector(
"span.yui-dt-label input")
button_sa.click()
waiting_sys_timer(wait)
# downloading sequence data
num_download_options = 1
current_option = 0
retry = 0
while retry <= rt and current_option < num_download_options:
try:
logging.info("Downloading sequences for selected genomes...")
button = driver.find_element_by_xpath(
"//td[@class='sys-datatable-info']/button[contains(text(), 'Download')]"
)
button.click()
waiting_sys_timer(wait)
# switch to iframe
iframe = waiting_for_iframe(wait, driver, rt, iv)
driver.switch_to.frame(iframe)
waiting_sys_timer(wait)
# selecting options
labels = driver.find_elements_by_xpath("//label")
num_download_options = len(labels)
labels[current_option].click()
current_option += 1
button = driver.find_element_by_xpath(
"//button[contains(text(), 'Download')]")
button.click()
waiting_sys_timer(wait)
driver.switch_to.default_content()
fn = wait_downloaded_filename(wait, driver, 1800)
logging.info(f"Downloaded to {fn}.")
except:
logging.info(f"retrying...#{retry} in {iv} sec(s)")
if retry == rt:
logging.error("Unexpected error:", sys.exc_info())
sys.exit(1)
else:
time.sleep(iv)
retry += 1
# close driver
driver.quit()
def write_parsed_to_csv(page_url,
map_info,
writer,
pscores,
page_number=2,
web_driver=None):
"""Given the current page URL, extract the information from each apartment in the list"""
# We start on page_number = 2, since we will already be parsing page_number 1
# if we are loading the page for the first time, we want to initailize the web driver
if (web_driver != None):
driver = web_driver
else:
options = Options()
options.headless = True
if ('debian' in platform.platform()):
driver = webdriver.Firefox(firefox_binary='/usr/bin/firefox-esr',
options=options)
else:
driver = webdriver.Firefox(options=options)
driver.get(page_url)
# read the current page
soup = BeautifulSoup(driver.page_source, 'html.parser')
# soupify the current page
soup.prettify()
# only look in this region
soup = soup.find('div', class_='placardContainer')
# append the current apartments to the list
for item in soup.find_all('article', class_='placard'):
url = ''
rent = ''
contact = ''
if item.find('a', class_='placardTitle') is None: continue
url = item.find('a', class_='placardTitle').get('href')
# get the rent and parse it to unicode
obj = item.find('span', class_='altRentDisplay')
if obj is not None:
rent = obj.getText().strip()
# get the phone number and parse it to unicode
obj = item.find('div', class_='phone')
if obj is not None:
contact = obj.getText().strip()
# get the other fields to write to the CSV
fields = parse_apartment_information(url, map_info)
# make this wiki markup
fields['name'] = '[' + str(fields['name']) + '](' + url + ')'
fields[
'address'] = '[' + fields['address'] + '](' + fields['map'] + ')'
# fill out the CSV file
row = [
fields['name'], contact, fields['address'], fields['size'], rent,
fields['monthFees'], fields['onceFees'], fields['petPolicy'],
fields['distance'], fields['duration'], fields['parking'],
fields['gym'], fields['kitchen'], fields['amenities'],
fields['features'], fields['space'], fields['lease'],
fields['services'], fields['info'], fields['indoor'],
fields['outdoor'], fields['img'], fields['description']
]
# add the score fields if necessary
if pscores:
for i in xrange(len(row), 0, -1):
row.insert(i, '5')
row.append('0')
# write the row
writer.writerow(row)
page_number_str = str(page_number)
# check for our next page number
try:
page_number_element = driver.find_element_by_xpath("//a[@data-page='" +
page_number_str +
"']")
page_number_element.click()
time.sleep(1)
# we will get a no element found exception, meaning our search has come to an end
except:
driver.quit()
return
# recurse until the last page
write_parsed_to_csv("none", map_info, writer, pscores, page_number + 1,
driver)
HEADLESS = os.access('YES', os.R_OK)
DB_DIR_SET = os.access('data', os.R_OK) and os.access(
'data/db', os.R_OK) and os.access('data/logs', os.R_OK)
debug = False
# -----------
ff = None # browser client
print(f'HEADLESS:{HEADLESS}')
#print(sys.argv)
print('setting up!')
browser = sys.argv[1].lower()
if browser == "firefox":
from selenium.webdriver.firefox.options import Options
options = Options()
options.headless = HEADLESS
ff = Firefox(executable_path=FF_PATH, options=options)
elif browser == "chrome":
from selenium.webdriver.chrome.options import Options
options = Options()
if sys.argv[-1].lower() == 'android':
options.add_experimental_option('androidPackage', 'com.android.chrome')
print('run on android')
ff = Chrome(CHRA_PATH, options=options)
else:
options.headless = HEADLESS
ff = Chrome(executable_path=CHR_PATH, options=options)
elif browser == "debug":
print("Debug mode")
print('run script in interactive mode with -i option')
debug = True
def setUp(self):
options = Options()
options.headless = True
self.driver = webdriver.Firefox(
options=options, executable_path=GeckoDriverManager().install())
def before_all(context):
options = Options()
options.headless = HEADLESS
context.browser = webdriver.Firefox(options=options)
context.browser.set_window_size(SCREEN_WIDTH, SCREEN_HEIGHT)
def create_driver(self):
opts = Options()
opts.headless = True
return webdriver.Firefox(
options=opts, executable_path=GeckoDriverManager().install())
def main():
# Set up Selenium options to enable headless access
options = Options()
options.headless = True
driver = webdriver.Firefox(options=options,
executable_path='/usr/local/bin/geckodriver')
# Get identifier from address
loc_id = address.split('/')[-1]
# Parse the Accuweather websites: Most data is collected from this page
curr_address = (address.split('weather-forecast')[0] + 'current-weather/' +
loc_id)
driver.get(curr_address)
curr_cond = BeautifulSoup(driver.page_source, 'lxml')
# Precipitation data is collected from this page
daily_address = (address.split('weather-forecast')[0] +
'daily-weather-forecast/' + loc_id)
driver.get(daily_address)
daily_cond = BeautifulSoup(driver.page_source, 'lxml')("div",
"column detail")[0]
# Extended forecasts are collected from this page
final_address = daily_address + '?day=6'
driver.get(final_address)
final_cond = BeautifulSoup(driver.page_source, 'lxml')
driver.quit()
# Strain 'curr_cond' soup
# NOTE: Accuweather redesigned their website sometime late 2018, so the
# code will need to be strained differently:
try:
forecast = curr_cond("div", "conditions-wrapper day")[0]
except IndexError:
forecast = curr_cond("div", "conditions-wrapper night")[0]
sun = curr_cond("div", "block rise-set sun")[0]
moon = curr_cond("div", "block rise-set moon")[0]
highs = (curr_cond("table", "temp-history text--size14")
[0].contents[3].contents[0].text.split('\n'))
lows = (curr_cond("table", "temp-history text--size14")
[0].contents[3].contents[2].text.split('\n'))
# Strain soup for daily forecasts
panel_list = curr_cond("div", "five-day")[0]
extended_list = final_cond("div", "five-day")[0]
# Get moon phase info
# Check that we have interent connection (using EAFP principle)
# Admittedly, this isn't as clean as I would like.
try:
HEADERS = REQUEST_HEADERS
requests.get(MOON_ADDRESS, headers=REQUEST_HEADERS, timeout=3.05)
except Exception as e:
HEADERS = WGET_HEADERS
pass
try:
requests.get(MOON_ADDRESS, headers=WGET_HEADERS, timeout=3.05)
except Exception as e:
sys.exit()
moon_info = BeautifulSoup(
requests.get(MOON_ADDRESS, headers=HEADERS).text, 'lxml')
# Get geolocation info for forecast
city = curr_cond.find(class_="locality").find().get("title")
region = curr_cond.find("abbr").get("title")
country = curr_cond.find(class_="country-name").get("title")
# Parse JSON data in moon_info
moon_dict = json.loads(
moon_info("script")[3].string.split("jArray")[1].rsplit(";")[0].lstrip(
"="))['2']
moon_minus = (moon_info('img', class_='moonNotToday')
[1].attrs['src'].split('moon_day_')[1].split('.')[0])
moon_icon = (moon_info.find(
id='todayMoonContainer').find().get('alt').split('moon_phase_')[1])
moon_plus = (moon_info('img', class_='moonNotToday')[2].attrs['src'].split(
'moon_day_')[-1].split('.')[0])
# Determine temperature scale used in HTML
scale = (curr_cond("div", "subnav-dropdown-container location-crumbs")
[0].contents[5].contents[3].text[-1])
# [2018-11-15 Thu] Accuweather changed their setup, and now there is no
# explicit declaration of the scale used on the page, so I scrape it from
# its appearance on the top of the page.
# I don't see Accuweather using Kelvin or other scales anytime soon.
# Collect the weather information
# NOTE: I strip the degree symbol to be able to do conditional comparisons
# of the temperatures in Conky. I later append the degree symbols in
# the Lua script.
# NOTE: '\xc2' is part of the UTF8 representation of the degree symbol
weather = OrderedDict()
weather['current_icon'] = image_conkyweather[forecast(
"div", "icon")[0].contents[1]['src'].split('/')[-1].split('.')[0]]
weather['current_cond'] = forecast("div", "phrase")[0].text
weather['current_temp'] = convert_item(
forecast("div", "hi")[0].text.strip('°'), 'temp', scale, out_scale)
weather['current_feel'] = convert_item(
forecast("div", "realfeel")[0].text.split()[1].strip('°'), 'temp',
scale, out_scale)
weather['windicon'] = image_conkywindnesw[forecast(
"div", "wind-point")[0]['class'][1]]
weather['wind_spd'] = (convert_item(
forecast("div", "speed")[0].text.split()[0], 'dist', scale, out_scale)
+ units_conversion[out_scale + 'speed'])
weather['humidity'] = (forecast("div",
"details")[0].contents[1].text.split()[1])
weather['pressure'] = (convert_item(
forecast("div", "details")[0].contents[3].text.split()[1], 'pressure',
scale, out_scale) + units_conversion[out_scale + 'pressure'])
weather['uv_index'] = (forecast("div",
"details")[0].contents[5].text.split()[-1])
weather['cloudcov'] = (forecast("div",
"details")[0].contents[7].text.split()[-1])
weather['dewpoint'] = convert_item(
forecast("div", "details")[0].contents[11].text.split()[2].strip('°'),
'temp', scale, out_scale)
weather['visiblty'] = (convert_item(
forecast("div", "details")[0].contents[13].text.split()[1], 'dist',
scale, out_scale) + units_conversion[out_scale + 'dist'])
weather['sunrise'] = convert_time(sun("time", "rise")[0].text)
weather['sunset'] = convert_time(sun("time", "set")[0].text)
weather['suntime'] = skytime(sun, weather['sunrise'], weather['sunset'])
weather['moonrise'] = convert_time(moon("time", "rise")[0].text)
weather['moonset'] = convert_time(moon("time", "set")[0].text)
weather['moontime'] = skytime(moon, weather['moonrise'],
weather['moonset'])
weather['moon_phase'] = moon_dict[7]
weather['moonshine'] = ''.join(d
for d in moon_dict[1] if d.isdigit()) + '%'
weather['moon_minus'] = image_moongiant[moon_minus]
weather['moon_icon'] = image_moongiant[moon_icon]
weather['moon_plus'] = image_moongiant[moon_plus]
# Make separate dictionary for temperature history
history = OrderedDict()
history['high_today'] = convert_item(highs[2].strip('°'), 'temp', scale,
out_scale)
history['high_mean'] = convert_item(highs[3].strip('°'), 'temp', scale,
out_scale)
if highs[5] == 'N/A':
history['high_record'] = 'No record'
history['high_record_year'] = ''
else:
history['high_record'] = convert_item(highs[5].split('°')[0], 'temp',
scale, out_scale)
history['high_record_year'] = highs[5].split()[1]
history['high_last_year'] = convert_item(highs[7].strip('°'), 'temp',
scale, out_scale)
history['low_today'] = convert_item(lows[2].strip('°'), 'temp', scale,
out_scale)
history['low_mean'] = convert_item(lows[3].strip('°'), 'temp', scale,
out_scale)
if lows[5] == 'N/A':
history['low_record'] = 'No record'
history['low_record_year'] = ''
else:
history['low_record'] = convert_item(lows[5].split('°')[0], 'temp',
scale, out_scale)
history['low_record_year'] = lows[5].split()[1]
history['low_last_year'] = convert_item(lows[7].strip('°'), 'temp', scale,
out_scale)
# Make a dictionary for the precipitation info
precipitation = OrderedDict()
precipitation['percent'] = daily_cond.text.split('\n')[2].split()[1]
precipitation['amount'] = (convert_item(
daily_cond.text.split('\n')[3].split()[1], 'measure', scale, out_scale)
+ units_conversion[out_scale + 'measure'])
precipitation['hours'] = daily_cond.text.split('\n')[7].split()[3] + " hrs"
# Collect forecast information and store in nested dictionaries)
daily_forecasts = strain_forecast(panel_list, scale, out_scale)
extended_forecasts = strain_forecast(extended_list, scale, out_scale)
# Create a timestamp
now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
# Change directory
os.chdir(outputpath)
# Write output
text = open("conditions", "w")
for key in weather.keys():
text.write("%s\n" % weather[key])
for key in history.keys():
text.write("%s\n" % history[key])
for key in precipitation.keys():
text.write("%s\n" % precipitation[key])
text.write("%s, %s\n" % (city, region))
text.write("%s\n" % country)
text.write("%s\n" % now)
text.write("%s\n" % out_scale)
text.close()
text = open("forecast", "w")
for x in range(0, 5):
for key in daily_forecasts[x].keys():
text.write("%s\n" % daily_forecasts[x][key])
for x in range(0, 5):
for key in extended_forecasts[x].keys():
text.write("%s\n" % extended_forecasts[x][key])
text.close()
return None
def __init__(self):
options = Options()
options.headless = True
self.driver = webdriver.Firefox(options=options)
self.driver.set_script_timeout(10)
self.driver.set_page_load_timeout(10)
def _get_driver():
options = Options()
options.headless = True
return webdriver.Firefox(options=options, executable_path=gecko_bin, service_log_path=gecko_log)
def runBot(self):
#TODO Proxy aufsetzten
#PROXY = "91.205.174.26:80"
#webdriver.DesiredCapabilities.FIREFOX['proxy'] = {
# "httpProxy": PROXY,
# "ftpProxy": PROXY,
# "sslProxy": PROXY,
# "proxyType": "MANUAL",
#}
#with webdriver.Firefox() as driver:
# # Open URL
# driver.get(url)
options = Options()
if self.headless:
options.headless = True
driver = webdriver.Firefox(
options=options) # executable_path=r"/usr/bin/geckodriver"
driver.delete_all_cookies()
driver.get(self.url)
# Get video duration for refresh time
duration_element = driver.find_element_by_class_name(
'ytp-time-duration')
runtime_full = duration_element.text
runtime_calc = runtime_full.split(":")
runtime_max_sec = (((int(runtime_calc[0]) * 60) / 100) *
int(self.runtime))
print(runtime_max_sec)
# Signin popup for sponsoered videos
if self.pro:
try:
sponsored_login = WebDriverWait(driver, 10).until(
EC.visibility_of_element_located(
(By.XPATH, "//div[@id='dismiss-button']")))
print(sponsored_login.__dict__)
for sponsored_element in sponsored_login.find_elements_by_xpath(
"//div[@id='dismiss-button']"):
print(sponsored_element.text)
print(sponsored_element.id)
print(sponsored_element.get_attribute("class"))
# sleep timer to imitaed a "real" person
time.sleep(random.uniform(1, 2.5))
ActionChains(driver).click(sponsored_element).perform()
except:
print("No Login Button")
# Cookie popup
WebDriverWait(driver, 10).until(
EC.frame_to_be_available_and_switch_to_it((By.ID, "iframe")))
cookieFrame = WebDriverWait(driver, 20).until(
EC.visibility_of_element_located(
(By.XPATH, "//div[@id='introAgreeButton']")))
print(cookieFrame.__dict__)
# print(driver.page_source) #html code
for cookie_element in cookieFrame.find_elements_by_xpath(
"//div[@id='introAgreeButton']"):
print(cookie_element.text)
print(cookie_element.id)
print(cookie_element.get_attribute("class"))
# sleep timer to imitated a "real" person
time.sleep(random.uniform(1, 2.5))
ActionChains(driver).click(cookie_element).perform()
# Wait for video container
time.sleep(5)
driver.switch_to.default_content()
# Skip ads
if self.pro is not None:
try:
# wait 5 sec till skip button is available
time.sleep(5)
ads_element = driver.find_element_by_class_name(
'ytp-ad-skip-button-container')
print(ads_element.__dict__)
print(ads_element.text)
ActionChains(driver).click(ads_element).perform()
except:
print("No Ads")
# Min runtime 10 sec
# Max runtime 70% of video runtime
print(self.only_view)
if self.only_view is None:
time.sleep(random.uniform(10, runtime_max_sec))
driver.close()
else:
driver.close()
# recursion for test
self.runBot()
def getBookInfo(bookID, userLogin, userPassword, dbPath):
def login(login, password):
loginButton = WebDriverWait(driver, 30).until(EC.element_to_be_clickable((By.CSS_SELECTOR, ".login")))
loginButton.click()
completeForm(login, password)
print("Login successful!")
def completeForm(login, password):
form = WebDriverWait(driver, 30).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "form")))
loginField = form.find_element_by_css_selector("input[name='login']")
passwordField = form.find_element_by_css_selector("input[name='password']")
loginField.send_keys(login)
passwordField.send_keys(password)
time.sleep(0.5)
submit = form.find_element_by_css_selector("button[type='submit']")
submit.click()
def getNumberOfPages():
pagesProperty = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//*[@data-state-content='online']" + "//ul" + "/li[text()[contains(., 'стр')]]")))
pagesCountString = pagesProperty.get_attribute('textContent')
return list(map(int, re.findall(r'\d+', pagesCountString)))[0]
def getBookTitle():
title = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, ".page-title h1")))
return title.get_attribute('textContent').rstrip()
def openReader():
URL = driver.current_url
driver.get(URL + "/reader")
print("Opened reader.")
def findRequestKey():
for request in driver.requests:
if request.response:
link = urlparse.unquote(request.url)
if (("image" in request.response.headers['Content-Type']) and ("OEBPS" in link)):
splitLink = link.rsplit(".", 1)
return splitLink[0], splitLink[1]
return False
rootURL = "https://bmstu.press"
homeURL = rootURL + "/catalog/item/" + str(bookID).zfill(4)
db = TinyDB(dbPath)
if readDB(bookID, db) != False:
print("Book found in database. No need to connect to server.")
else:
print("Book not found in database. Connecting to server...")
try:
swireOptions = {
'suppress_connection_errors': True,
'connection_timeout': 50,
}
selOptions = Options()
selOptions.headless = True
selOptions.capabilities["pageLoadStartegy"] = "eager"
driver = webdriver.Firefox(options=selOptions, seleniumwire_options=swireOptions) # pylint: disable=unexpected-keyword-arg
driver.get(homeURL)
print("Connected to server.")
login(userLogin, userPassword)
endPage = getNumberOfPages()
title = getBookTitle()
openReader()
time.sleep(10)
if findRequestKey() == False:
raise ValueError("Did not find the right request.")
key, imgFormat = findRequestKey()
writeDB(bookID, title, endPage, key, imgFormat, db)
except:
print("Could not receive response from server or this book ID might not exist. Please try again or change ID.")
finally:
driver.close()
driver.quit()
return readDB(bookID, db)
def init_driver():
options = Options()
options.headless = True
driver = webdriver.Firefox(options=options)
return driver
def start():
opts = Options()
opts.headless = True
firefox_driver = webdriver.Firefox(
options=opts, executable_path='firefox\\geckodriver.exe')
return firefox_driver
from json import load
from sys import exit
import argparse, getpass
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
with open(r"urls.json") as f:
urls = load(f)
f = None
options = Options()
options.headless = False
parser = argparse.ArgumentParser()
parser.add_argument("url",
type=str,
help="expects custom codes of sites",
choices=[x for x in urls.keys()])
parser.add_argument("username", type=str, help="expects your username")
parser.add_argument("-p",
"--private",
action="store_true",
help="browse inprivate")
args = parser.parse_args()
uname = args.username
passw = getpass.getpass("password? ")
if args.private:
firefox_profile = webdriver.FirefoxProfile()
firefox_profile.set_preference("browser.privatebrowsing.autostart", True)
def get_freq(self) -> dict:
'''This function gets the minor allele frequencies for variants of interest'''
# Creating four list that will work be used to keep track of the variants and the
variant_list: list = []
allele_freq_list: list = []
exome_filter_list: list = []
genome_filter_list: list = []
# creating a headless mode
options = Options()
options.headless = True
# Creating a dictionary to act as a handler based on what browser is used
browser_handling_dict: dict = {
"firefox": determine_webdriver(self.browser, options),
"chrome": determine_webdriver(self.browser, options),
}
# creating the driver object
browser = browser_handling_dict[self.browser]
# iterating through each variant
for variant in self.var_list:
# Getting the full page path for a specific variant
full_url: str = "".join([self.url_start, variant, self.url_end])
browser.get(full_url)
# finding the table element
element = WebDriverWait(browser, self.time).until(
EC.presence_of_element_located((
By.CLASS_NAME,
"Table__BaseTable-sc-7fgtt2-0.PopulationsTable__Table-yt4zj1-0.gRZyOM"
)))
frequencies_table = browser.find_element_by_class_name(
"Table__BaseTable-sc-7fgtt2-0.PopulationsTable__Table-yt4zj1-0.gRZyOM"
)
# Getting the text from the table element
pop_freq_txt: str = frequencies_table.text
# This section gets the first index
start_indx: int = pop_freq_txt.find("(non-Finnish)")
start_indx = start_indx + len("(non-Finnish)")
# Get a substring that starts at the end of the non-Finnish
pop_freq_substring: str = pop_freq_txt[start_indx:]
# Use a regular expression to find The first letter in the substring
letter_match = re.search("[a-zA-Z]", pop_freq_substring)
first_letter: str = letter_match.group(0)
# find second index
end_indx: int = pop_freq_substring.find(first_letter)
allele_freq: str = pop_freq_substring[:end_indx].split(
" ")[3].strip("\n").strip(" ")
print(
f"This is the allele frequencies for the variant {variant} is {allele_freq}"
)
# also have to get the filter status
filter_tuple: tuple = self.get_filter_status(browser)
# adding the variant and the allele frequencies to a dictionary
variant_list.append(variant)
allele_freq_list.append(allele_freq)
exome_filter_list.append(filter_tuple[0])
genome_filter_list.append(filter_tuple[1])
# creating a dictionary to keep track of the allele frequencies
gnomad_freq_dict: dict = {
"RS Name": variant_list,
"MAF": allele_freq_list,
"exome_filter_status": exome_filter_list,
"genome_filter_status": genome_filter_list
}
# closign the webdriver
browser.close()
browser.quit()
return gnomad_freq_dict
def test_verisini_cek():
options = Options()
options.headless = True
global garanti_test
global X_test
# =============================================================================
# BIST100
# =============================================================================
driver_path = r"..\geckodriver\geckodriver.exe"
fox = webdriver.Firefox(options=options,executable_path=driver_path)
url = "https://tr.investing.com/indices/ise-100-historical-data"
fox.get(url)
time.sleep(5)
bist100_open = fox.find_element_by_xpath('//*[@id="curr_table"]/tbody/tr[2]/td[3]')
bist100_open = bist100_open.get_attribute('innerHTML')
fox.close()
# =============================================================================
# Garanti Verilerinin Çekilmesi
# =============================================================================
browser = webdriver.Firefox(options=options,executable_path=driver_path)
url = "https://finance.yahoo.com/quote/GARAN.IS/history?p=GARAN.IS"
browser.get(url)
time.sleep(10)
garan_open = browser.find_element_by_xpath('//*[@id="Col1-1-HistoricalDataTable-Proxy"]/section/div[2]/table/tbody/tr[2]/td[2]/span')
garan_high = browser.find_element_by_xpath('//*[@id="Col1-1-HistoricalDataTable-Proxy"]/section/div[2]/table/tbody/tr[2]/td[3]/span')
garan_low = browser.find_element_by_xpath('//*[@id="Col1-1-HistoricalDataTable-Proxy"]/section/div[2]/table/tbody/tr[2]/td[4]/span')
garan_close = browser.find_element_by_xpath('//*[@id="Col1-1-HistoricalDataTable-Proxy"]/section/div[2]/table/tbody/tr[2]/td[5]/span')
garan_volume = browser.find_element_by_xpath('//*[@id="Col1-1-HistoricalDataTable-Proxy"]/section/div[2]/table/tbody/tr[2]/td[7]/span')
garan_is = []
garanti = [garan_open,garan_high,garan_low,garan_close,garan_volume]
for i in garanti:
garan_is.append((i.get_attribute('innerHTML')))
browser.close()
# =============================================================================
# Dolar Verilerinin Çekilmesi
# =============================================================================
browser = webdriver.Firefox(options=options,executable_path=driver_path)
url = "https://finance.yahoo.com/quote/TRY%3DX/history?p=TRY%3DX"
browser.get(url)
time.sleep(5)
usd_try = browser.find_element_by_xpath('//*[@id="Col1-1-HistoricalDataTable-Proxy"]/section/div[2]/table/tbody/tr[2]/td[2]/span')
dolar = usd_try.get_attribute('innerHTML')
browser.close()
# =============================================================================
# Petrol Verilerinin Çekilmesi
# =============================================================================
browser = webdriver.Firefox(options=options,executable_path=driver_path)
url = "https://tr.investing.com/commodities/crude-oil-historical-data"
browser.get(url)
time.sleep(5)
ham_petrol = browser.find_element_by_xpath('//*[@id="curr_table"]/tbody/tr[2]/td[3]')
petrol = ham_petrol.get_attribute('innerHTML')
browser.close()
# =============================================================================
# Verilerin Modele Sokulmadan Hazırlanması
# Eğitimde kullanılan veri sıralaması: <br>
# Garan_Open, Garan_High, Garan_Low, Garan_Close, Garan_Volume, Bist_Open, Dolar_Open, Petrol_Open
# =============================================================================
garan_is[-1] = garan_is[-1].replace(',', '')
garanti_test = []
for i in garan_is:
garanti_test.append(float(i))
bist100_open = bist100_open.replace('.','')
bist100_open = bist100_open.replace(',','.')
bist100_open= float(bist100_open)
garanti_test.append(bist100_open)
garanti_test.append(float(dolar))
petrol = petrol.replace(',','.')
garanti_test.append(float(petrol))
garanti_np = np.asarray(garanti_test).reshape(-1,1)
sc = StandardScaler()
test = sc.fit_transform(garanti_np)
X_test = test.reshape(test.shape[1], test.shape[0], 1)
print('test verimiz: ', garanti_test)
return garanti_test, X_test
def setUpClass(cls):
super().setUpClass()
options = Options()
options.headless = bool(os.environ.get("TRAVIS"))
cls.selenium = webdriver.Firefox(options=options)
def setUp(self):
options = Options()
options.headless = True
self.driver = Firefox(options=options)
#BEGIN IMPORT STATEMENTS
from selenium.webdriver import Firefox
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.common.keys import Keys
import requests
#END IMPORT STATEMENTS
#BEGIN GLOBAL VARIABLES
opts = Options()
opts.headless = True
assert opts.headless #Operating in headless mode
browser = Firefox(options=opts)
browser.get('https://cfo.asu.edu/buildingmaintenance-outages-shutdowns')
#WEBEX VARIABLES
url = "https://api.ciscospark.com/v1/messages"
message = ""
body = {
"roomId":
"Y2lzY29zcGFyazovL3VzL1JPT00vODc0YmU4ZjAtMWMyOC0xMWVhLWJlYmQtMjk2MDA0YWI2MDdm",
"text": message
}
headers = {
'content-type': 'application/json',
'authorization': 'Bearer INSERTBEARERTOKENHERE',
'User-Agent': "JustMe",
'Accept': "*/*",
}
#END GLOBAL VARIABLES
#BEGIN GATHER FINDINGS
results = browser.find_elements_by_id('pData')
def run(self):
if self.browser == "firefox":
# webdriver.DesiredCapabilities.FIREFOX['proxy'] = {
# "httpProxy": self.proxy,
# "ftpProxy": self.proxy,
# "sslProxy": self.proxy,
#
# "proxyType": "MANUAL",
#
# }
options = Options()
options.headless = True
self.driver = webdriver.Firefox(options=options)
elif self.browser == "chrome":
self.driver = webdriver.Chrome(
chrome.ChromeDriverManager().install())
elif self.browser == "opera":
self.driver = webdriver.Opera()
else:
return "Wrong type of browser"
# self.driver.get('https://www.expressvpn.com/what-is-my-ip')
self.driver.get(self.url)
sleep(14)
element = self.driver.find_elements_by_name("emailAddress")
while len(element) == 0:
element = self.driver.find_elements_by_name("emailAddress")
element[0].send_keys(self.login)
self.driver.find_element_by_name("password").send_keys(self.password)
self.driver.find_element_by_class_name(
"nike-unite-component.nike-unite-submit-button").click()
el_test = self.driver.find_elements_by_id("middleName")
while len(el_test) != 1:
sleep(2)
element = self.driver.find_elements_by_class_name(
"nike-unite-error-close")
if len(element) != 0:
element[0].click()
self.driver.find_element_by_name("password").send_keys(
self.password)
self.driver.find_element_by_class_name(
"nike-unite-component.nike-unite-submit-button").click()
el_test = self.driver.find_elements_by_id("middleName")
element = self.driver.find_elements_by_id("middleName")
while len(element) != 1:
element = self.driver.find_elements_by_id("middleName")
element[0].send_keys(self.middle_name)
element = self.driver.find_elements_by_xpath(
"//button[@class='button-continue'][.='Сохранить и продолжить']")
while len(element) != 0:
element[0].click()
element = self.driver.find_elements_by_xpath(
"//button[@class='button-continue'][.='Сохранить и продолжить']"
)
sleep(1)
element = self.driver.find_elements_by_class_name("stored-card-text")
if len(element) != 0:
iframeSwitch = self.driver.find_element_by_class_name("cvv")
self.driver.switch_to.frame(iframeSwitch)
self.driver.find_element_by_id("cardCvc-input").send_keys(
self.cardCvc)
else:
iframeSwitch = self.driver.find_element_by_class_name("newCard")
self.driver.switch_to.frame(iframeSwitch)
self.driver.find_element_by_id("cardNumber-input").send_keys(
self.cardNumber)
self.driver.find_element_by_id("cardExpiry-input").send_keys(
self.cardExpiry)
self.driver.find_element_by_id("cardCvc-input").send_keys(
self.cardCvc)
self.driver.switch_to.parent_frame()
element = self.driver.find_elements_by_xpath(
"//button[@class='button-continue'][.=' Продолжить ']")
while len(element) != 0:
try:
element[0].click()
except:
break
element = self.driver.find_elements_by_xpath(
"//button[@class='button-continue'][.=' Продолжить ']")
element = self.driver.find_elements_by_xpath(
"//button[@class='button-submit'][.=' Отправить заказ ']")
wait_time = self.drop_time - int(datetime.now().timestamp())
if wait_time < 0:
wait_time = 0
sleep(wait_time)
while len(element) != 0:
try:
element[0].click()
except:
break
element = self.driver.find_elements_by_xpath(
"//button[@class='button-submit'][.=' Отправить заказ ']")
print("Каеф")
def __init__(self, tag):
options = Options()
options.headless = True
self.tag = tag
self.bot = webdriver.Firefox(options=options)
def buy_ticket(args):
# FIREFOX DESTINATION FOLDER
folder = os.path.dirname(os.path.realpath(__file__))
# FIREFOX OPTIONS
options = Options();
options.headless = args.headless
# FIREFOX PROFILE
profile = FirefoxProfile()
profile.set_preference('browser.download.folderList',2);
profile.set_preference('browser.download.manager.showWhenStarting', False);
profile.set_preference('browser.helperApps.neverAsk.saveToDisk', 'application/octet-stream,application/vnd.ms-excel');
profile.set_preference('browser.download.dir',str(folder));
log_info('FirefoxProfile: "browser.download.dir" = "{}"'.format(str(folder)))
# Launch Firefox web browser.
driver = webdriver.Firefox(options=options, firefox_profile=profile)
# Use full screen mode.
# driver.maximize_window()
# Logging information
log_info('Opened {} (version {})'.format(
driver.capabilities['browserName'],
driver.capabilities['browserVersion']
)
)
# Sets a sticky timeout to implicitly wait for an element to be found, or a command to complete.
driver.implicitly_wait(30)
# Load page 'Vyhľadanie spojenia'
driver.get('https://ikvc.slovakrail.sk/esales/search')
# Info
log_info('Loading page "Vyhľadanie spojenia"')
try:
delay = 30 # wait seconds for web page to load, added more second
WebDriverWait(driver, delay).until(
EC.presence_of_all_elements_located(
(By.ID, 'searchPanel')
)
)
# Logging information
log_info('Loaded page "Vyhľadanie spojenia"')
except TimeoutException:
log_info('Loading took too much time.')
log_info('Closed {} (version {}).'.format(
driver.capabilities['browserName'],
driver.capabilities['browserVersion']
)
)
driver.close()
die('Page loading failure.',1)
sleep(1)
# Info
log_info('Page title is "{}".'.format(driver.title))
# Check if 'ZSSK' is in the page title
assert 'ZSSK' in driver.title
# FROM
elem_city_from = driver.find_element_by_id('fromInput')
elem_city_from.clear()
elem_city_from.send_keys(args.departure)
driver.find_element_by_xpath(
'/html/body/div[1]/div/div[2]/div[3]/div/div/form/div[1]/div/div[1]/div[1]/div[1]/ul/li/a'
).click
log_info('Filling "Odkiaľ" in "Vyhľadanie spojenia" with "{}".'.format(args.departure))
sleep(0.5)
# TO
elem_city_to = driver.find_element_by_id('toInput')
elem_city_to.clear()
elem_city_to.send_keys(args.arrival)
driver.find_element_by_xpath(
'/html/body/div[1]/div/div[2]/div[3]/div/div/form/div[1]/div/div[1]/div[1]/div[3]/ul/li/a'
).click
log_info('Filling "Kam" in "Vyhľadanie spojenia" with "{}".'.format(args.arrival))
sleep(0.5)
# DATE
elem_date = driver.find_element_by_id('departDate')
elem_date.clear()
elem_date.send_keys(args.date)
driver.find_element_by_xpath('//html').click();
log_info('Filling "Dátum cesty tam" in "Vyhľadanie spojenia" with "{}".'.format(args.date))
sleep(0.5)
# TIME
elem_time = driver.find_element_by_id('departTime')
elem_time.clear()
elem_time.send_keys(args.time)
driver.find_element_by_xpath('//html').click();
log_info('Filling "Odchod" in "Vyhľadanie spojenia" with "{}".'.format(args.time))
sleep(0.5)
log_info('Filled train credentials in "Vyhľadanie spojenia".')
# CONFIRM
driver.find_element_by_id('actionSearchConnectionButton').click()
log_info('Clicked on "Vyhľadať spojenie".')
sleep(2)
# CLICK ON FIRST
driver.find_element_by_css_selector(
'div.connection-group:nth-child(2) > div:nth-child(1)'
).click()
log_info('Clicked on first train.')
sleep(1)
# BUY TICKET
driver.find_element_by_xpath(
'//*[@id="dayGroupLoop:0:eSalesConnectionLoop:0:j_idt302"]'
).click()
log_info('Clicked on "Kúpiť lístok".')
sleep(1)
# PASSENGER TYPE SELECTION
driver.find_element_by_xpath(
'/html/body/div[1]/div/div[2]/div[3]/span/div/div[1]/form/div/div/div/div/div/div/div[1]/div[1]/div/div/div/div/a[1]/span[2]'
).click()
log_info('Choosing passenger type.')
sleep(1)
# JUNIOR SELECTION
driver.find_element_by_xpath(
'/html/body/div[1]/div/div[2]/div[3]/span/div/div[1]/form/div/div/div/div/div/div/div[1]/div[1]/div/div/div[1]/div/a[1]/ul/li[3]'
).click()
log_info('Selected "Mladý (16 - 25 r.)".')
sleep(1)
# DISCOUNT SELECTION
driver.find_element_by_xpath(
'/html/body/div[1]/div/div[2]/div[3]/span/div/div[1]/form/div/div/div/div/div/div/div[1]/div[1]/div/div/div/div/a[2]/span[2]'
).click()
log_info('Choosing card type.')
sleep(1)
# CARD SELECTION
driver.find_element_by_xpath(
'/html/body/div[1]/div/div[2]/div[3]/span/div/div[1]/form/div/div/div/div/div/div/div[1]/div[1]/div/div/div[1]/div/a[2]/ul/li[2]'
).click()
log_info('Selected "Preukaz pre žiaka/Študenta".')
sleep(1)
# ENABLED OPTION FOR FREE TICKET
driver.find_element_by_xpath(
'/html/body/div[1]/div/div[2]/div[3]/span/div/div[1]/form/div/div/div/div/div/div/div[1]/div[1]/div/div/div[2]/div/div/label'
).click()
log_info('Checkbox enabled for "Nárok na bezplatnú prepravu".')
sleep(1)
# CONTINUE
driver.find_element_by_xpath(
'//*[@id="actionIndividualContinue"]'
).click()
log_info('Clicked on "Pokračovať" at "Voľba cestujúcich".')
sleep(3)
# CONTINUE
driver.find_element_by_xpath(
'//*[@id="ticketsForm:connection-offer:final-price:j_idt198"]'
).click()
log_info('Clicked on "Pokračovať" at "Voľba cestovného lístka".')
sleep(1)
# CONTINUE
driver.find_element_by_xpath(
'//*[@id="ticketsForm:j_idt97"]'
).click()
log_info('Clicked on "Pokračovať" at "Doplnkové služby".')
sleep(1)
# CONTINUE
driver.find_element_by_xpath(
'//*[@id="cartForm:j_idt284"]'
).click()
log_info('Clicked on "Pokračovať" at "Obsah košíka (1)".')
sleep(1)
# LOAD PERSONAL INFORMATION
person = load_user_credentials('person.txt')
# FILL EMAIL
email = driver.find_element_by_id('email')
email.clear()
email.send_keys(person.email)
log_info('Filling "Váš e-mail" at "Osobné údaje (2)".')
# FILL NAME
name = driver.find_element_by_id('cartItemLoop:0:connectionPersonal:passengerLoop:0:firstname')
name.clear()
name.send_keys(person.name)
log_info('Filling "Meno" at "Osobné údaje (2)".')
# FILL SURNAME
surname = driver.find_element_by_id('cartItemLoop:0:connectionPersonal:passengerLoop:0:lastname')
surname.clear()
surname.send_keys(person.surname)
log_info('Filling "Priezvisko" at "Osobné údaje (2)".')
# FILL REGISTRATION NUMBER
card_number = driver.find_element_by_id('cartItemLoop:0:connectionPersonal:passengerLoop:0:cislo-registracie-p1')
card_number.clear()
card_number.send_keys(person.train_card)
log_info('Filling "Číslo registrácie:" at "Osobné údaje (2)".')
log_info('All personal informations filled at "Osobné údaje (2)".')
# CONTINUE
driver.find_element_by_xpath(
'//*[@id="j_idt177"]'
).click()
log_info('Clicked on "Pokračovať" at "Osobné údaje (2)".')
sleep(1)
# I AGREE WITH THE TERMS AND CONDITIONS
driver.find_element_by_xpath(
'/html/body/div[1]/div/div[2]/div[3]/div[2]/div/form/div/div/div[1]/div/div/div/label'
).click()
log_info('Checkbox enabled for "Súhlasím s obchodnými podmienkami " at "Výber platby (3)".')
sleep(1)
# CONTINUE
driver.find_element_by_xpath(
'//*[@id="j_idt107"]'
).click()
log_info('Clicked on "Pokračovať" at "Výber platby (3)".')
sleep(1)
# PAY
driver.find_element_by_xpath(
'//*[@id="cartForm:j_idt240"]'
).click()
log_info('Clicked on "Zaplatiť" at "Súhrn (4)".')
sleep(1)
"""
TODO
# DOWNLOAD PDF
log_info('DOWNLOAD: Clicked on "Uložiť lístok".')
driver.find_element_by_xpath(
'/html/body/div[1]/div/div[2]/div[3]/div[2]/div/form/div/div/div[3]/div[1]/a'
).click()
log_info('DOWNLOAD: PDF downloaded to "{}".'.format(str(folder)))
"""
# Waiting 10 seconds
log_info('Waiting 10 seconds before closing {} (version {}).'.format(
driver.capabilities['browserName'],
driver.capabilities['browserVersion']
)
)
sleep(10)
# Close the web browser (Firefox).
driver.close()
# Info
log_info('Closed {} (version {}).'.format(
driver.capabilities['browserName'],
driver.capabilities['browserVersion']
)
)
def paradox_polling():
options = Options()
options.headless = True
# Set screen resolution to 1366 x 768 like most 15" laptops
display = Display(visible=0, size=(1366, 768))
display.start()
driver = webdriver.Firefox(executable_path='/usr/local/bin/geckodriver',
options=options)
ipaddress = os.environ['PARADOX_IPADDRESS']
if not paradox_login(driver, ipaddress, os.environ['PARADOX_USERCODE'],
os.environ['PARADOX_PASSWORD']):
print('Login failed.')
driver.quit()
exit()
producer = boto3.client('firehose')
try:
#Getting Info
print('Getting info..')
driver.get('http://{}/index.html'.format(ipaddress))
sleep(2)
area_name = get_array_from_source('tbl_areanam', driver.page_source)
area_name = [x.replace('"', '') for x in area_name]
print('Area Name: {}'.format(area_name))
zone_name = get_array_from_source('tbl_zone', driver.page_source)
zone_name = [x.replace('"', '') for x in zone_name]
zone_name = [x.replace(' ', '_') for x in zone_name]
print('Zone Name: {}'.format(zone_name))
if len(area_name) == 0:
print('Server in use. Exit.')
driver.quit()
exit()
if 'KEYPRESS_CHECK' in os.environ:
print('Starting loop. Press ENTER to exit.')
stay = True
zone_status, last_zone_status = [], []
area_status, last_area_status = [], []
while stay:
sleep(1)
driver.get('http://{}/statuslive.html'.format(ipaddress))
zone_status = get_array_from_source('tbl_statuszone',
driver.page_source)
zone_status = [int(x) for x in zone_status]
area_status = get_array_from_source('tbl_useraccess',
driver.page_source)
area_status = [int(x) for x in area_status]
if (len(zone_status) == 0) or (len(area_status) == 0):
stay = False
else:
if (zone_status != last_zone_status) or (area_status !=
last_area_status):
print('Status Zone: {}'.format(zone_status))
print('Status Area: {}'.format(area_status))
firehose_record = {
'time':
datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")
}
for i in range(len(area_status)):
if area_status[i] != 0:
firehose_record['area.{}.{}'.format(
i + 1, area_name[i])] = area_status[i]
for i in range(len(zone_name) // 2):
if int(zone_name[i * 2]) != 0:
firehose_record['area.{}.zone.{}'.format(
zone_name[i * 2],
zone_name[i * 2 + 1])] = zone_status[i]
response = producer.put_record(
DeliveryStreamName=os.environ['KINESIS_STREAM'],
Record={'Data': json.dumps(firehose_record) + '\n'})
last_area_status = area_status
last_zone_status = zone_status
if 'KEYPRESS_CHECK' in os.environ:
if sys.stdin in select.select([sys.stdin], [], [], 0)[0]:
line = input()
break
finally:
print('Logout..')
driver.get('http://{}/logout.html'.format(ipaddress))
sleep(1)
driver.quit()
display, stop()
