def setup(self, headless=False, date_input=None):
# Validate date variable
if date_input is not None:
self.date = datetime.strptime(date_input, "%d.%m.%Y").date()
# Create directory if fetched folder is not available
if not os.path.exists(self.__download_dir):
os.mkdir(self.__download_dir)
# Set headless option and firefox profile
options = Options()
options.headless = headless
fp = webdriver.FirefoxProfile()
fp.set_preference("browser.download.folderList", 2)
fp.set_preference("browser.download.manager.showWhenStarting", False)
fp.set_preference("browser.download.dir", self.__download_dir)
fp.set_preference(
"browser.helperApps.neverAsk.saveToDisk",
"text/plain, application/vnd.ms-excel, text/csv, text/comma-separated-values, "
"application/octet-stream")
# Initialize Firefox() object to navigate
self.driver = Firefox(firefox_profile=fp, options=options)
return self
def main():
# Scrape url
URL = BASE_URL + "1&proposal=false&limit=0"
login_link = "https://iqtools-grocery-prod.intrtl.com/site/login"
link = "https://iqtools-grocery-prod.intrtl.com/validate?klass_id=5e7dc0691c007-3006&lots_id&size_id&confirm=&page=1&proposal=false&limit=0&klass_search="
options = webdriver.FirefoxOptions()
options.add_argument("--headless")
driver = Firefox(options = options)
try:
response = driver.request('POST', login_link, data={
#"_csrf-markup": "erEQFG7wJ6l9K9AHhbg_mLHT1aASSO_XzoY9DPPJgSogxChmJ7l1-h5Dvmb2_hLP3J74ynE62YGayV5LuKfHaA==",
"LoginForm[name]": "elina.ducohinskaya",
"LoginForm[password]": "8KGGpg14",
"LoginForm[rememberMe]": "0",
"login-button": ""
})
except TimeoutException as e:
pass
time.sleep(2)
print(response)
def browser(self, url):
try:
driver = Firefox(options=self.options)
driver.get(url)
return driver
except Exception as e:
raise SeleniumUatError(
f"Error setting instance of Firefox.\nError: {e}")
def browser(self, url):
try:
driver = Firefox(options=self.options)
headers = self.headers
result = driver.request(self.method, url, data=self.data)
import pry;pry()
return driver
except Exception as e:
raise SeleniumError(f"Error setting instance of Firefox.\nError: {e}")
def get_firefox():
caps = DesiredCapabilities().FIREFOX
options = FirefoxOptions()
options.add_argument("--headless")
caps["pageLoadStrategy"] = "eager" # interactive
profile = FirefoxProfile()
profile.set_preference("dom.disable_beforeunload", True)
browser = Firefox(desired_capabilities=caps,
firefox_profile=profile,
options=options)
browser.set_page_load_timeout(6)
return browser
def __init__(self):
self.options = Options()
self.options.set_preference("intl.accept_languages", 'en-us')
self.options.set_preference('useAutomationExtension', False)
self.options.set_preference('dom.webdriver.enabled', False)
self.driver = Firefox(options=self.options)
self.driver.execute_script(
"Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"
)
self.driver.get("https://amiunique.org/fp")
time.sleep(5)
self.driver.get("https://antcpt.com/score_detector/")
time.sleep(5)
class DefaultVisitorTest(unittest.TestCase):
def setUp(self):
self.browser = Firefox()
def tearDown(self):
self.browser.quit()
def test_homepage(self):
# Test Sample project homepage
self.browser.get('http://localhost:5000/NLP')
# Default title test
self.assertIn('Summary Project', self.browser.title)
def test_post(self):
test_json = {"opinions":
[
{
"id": 1,
"text": "The cats are really cute animals"
},
{
"id": 2,
"text": "The cutest pets are cats. They are my favorite animal."
},
{
"id": 3,
"text": "I am a dog lover and hate cats. I will never have a cat as a pet."
}
]
,
"n_ideas": 1
}
res = self.browser.request('POST', 'http://localhost:5000/NLP/NLP/Summary',
json=test_json)
self.assertEqual({'success': True, 'data': 'The cats are really cute animals.', 'message': ''}, res.json())
def main(args=None):
args = parse_args(args)
webdriver = Firefox()
webdriver.get('https://serverdensity.io')
response = webdriver.request('POST',
'{}/sessions'.format(DOMAIN),
data={
'username': args.username,
'password': args.password,
'MFAotpcode': 'blaap',
'MFAremember': 'false',
'accountName': args.username
})
print(response)
def get_driver():
"""Get Firefox driver
Returns# the Firefox driver object and handles the path.
"""
configure_log(log_path)
return Firefox(executable_path=gecko_path)
def webdriver_class():
profile = FirefoxProfile()
# Make sure Firefox WebDriver addon works, even if it could not be verified
profile.set_preference('xpinstall.signatures.required', False)
webdriver = Firefox(profile)
return webdriver
def check_ajax_page_for_msg(label, msg, url):
try:
# headers = {
# "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
# "Accept-Encoding":"gzip, deflate, br",
# "Accept-Language":"en-US,en;q=0.5",
# "Cache-Control":"no-cache",
# "Connection":"keep-alive",
# "Host":"www.nvidia.com",
# "Pragma":"no-cache",
# "Referer":"https://www.google.com/",
# "TE":"Trailers",
# "Upgrade-Insecure-Requests":"1",
# "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:80.0) Gecko/20100101 Firefox/80.0"
# }
webdriver = Firefox()
webdriver.implicitly_wait(30)
response = webdriver.request('GET', url)
response2 = webdriver.request('GET', 'https://www.nvidia.com/etc/designs/nvidiaGDC/clientlibs_foundation.min.3a16fd19562feeb504bb63525a249962.js')
webdriver.execute_script(response2.text)
# webdriver.find_element_by_class_name('availability')
# with open('clientlibs_foundation.min.3a16fd19562feeb504bb63525a249962.js') as f:
# webdriver.execute_script(f.read())
# element = WebDriverWait(webdriver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, "availability")))
wait_for_ajax(webdriver)
# webdriver.find_element_by_class_name('availability')
if response.status_code != 200:
send_sms_msg('Attention needed. Got a response other than 200 OK from ' + label + ' page.')
html = response.text
soup = BeautifulSoup(html, 'html.parser')
disclaimer_msg_present = msg in html
html_file = open(label + '.html', 'w', encoding='utf-8')
html_file.write(html)
html_file.close()
availability = soup.find('div', {'class', 'availability'})
print('Value of disclaimer_msg_present for ' + label + ' page: ' + str(disclaimer_msg_present))
if not disclaimer_msg_present:
send_sms_msg('ATTENTION! ' + label + ' page message has changed.')
print('\nResponse from ' + label + ' page link: ' + str(response.status_code) + '\n')
except Exception as e:
send_sms_msg('Exception occurred while sending request to Ironmaster website.')
print(e)
def make_webdriver():
capabilities = DesiredCapabilities.FIREFOX
# We're pinning to (outdated) Firefox 45.0.2 for now, which doesn't
# work with the new Marionette/geckodriver stuff - it uses webdriver
# instead. Disable marionette.
capabilities["marionette"] = False
return Firefox(capabilities=capabilities)
def _firefox(show):
# declare options
opt = firefox.options.Options()
if show is False:
opt.headless = True
browser = Firefox(options=opt,
executable_path='driver/linux/geckodriver',
log_path="log/geckodriver.log")
return browser
def set_up_browser():
profile = webdriver.FirefoxProfile()
profile.set_preference("dom.max_script_run_time", 600)
profile.set_preference("webdriver.log.file", "/tmp/firefox_console")
driver = Firefox(firefox_profile=profile)
driver.set_window_size(1920, 1080)
driver.implicitly_wait(LONG)
return driver
def __init__(self, login, password):
self.start_display()
profile = webdriver.FirefoxProfile()
profile.set_preference('browser.download.folderList', 2) # custom location
profile.set_preference('browser.download.manager.showWhenStarting', False)
profile.set_preference('browser.download.dir', current_dir)
profile.set_preference('browser.helperApps.neverAsk.saveToDisk', "text/csv,application/vnd.ms-excel")
#profile.set_preference('browser.helperApps.neverAsk.saveToDisk', "text/plain")
self.driver = Firefox(profile)
self.login(login, password)
self._login = login
self._password = password
def get_driver(self):
profile = FirefoxProfile()
profile.set_preference('browser.download.folderList', 2)
profile.set_preference('browser.download.manager.showWhenStarting',
False)
profile.set_preference('browser.download.dir', TEMP_PATH)
profile.set_preference(
'browser.helperApps.neverAsk.saveToDisk',
('application/csv,text/csv,application/vnd.ms-excel,'
'application/x-msexcel,application/excel,'
'application/x-excel,text/comma-separated-values'))
driver = Firefox(firefox_profile=profile, executable_path=GECKO_PATH)
return driver
def setUpClass(cls):
super().setUpClass()
try:
""" Instantiate selenium driver instance """
binary = FirefoxBinary('/usr/bin/firefox')
opts = FirefoxOptions()
opts.add_argument("--headless")
executable_path = GeckoDriverManager().install()
cls.selenium = Firefox(firefox_binary=binary,
firefox_options=opts,
executable_path=executable_path)
cls.selenium.implicitly_wait(10)
except Exception as e:
logger.error(e)
def init_driver(interactive):
"""
Initializes the (Firefox) driver.
:param interactive: whether to run in interactive mode (ie displaying the browser), required for when off-campus
due to the interactive 2FA
:type interactive: bool
:return: the driver instance
:rtype: webdriver.Firefox
"""
logger().debug("initializing driver (interactive=%s)" % str(interactive))
options = webdriver.FirefoxOptions()
options.headless = not interactive
return Firefox(options=options)
def connect(browser='firefox', session=None):
from seleniumrequests import Firefox
driver = Firefox()
driver.get("https://www.backstabbr.com/")
# Restore previous session if specified
if session:
for cookie in pickle.load(open(session, "rb")):
driver.add_cookie(cookie)
return driver
def driver():
use_chromedriver = True
if use_chromedriver:
capabilities = DesiredCapabilities.CHROME
capabilities['loggingPrefs'] = {
'browser': 'ALL',
# 'driver': 'ALL',
}
options = webdriver.ChromeOptions()
options.add_argument('headless')
wd = Chrome(options=options, desired_capabilities=capabilities)
else:
options = webdriver.FirefoxOptions()
options.add_argument('--headless')
wd = Firefox(firefox_options=options)
yield wd
if use_chromedriver:
print('Browser', wd.get_log('browser'))
# print('Driver', wd.get_log('driver'))
wd.quit()
def load_contact_list(driver: Firefox, limit: int):
contacts = []
total_contacts = 0
offset = 0
first = True
while offset < total_contacts or first:
first = False
data = contact_list_query
data["variables"]["limit"] = limit
data["variables"]["offset"] = offset
response = driver.request('POST', xing_api, data=json.dumps(data),
headers={'content-type': 'application/json', 'Accept': '*/*'})
if response.status_code == 200:
data = json.loads(response.content)
network = data['data']['viewer']['contactsNetwork']
total_contacts = network['total']
contacts = contacts + network['collection']
offset = len(contacts)
else:
print('Contact fetch failed')
exit(1)
print(f"{offset}/{total_contacts}")
return contacts
def get_driver(debug: bool = False):
"""
Returns an alive Firefox WebDriver
"""
global driver
### webdriver with features from the normal requests lib
from seleniumrequests import Firefox
### we need options to start a headless firefox instance
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.remote.command import Command
is_alive = None
try:
driver.execute(Command.STATUS)
is_alive = True
except:
is_alive = False
if not is_alive:
browser_options = Options()
browser_options.add_argument('--headless')
browser_options.add_argument('--window-size=1920x1080')
driver = Firefox(options=browser_options,
executable_path=geckodriver_location)
### load existing cookies
if cookies_path.exists():
driver.get(urls['login'])
if debug:
dprint("Found existing cookies. Attempting to reuse session...")
import pickle
with open(cookies_path, 'rb') as cookies_file:
cookies = pickle.load(cookies_file)
for cookie in cookies:
driver.add_cookie(cookie)
return driver
# Import any WebDriver class that you would usually import from
# selenium.webdriver from the seleniumrequests module
import sys
from seleniumrequests import Firefox
url = sys.argv[1]
# Simple usage with built-in WebDrivers:
webdriver = Firefox()
response = webdriver.request(
'GET', '%s/xss.php?xss=<script>document.write(INJECTX)</script>' % url)
if '<script>document.write(INJECTX)</script>' in response.text:
print("Vulnerable!")
print(response.text)
webdriver.quit()
SECONDARY_COMMANDS = ''
'firstname': xing_id['firstName'],
'lastname': xing_id['lastName'],
'contact_since': raw['contactCreatedAt'],
'note': raw['memo'],
'org': xing_id['profileOccupation']['occupationOrg'],
'title': xing_id['profileOccupation']['occupationTitle']
}
if contact['note'] is None:
contact['note'] = ''
results.append(contact)
return results
all_contacts = []
with contextlib.closing(Firefox()) as driver:
wait = ui.WebDriverWait(driver, 300)
driver.get('https://www.xing.com/settings')
wait.until(lambda driver: str(driver.current_url).startswith(start_url))
raw_contacts = load_contact_list(driver, 25)
all_contacts = parse_contacts(raw_contacts)
workbook = xlsxwriter.Workbook('XingNetwork.xlsx')
contacts_sheet = workbook.add_worksheet('Contacts')
col_width = [0] * 5 + [12]
cols = []
for col in ['Name', 'First Name', 'Organisation', 'Title', 'Note', 'Xing-Profile']:
cols.append({'header': col})
args = parser.parse_args()
my_opt.URL = args.url
print args
if not args.url:
print 'You must specify a URL.', sys.argv[0], '--help for more details'
exit(1)
if args.auth == True:
print 'link to the next bit.', sys.argv[1], '--help for more details'
auth(args.auth)
return args
# ['-a', '-u','http://192.168.127.178/dvwa/login.php']
# load what you want to test with (change to phantomjs later)
driver = Firefox()
driver.set_window_size(1000,500)
# uncomment to reparse args
# my_opt()
# args-url location
# driver.get(my_opt.URL)
#Testing Delete later
#Testing Auth____ Comment IN TO autologin Bwap
driver.get("http://192.168.127.144/bWAPP/login.php")
select = Select(driver.find_element_by_name('security_level'))
select.select_by_visible_text("medium")
username = driver.find_element_by_id("login")
password = driver.find_element_by_name("password")
submit = driver.find_element_by_xpath("//button[@type='submit']")
from seleniumrequests import Firefox
import base64
from PIL import Image
from captcha2upload import CaptchaUpload
captcha_fn = "captcha.png"
driver = Firefox()
#driver.set_page_load_timeout(5);
response = driver.get(
"http://www4.tjrj.jus.br/consultaProcessoWebV2/consultaProc.do?v=2&FLAGNOME=&back=1&tipoConsulta=publica&numProcesso=2020.002.017258-3"
)
def get_captcha():
element = driver.find_element_by_id(
"imgCaptcha") # element name containing the catcha image
location = element.location
size = element.size
driver.save_screenshot("temp.png")
x = location['x']
y = location['y']
w = size['width']
h = size['height']
width = x + w
height = y + h
im = Image.open('temp.png')
im = im.crop((int(x), int(y), int(width), int(height)))
im.save(captcha_fn)
if i==1000000:
URL=URL[:-6]+str(i)
else:
URL=URL[:-7]+str(i)
return URL
txt_file=open('chbio_cos.txt', 'w', encoding='utf-8')
j=1
while(j<=2235):
# web_driver=webdriver.Chrome('D:\scrapper\web_driver\chromedriver.exe')
web_driver=Firefox()
# web_driver.get(url)
web_driver.request('POST', url, data={"tableId": "68", "State":"1", "bcId":"138009396676753955941050804482")
bs_tmp=BeautifulSoup(web_driver.page_source, 'lxml')
the_list=bs_tmp.find_all('a')
for i in the_list:
print(i.get_text())
txt_file.write(i.get_text()+'\n')
# web_driver.find_element_by_name('goInt').clear()
# web_driver.find_element_by_name('goInt').send_keys(str(j))
# web_driver.find_element_by_xpath("//div[@id='content']/div/table[4]/tbody/tr/td[7]/input").click()
j+=1
url=chkURL(j, url)
web_driver.quit()
print(ay)
print(cb)
print(jb)
print(ri)
pass
if __name__ == "__main__":
url_prefix = "http://www.hshfy.sh.cn/shfy/gweb/flws_list_content.jsp"
page_id = 1
while True:
print("Current page: %d" % page_id)
url = url_prefix
webdriver = Firefox()
response = webdriver.request('POST', url_prefix, data={'fydm':'200', 'ajlb':'%E6%B0%91%E4%BA%8B', 'pagesnum':'2'})
r_read = response.text
soup = BeautifulSoup(r_read)
info_soup = soup.findAll('tr', attrs={"style":"cursor:hand"})
if info_soup:
print("LEN: %d" %(len(info_soup)))
for item in info_soup:
track_info(item)
webdriver.close()
page_id += 1
if page_id > 234:
break
# Import any WebDriver class that you would usually import from
# selenium.webdriver from the seleniumrequests module
import sys
from seleniumrequests import Firefox
url = sys.argv[1]
# Simple usage with built-in WebDrivers:
webdriver = Firefox()
response = webdriver.request(
'GET',
'%s/www/delivery/afr.php?refresh=10000&")\',10000000);document.write(1+1336);setTimeout(\'alert("'
% url)
if '1337' in response.text:
print("Vulnerable!")
webdriver.quit()
# http://propertydata.orangecountygov.com/imate/viewlist.aspx?sort=printkey&swis=3311
# get this cookie:
cookie_url = 'http://www.co.orange.ny.us/content/124/1368/4136.aspx'
towncode = str(swis)[0:4]
search = '/'.join([BASE_URL, 'viewlist..aspx?sort=printkey&swis={towncode}'])
# SWS = 332489; first four are town code
# 02200000090030000000
printkey = str(pin).strip(str(swis))
SEARCH_URL = ''.join([BASE_URL, '/propdetail.aspx?'])
prop_search = '&'.join(['swis={}'.format(swis),
'printkey={}'.format(printkey)])
full_url = SEARCH_URL + prop_search
# create driver
driver = Firefox()
driver.get(cookie_url)
link = driver.find_element_by_class_name('ApplyClass')
link.click()
response = driver.get(full_url)
response = driver.request('GET', full_url)
print(response)
# driver.save_screenshot(''.join([pin,'.png']))
# property_info = driver.find_element_by_id('pnlRTaxID')
# property_info.screenshot(''.join([pin,'.png']))
from seleniumrequests import Firefox
from bs4 import BeautifulSoup as bs
import json
driver = Firefox()
movie_links = []
data = {}
data['movie'] = []
MAX_NO_FILMS = 250
imdb_url = 'https://www.imdb.com/chart/top?ref_=nv_mv_250'
class Scarper:
def __init__(self, url):
self.url = url
self.response = ''
def load_url(self):
self.response = driver.request('GET', self.url)
def get_movie_links(self):
raw_data = bs(self.response.text, 'lxml')
list_of_movies = raw_data.find('tbody', {'class' : 'lister-list'})
movies = list_of_movies.findAll('tr')
for movie in movies:
movie_details = movie.findAll('td')
movie_links.append(movie_details[0].find('a')['href'])
def get_movie_details(self):
for i in range(0,250):
print((i+1),'/',250,end='\r')
movie_detail_response = driver.request('GET', 'https://www.imdb.com/'+movie_links[i])
#parse the html response
def test_task_journey(driver: Firefox, test_app):
#check board created
driver.get(_TODO_URL)
assert driver.title == 'To-Do App'
#check item added
elem_form_task_title = driver.find_element_by_name("name")
elem_form_task_title.send_keys('my very own task')
elem_form_task_due = driver.find_element_by_name("due")
elem_form_task_due.send_keys('07/07/2022')
elem_form_task_desc = driver.find_element_by_name("desc")
elem_form_task_desc.send_keys('a test description')
driver.find_element_by_id("todoform").submit()
driver.get(_TODO_URL)
driver.refresh()
assert ('a test description' in driver.page_source)
#check item status change to Doing
select = Select(driver.find_element_by_id('status_select_1'))
select.select_by_index(1)
driver.get(_TODO_URL)
driver.refresh()
assert ('/static/status_1.png' in driver.page_source)
#check item deleted
driver.find_element_by_id("delete_1").click()
driver.get(_TODO_URL)
driver.refresh()
assert ('a test description' not in driver.page_source)
def driver():
os.environ['MOZ_HEADLESS'] = '1'
with Firefox() as driver:
yield driver
def start():
list = select_all()
for date in list:
login = date['login']
password = date['password']
try:
print('Запускаю Браузер')
options = Options()
options.headless = True
browser = Firefox(options=options)
browser.delete_all_cookies()
browser.set_window_position(0, 0)
browser.set_window_size(1024, 1024)
browser.get('https://www.next.co.uk/secure/account/Login')
login_input = browser.find_element_by_id('EmailOrAccountNumber')
ActionChains(browser).move_to_element(login_input).perform()
paswword_input = browser.find_element_by_id('Password')
ActionChains(browser).move_to_element(paswword_input).perform()
paswword_input.send_keys(random.choice(rany))
login_input.send_keys(login)
paswword_input.send_keys(password)
paswword_input.send_keys(random.choice(rany))
paswword_input.send_keys(random.choice(rany))
paswword_input.send_keys(Keys.ENTER)
time.sleep(random.uniform(3, 5))
soup = BeautifulSoup(browser.page_source, 'lxml')
titleTag = soup.find('title').text
print(titleTag)
while titleTag == 'Access Denied':
browser.quit()
print('close')
proxy_list = choice(get_proxy())
proxy_host = proxy_list['ip']
proxy_port = int(proxy_list['port'])
print(proxy_host, proxy_port)
options = Options()
options.headless = True
fp = webdriver.FirefoxProfile()
fp.set_preference("browser.privatebrowsing.autostart", True)
fp.set_preference("network.proxy.type", 1)
fp.set_preference("network.proxy.http", proxy_host)
fp.set_preference("network.proxy.http_port", proxy_port)
fp.set_preference("network.proxy.https", proxy_host)
fp.set_preference("network.proxy.https_port", proxy_port)
fp.set_preference("network.proxy.ssl", proxy_host)
fp.set_preference("network.proxy.ssl_port", proxy_port)
fp.set_preference("network.proxy.ftp", proxy_host)
fp.set_preference("network.proxy.ftp_port", proxy_port)
fp.set_preference("network.proxy.socks", proxy_host)
fp.set_preference("network.proxy.socks_port", proxy_port)
fp.update_preferences()
print('open browser')
browser = Firefox(options=options, firefox_profile=fp)
browser.set_window_position(0, 0)
browser.set_window_size(1024, 648)
browser.get('https://api.ipify.org/')
test_ip = browser.find_element_by_tag_name('pre').text
print(test_ip + ' ip полученное с сайта')
print(proxy_list['ip'] + ' ip полученное с прокси')
if test_ip == proxy_list['ip']:
browser.delete_all_cookies()
browser.get('https://www.next.co.uk/secure/account/Login')
login_input = browser.find_element_by_id(
'EmailOrAccountNumber')
ActionChains(browser).move_to_element(
login_input).perform()
paswword_input = browser.find_element_by_id('Password')
ActionChains(browser).move_to_element(
paswword_input).perform()
paswword_input.send_keys(random.choice(rany))
login_input.send_keys(login)
paswword_input.send_keys(password)
paswword_input.send_keys(random.choice(rany))
paswword_input.send_keys(random.choice(rany))
paswword_input.send_keys(Keys.ENTER)
time.sleep(random.uniform(3, 5))
soup = BeautifulSoup(browser.page_source, 'lxml')
titleTag = soup.find('title').text
print(titleTag)
browser.get('https://www2.next.co.uk/shoppingbag')
time.sleep(random.uniform(3, 5))
r = browser.request('GET', basket)
write_log(r.json())
data = browser.request('GET', basket).json()
coint_bug = len(data["ShoppingBag"]['Items'])
a = coint_bug - 1
while coint_bug >= 1:
message = data["ShoppingBag"]['Items'][a]["StockMessage"]
thing = data["ShoppingBag"]['Items'][a]["Description"]
items = data["ShoppingBag"]['Items'][a]["ItemNumber"]
size = data["ShoppingBag"]['Items'][a]["SizeDescription"]
if "In Stock" in message:
coint_bug = coint_bug - 1
a = a - 1
text_telegram = 'в этом логине нашлось ' + login + ': ' + thing + ' ' + items + ' ' + size
web_get = tgapi + '/sendmessage?chat_id={}&text={}'.format(
chat_id, text_telegram)
requests.get(web_get)
print(message)
else:
coint_bug = coint_bug - 1
a = a - 1
print(login)
print(datetime.datetime.now())
browser.quit()
except Exception as e:
with open('error.txt', 'a') as f:
f.write(str(e) + '\n')
class Sunny(object):
def __init__(self, login, password):
self.start_display()
profile = webdriver.FirefoxProfile()
profile.set_preference('browser.download.folderList', 2) # custom location
profile.set_preference('browser.download.manager.showWhenStarting', False)
profile.set_preference('browser.download.dir', current_dir)
profile.set_preference('browser.helperApps.neverAsk.saveToDisk', "text/csv,application/vnd.ms-excel")
#profile.set_preference('browser.helperApps.neverAsk.saveToDisk', "text/plain")
self.driver = Firefox(profile)
self.login(login, password)
self._login = login
self._password = password
def start_display(self):
self.display = Display(visible=0, size=(800, 600))
self.display.start()
def close(self):
self.driver.close()
self.display.stop()
def login(self, login=None, password=None):
"""Login on the Sunny portal website using the credentials
Parameters
----------
login: str
The login credential to sunnyportal
password: str
The password credential of sunnyportal
"""
if not login:
login = self._login
password = self._password
self.driver.get("https://www.sunnyportal.com/Templates/Start.aspx?ReturnUrl=%2f")
self.driver.find_element_by_id("txtUserName").clear()
self.driver.find_element_by_id("txtUserName").send_keys(login)
self.driver.find_element_by_id("txtPassword").clear()
self.driver.find_element_by_id("txtPassword").send_keys(password)
self.driver.find_element_by_id("ctl00_ContentPlaceHolder1_Logincontrol1_LoginBtn").click()
#time.sleep(0.5)
def wait_n_get(self, element_type, value):
""" Wait for an element to be present and get it
Paramters
---------
element_type: By.ID | By.LINK_TEXT...
The type of value to identify the element to get
value: str
the value describing the element to get
Returns
-------
el: element
The driver element requested
"""
return WebDriverWait(self.driver, TIME_DELAY).until(EC.presence_of_element_located((element_type, value)))
def goto(self, n_house):
"""Go to the page of an house given it's number, from the plant list page
Parameters
----------
n_house: int
The number of the house to go to
"""
el = self.wait_n_get(By.LINK_TEXT, houses[n_house])
el.click()
def goto_2(self, n_house):
"""Go to a house from the plant pannel on the Dashboard page
Parameters
----------
n_house: int
The number of the house to go to
"""
self.wait_n_get(By.CLASS_NAME, 'plantselect').click()
self.wait_n_get(By.LINK_TEXT, houses[n_house]).click()
def hover_over(self, id):
"""Hover over an element of the page given its id
Parameter
---------
id: str
The id of the element to hover over
"""
el = self.wait_n_get(By.ID, id)
hover = ActionChains(self.driver).move_to_element(el)
hover.perform()
def click(self, id):
"""Click on an element of the page given its id
Parameter
---------
id: str
The id of the element to click on
"""
el = self.wait_n_get(By.ID, id)
el.click()
def select_date(self, day, month, year):
id_date = 'ctl00_ContentPlaceHolder1_UserControlShowDashboard1_UserControlShowEnergyAndPower1__datePicker_textBox'
id_before = 'ctl00_ContentPlaceHolder1_UserControlShowDashboard1_UserControlShowEnergyAndPower1_btn_prev'
id_after = 'ctl00_ContentPlaceHolder1_UserControlShowDashboard1_UserControlShowEnergyAndPower1_btn_next'
try:
el = self.wait_n_get(By.ID, id_date)
self.driver.execute_script('$("#%s").val("%d/%d/%d")'%(id_date, month, day, year))
sleep(0.2)
self.click(id_before)
sleep(0.2)
self.click(id_after)
sleep(0.2)
except Exception as e:
if "Element is not clickable at point" in str(e):
print(e)
print('trying again!')
self.select_date(day, month, year)
def download(self, day=None, month=None, year=None):
"""Download the CSV file
"""
# Make sure we see the "Day" pannel
tabactive = self.wait_n_get(By.CLASS_NAME, 'tabactive')
if not tabactive.text == 'Day':
self.click(id_day)
# Select the right day
if day:
self.select_date(day, month, year)
# Hover over the download button
try:
self.hover_over(id_hover)
self.click(id_click)
except Exception as e_1:
# Check if the data is available for that day by looking for the info bubble
try:
el = self.wait_n_get(By.ID, id_info)
if 'info.png' in el.get_attribute('src'):
print('no data available for this day')
return None
else:
# Not sure what just happen there
raise(e_1)
except Exception as e_2:
if 'Unable to locate element' in str(e_2):
# The info icon isn't available
print(e_2)
raise(e_1)
else:
# Not sure what just happen there
print(e_1)
print(e_2)
#raise (e1, e2)
# Download the data for the day
res = self.driver.request('GET', url_data_graph)
if res.status_code == 200:
print('sucess')
else:
raise Exception('Error:', res.text)
return res
def download_house(self, n, day=None, month=None, year=None):
""" Download the house power production of the day
Parameters
----------
driver: WebDriver
The WebDriver instance to action
n_house: int
The number of the house to go to
Return
------
df: pandas.DataFrame | None
A dataframe containing the house day power production, or None if there isn't any data available
"""
try:
# Check what is the starting point
if 'Start.aspx' in self.driver.current_url:
# We are on the login screen, we first need to login
print('-- login in main screen')
self.login()
print('-- accessing house', n)
self.goto(n)
elif 'sunnyportal.com/Plants' in self.driver.current_url:
# We are on the plant list, lets
self.goto(n)
elif 'sunnyportal.com/FixedPages/Dashboard.aspx' in self.driver.current_url:
# We are on a dashboard, so we should be able to click on the left hand pannel to go to the new house
self.goto_2(n)
else:
# No idea where we are
raise Exception('I dont know where we are:', self.driver.current_url)
print('-- downloading house', n, 'power data')
res = self.download(day, month, year)
self.date = self.wait_n_get(By.ID, id_date).get_attribute('value')
if day:
if not self.date == "%d/%d/%d"%(month, day, year):
print('Error the date wasnt fixed correctly: '+self.date)
if res:
# There seems to be a positive response, so let's put it in a pandas dataframe
df = pd.read_csv(StringIO(res.text), sep=';', names=['power', 'avg'], skiprows=1)
print('-- download sucessful')
return df
else:
print('-- download failed')
# No response, we return a None object
return res
except Exception as e_1:
# Something whent wrong
try:
# Check if sunny portal has banned us for some time
text = self.wait_n_get(By.ID, 'ctl00_ContentPlaceHolder1_Logincontrol1_DivLogin').text
if 'Login failed! Login will be blocked for' in text:
# It does seem like we have been banned for some time
print(text)
n_sec = int(text.split('for')[1].split(' seconds')[0])
print('going to sleep for %d sec'%(n_sec))
time.sleep(n_sec)
print('retrying this house')
return self.download_house(n, day, month, year)
except Exception as e_2:
# I don't know what went wrong
print(e_1)
print(e_2)
raise(e_1)
def img(self):
"""A simple screenshot function to show on the notebook"""
return Image(self.driver.get_screenshot_as_png())
def download_all(self, day=None, month=None, year=None):
df_dict = {}
for k, v in houses.items():
print(k)
df = self.download_house(k, day, month, year)
if isinstance(df, pd.DataFrame):
df_dict['House %d'%(k)] = df
# Save the data into a DataFrame
self.data = pd.DataFrame({k:v.power for k, v in df_dict.items() if isinstance(v, pd.DataFrame)}, index=df.index)
# Save the data into a file
m,d,y = self.date.split('/')
self.data.to_csv('svalin_%s_%s_%s.csv'%(d,m,y))
return self.data