def test_task_journey(driver: Firefox, test_app):
#check board created
driver.get(_TODO_URL)
assert driver.title == 'To-Do App'
#check item added
elem_form_task_title = driver.find_element_by_name("name")
elem_form_task_title.send_keys('my very own task')
elem_form_task_due = driver.find_element_by_name("due")
elem_form_task_due.send_keys('07/07/2022')
elem_form_task_desc = driver.find_element_by_name("desc")
elem_form_task_desc.send_keys('a test description')
driver.find_element_by_id("todoform").submit()
driver.get(_TODO_URL)
driver.refresh()
assert ('a test description' in driver.page_source)
#check item status change to Doing
select = Select(driver.find_element_by_id('status_select_1'))
select.select_by_index(1)
driver.get(_TODO_URL)
driver.refresh()
assert ('/static/status_1.png' in driver.page_source)
#check item deleted
driver.find_element_by_id("delete_1").click()
driver.get(_TODO_URL)
driver.refresh()
assert ('a test description' not in driver.page_source)
def start():
list = select_all()
for date in list:
login = date['login']
password = date['password']
try:
print('Запускаю Браузер')
options = Options()
options.headless = True
browser = Firefox(options=options)
browser.delete_all_cookies()
browser.set_window_position(0, 0)
browser.set_window_size(1024, 1024)
browser.get('https://www.next.co.uk/secure/account/Login')
login_input = browser.find_element_by_id('EmailOrAccountNumber')
ActionChains(browser).move_to_element(login_input).perform()
paswword_input = browser.find_element_by_id('Password')
ActionChains(browser).move_to_element(paswword_input).perform()
paswword_input.send_keys(random.choice(rany))
login_input.send_keys(login)
paswword_input.send_keys(password)
paswword_input.send_keys(random.choice(rany))
paswword_input.send_keys(random.choice(rany))
paswword_input.send_keys(Keys.ENTER)
time.sleep(random.uniform(3, 5))
soup = BeautifulSoup(browser.page_source, 'lxml')
titleTag = soup.find('title').text
print(titleTag)
while titleTag == 'Access Denied':
browser.quit()
print('close')
proxy_list = choice(get_proxy())
proxy_host = proxy_list['ip']
proxy_port = int(proxy_list['port'])
print(proxy_host, proxy_port)
options = Options()
options.headless = True
fp = webdriver.FirefoxProfile()
fp.set_preference("browser.privatebrowsing.autostart", True)
fp.set_preference("network.proxy.type", 1)
fp.set_preference("network.proxy.http", proxy_host)
fp.set_preference("network.proxy.http_port", proxy_port)
fp.set_preference("network.proxy.https", proxy_host)
fp.set_preference("network.proxy.https_port", proxy_port)
fp.set_preference("network.proxy.ssl", proxy_host)
fp.set_preference("network.proxy.ssl_port", proxy_port)
fp.set_preference("network.proxy.ftp", proxy_host)
fp.set_preference("network.proxy.ftp_port", proxy_port)
fp.set_preference("network.proxy.socks", proxy_host)
fp.set_preference("network.proxy.socks_port", proxy_port)
fp.update_preferences()
print('open browser')
browser = Firefox(options=options, firefox_profile=fp)
browser.set_window_position(0, 0)
browser.set_window_size(1024, 648)
browser.get('https://api.ipify.org/')
test_ip = browser.find_element_by_tag_name('pre').text
print(test_ip + ' ip полученное с сайта')
print(proxy_list['ip'] + ' ip полученное с прокси')
if test_ip == proxy_list['ip']:
browser.delete_all_cookies()
browser.get('https://www.next.co.uk/secure/account/Login')
login_input = browser.find_element_by_id(
'EmailOrAccountNumber')
ActionChains(browser).move_to_element(
login_input).perform()
paswword_input = browser.find_element_by_id('Password')
ActionChains(browser).move_to_element(
paswword_input).perform()
paswword_input.send_keys(random.choice(rany))
login_input.send_keys(login)
paswword_input.send_keys(password)
paswword_input.send_keys(random.choice(rany))
paswword_input.send_keys(random.choice(rany))
paswword_input.send_keys(Keys.ENTER)
time.sleep(random.uniform(3, 5))
soup = BeautifulSoup(browser.page_source, 'lxml')
titleTag = soup.find('title').text
print(titleTag)
browser.get('https://www2.next.co.uk/shoppingbag')
time.sleep(random.uniform(3, 5))
r = browser.request('GET', basket)
write_log(r.json())
data = browser.request('GET', basket).json()
coint_bug = len(data["ShoppingBag"]['Items'])
a = coint_bug - 1
while coint_bug >= 1:
message = data["ShoppingBag"]['Items'][a]["StockMessage"]
thing = data["ShoppingBag"]['Items'][a]["Description"]
items = data["ShoppingBag"]['Items'][a]["ItemNumber"]
size = data["ShoppingBag"]['Items'][a]["SizeDescription"]
if "In Stock" in message:
coint_bug = coint_bug - 1
a = a - 1
text_telegram = 'в этом логине нашлось ' + login + ': ' + thing + ' ' + items + ' ' + size
web_get = tgapi + '/sendmessage?chat_id={}&text={}'.format(
chat_id, text_telegram)
requests.get(web_get)
print(message)
else:
coint_bug = coint_bug - 1
a = a - 1
print(login)
print(datetime.datetime.now())
browser.quit()
except Exception as e:
with open('error.txt', 'a') as f:
f.write(str(e) + '\n')
)
def get_captcha():
element = driver.find_element_by_id(
"imgCaptcha") # element name containing the catcha image
location = element.location
size = element.size
driver.save_screenshot("temp.png")
x = location['x']
y = location['y']
w = size['width']
h = size['height']
width = x + w
height = y + h
im = Image.open('temp.png')
im = im.crop((int(x), int(y), int(width), int(height)))
im.save(captcha_fn)
get_captcha()
captcha = CaptchaUpload("hvxrw69kkdcpfzjt38jtm2bxyzr7ypvn")
captcha_resolved = captcha.solve(captcha_fn)
driver.find_element_by_id("captcha").send_keys(captcha_resolved)
driver.find_element_by_xpath(
'//*[@id="container_captcha"]/fieldset/table/tbody/tr[2]/td[2]/input'
).click()
GMF_BAND_ID = 79162159
JBS_BAND_ID = 79463189
browser = Firefox()
browser.get('https://auth.band.us/email_login')
browser.implicitly_wait(10)
textBox = browser.find_element_by_xpath('//*[@id="email_login_form"]')
textBox.find_element_by_id('input_email').send_keys('', Keys.RETURN)
time.sleep(2.5)
browser.find_element_by_id('pw').send_keys('', Keys.RETURN)
time.sleep(5)
browser.get('https://band.us/band/{}/create-live'.format(JBS_BAND_ID))
time.sleep(5)
browser.find_element_by_xpath(
'/html/body/div[1]/section/div/div[1]/div/div[2]/button[1]').click()
browser.find_element_by_xpath(
'/html/body/div[1]/section/div/div[2]/div[3]/div[4]/div/button').click()
button = browser.find_element_by_xpath(
'/html/body/div[1]/section/div/div[2]/div[3]/div[3]/div/button')
action = ActionChains(browser)
action.move_to_element(button)
def atc():
browser = Firefox()
browser2 = Firefox()
getLink()
link = "https://yeezysupply.com/products/womens-tubular-boot-pvc-transparent"
atc = "https://yeezysupply.com/cart/add.js"
size = variants[input("Enter size: ")]
payload = {"quantity": "1", "id": size}
input("Press Enter load and add to cart...")
# -------------- Go to link and ATC---------------
browser.get(link)
response = browser.request('POST', atc, data=payload)
browser.get("https://yeezysupply.com/cart")
browser.execute_script(
"document.getElementsByClassName('K__button CA__button-checkout')[0].click();"
)
browser2.get(link)
response = browser2.request('POST', atc, data=payload)
browser2.get("https://yeezysupply.com/cart")
browser2.execute_script(
"document.getElementsByClassName('K__button CA__button-checkout')[0].click();"
)
# -------------- Go to shipping --------------
input("CONTINUE TO SHIPPING...")
for i in checkoutPayload:
inputMsg = browser.find_element_by_id(i[0])
inputMsg.send_keys(i[1])
mySelect = Select(
browser.find_element_by_id("checkout_shipping_address_province"))
mySelect.select_by_value('Maryland')
browser.execute_script(
"document.getElementsByClassName('step__footer__continue-btn btn')[0].click();"
)
for i in checkoutPayload:
inputMsg = browser2.find_element_by_id(i[0])
inputMsg.send_keys(i[1])
mySelect = Select(
browser2.find_element_by_id("checkout_shipping_address_province"))
mySelect.select_by_value('Maryland')
browser2.execute_script(
"document.getElementsByClassName('step__footer__continue-btn btn')[0].click();"
)
# -------------- Go to payment --------------
input("CONTINUE TO PAYMENT METHOD...")
browser.execute_script(
"document.getElementsByClassName('step__footer__continue-btn btn')[0].click();"
)
browser2.execute_script(
"document.getElementsByClassName('step__footer__continue-btn btn')[0].click();"
)
# -------------- Fill card --------------
input("FILL CREDIT CARD...")
eachFrame = 0
for i in creditCard:
frame = browser.find_elements_by_xpath(
'//iframe[@frameborder="0"]')[eachFrame]
browser.switch_to.frame(frame)
inputMsg = browser.find_element_by_id(i[0])
for e in range(0, len(i)):
inputMsg.send_keys(i[e])
browser.switch_to.default_content()
eachFrame += 1
eachFrame = 0
for i in creditCard:
frame = browser2.find_elements_by_xpath(
'//iframe[@frameborder="0"]')[eachFrame]
browser2.switch_to.frame(frame)
inputMsg = browser2.find_element_by_id(i[0])
for e in range(0, len(i)):
inputMsg.send_keys(i[e])
browser2.switch_to.default_content()
eachFrame += 1
# -------------- FINAL STEP CHECKOUT --------------
print_warn("CHECKOUT?")
input("")
browser.execute_script(
"document.getElementsByClassName('step__footer__continue-btn btn')[0].click();"
)
browser2.execute_script(
"document.getElementsByClassName('step__footer__continue-btn btn')[0].click();"
)
time.sleep(10)
browser.quit()
def main():
iqname = str(input("Enter iqtools name: "))
classid = str(input("Enter class id: "))
name = str(input("Enter name: "))
password = str(input("Enter pass: "******"https://iqtools-{iqname}.intrtl.com/validate?klass_id={classid}&lots_id&size_id&confirm=&page=1&proposal=false&limit=0&klass_search="
base_url = f"https://iqtools-{iqname}.intrtl.com/validate?confirm=&page=1&proposal=false&limit=0"
#profile = webdriver.FirefoxProfile()
#profile.accept_untrusted_certs = True
#options = webdriver.FirefoxOptions()
#options.add_argument('--headless')
driver = Firefox(
executable_path=r'C:\Users\elili\Documents\Python\geckodriver')
#driver = webdriver.Chrome(executable_path=r'C:\Users\elili\Documents\Python\chromedriver')
driver.get(base_url)
try:
element_present = EC.presence_of_element_located(
(By.NAME, 'login-button'))
WebDriverWait(driver, 10).until(element_present)
except TimeoutException:
print("Timed out waiting for page to load")
driver.find_element_by_id("loginform-name").send_keys(name)
driver.find_element_by_id("loginform-password").send_keys(password)
driver.find_element_by_name("login-button").click()
try:
element_present = EC.presence_of_element_located(
(By.CLASS_NAME, 'klass-wrapper'))
WebDriverWait(driver, 10).until(element_present)
except TimeoutException:
print("Timed out waiting for page to load")
# Create payload
payload = {
"confirm": "",
"klass_id": classid,
#"klass_search": "",
#"limit": "0",
#"lots_id": "null",
#"page": 1,
#"proposal": "false",
#"size_id": "null"
}
driver.request('POST', base_url, data=payload)
try:
element_present = EC.presence_of_element_located(
(By.CLASS_NAME, 'klass-wrapper'))
WebDriverWait(driver, 10).until(element_present)
except TimeoutException:
print("Timed out waiting for page to load")
classes = driver.find_elements_by_class_name('klass-wrapper')
for clas in classes:
image = clas.find_element_by_tag_name('img')
print(image.get_attribute('src'))
markup = clas.find_element_by_tag_name('a')
print(markup.get_attribute('href'))
while (True):
pass
class EntsoeDownloader:
"""
A class used to automatically scrap CSV files from ENTSOE Transparecny Platform
...
Attributes
----------
__username : str
a string representing username of an existing ENTSOE account
__password : str
a string representing password of an existing ENTSOE account
__login_url : str
the homepage link of ENTSOE
__download_dir : str
the folder path where CSV files are stored
date: str
string representing today's date
__day_ahead_price_url: None
the download url for scraping day ahead prices
__generation_per_product_url: None
the download url for scraping electricity generation per products
__actual_total_load_url: None
the donwload url for scraping actual loads
__generation_forecasted_url
the download url for scraping the generation forecasted
driver: None
contains the selenium chrome driver class
Methods
-------
check_download(i=0)
Checks whether the CSV file is downloaded within a certain time-period else it terminates
setup(headless=False, date_input=None)
Initializes the chrome webdriver for scraping
set_download_url()
Initializes the download urls
login_and_download()
Interacts with the firefox browser and executes a sequence of tasks for scraping
"""
def __init__(self, last_date_fetched, username="", password=""):
last_date_fetched = datetime.strptime(last_date_fetched,
"%Y-%m-%d %H:%M:%S")
new_date = last_date_fetched.date()
new_date += timedelta(days=1)
self.__username = username
self.__password = password
self.__login_url = "https://transparency.entsoe.eu/homepageLogin"
self.__download_dir = os.path.join(os.getcwd(), "download")
# choose another date if the last hour of fetched data is 23:00:00
self.date = last_date_fetched.date(
) if last_date_fetched.time().hour != 23 else new_date
self.__day_ahead_price_url = None
self.__generation_per_product_url = None
self.__actual_total_load_url = None
self.__generation_forecasted_url = None
self.driver = None
@staticmethod
def check_download(i=0):
"""
Checks first whether files have been downloaded within a 20 second time interval
Parameters
-------------
i:
File counter, to check how many files have been downloaded
"""
time_to_wait = 150
time_counter = 0
# Validate whether file has been downloaded
while True:
if time_counter > time_to_wait:
raise FileNotFoundError(
"Necessary files not found in directory (20 seconds timeout reached)!"
)
# List all the files in the : 'current_working_dir'/download
file_names = os.listdir(os.getcwd() + "/download")
if len(file_names) == i:
return True
time.sleep(1)
time_counter += 1
def setup(self, headless=False, date_input=None):
# Validate date variable
if date_input is not None:
self.date = datetime.strptime(date_input, "%d.%m.%Y").date()
# Create directory if fetched folder is not available
if not os.path.exists(self.__download_dir):
os.mkdir(self.__download_dir)
# Set headless option and firefox profile
options = Options()
options.headless = headless
fp = webdriver.FirefoxProfile()
fp.set_preference("browser.download.folderList", 2)
fp.set_preference("browser.download.manager.showWhenStarting", False)
fp.set_preference("browser.download.dir", self.__download_dir)
fp.set_preference(
"browser.helperApps.neverAsk.saveToDisk",
"text/plain, application/vnd.ms-excel, text/csv, text/comma-separated-values, "
"application/octet-stream")
# Initialize Firefox() object to navigate
self.driver = Firefox(firefox_profile=fp, options=options)
return self
def set_download_url(self):
"""
Sets class url attributes if date is given
"""
if self.date is None:
raise ValueError("Attribute 'date' needs to be defined")
self.__day_ahead_price_url = "https://transparency.entsoe.eu/transmission-domain/r2/dayAheadPrices/export?" \
"name=&defaultValue=false&viewType=TABLE&areaType=BZN&atch=false&dateTime.dateTime=" + self.date.strftime(
"%d.%m.%Y") + "+00%3A00%7CCET%7CDAY&biddingZone.values=CTY%7C10Y1001A1001A83F!BZN%7C10Y1001A1001A82H&dateTime.timezone=CET_CEST&dateTime" \
".timezone_input=CET+(UTC%2B1)+%2F+CEST+(UTC%2B2)&dataItem=ALL&timeRange=YEAR&exportType=CSV"
self.__generation_per_product_url = "https://transparency.entsoe.eu/generation/r2/actualGenerationPerProductionType/export?" \
"name=&defaultValue=false&viewType=TABLE&areaType=BZN&atch=false&datepicker-day-offset-select-dv-date-from_input=D&dateTime.dateTime="+self.date.strftime(
"%d.%m.%Y")+"+00%3A00%7CCET%7CDAYTIMERANGE&dateTime.endDateTime="+self.date.strftime(
"%d.%m.%Y")+"+00%3A00%7CCET%7CDAYTIMERANGE&area.values=CTY%7C10Y1001A1001A83F!BZN%7C10Y1001A1001A82H&productionType.values=B01&productionType." \
"values=B02&productionType.values=B03&productionType.values=B04&productionType.values=B05&productionType.values=B06&productionType." \
"values=B07&productionType.values=B08&productionType.values=B09&productionType.values=B10&productionType.values=B11&productionType." \
"values=B12&productionType.values=B13&productionType.values=B14&productionType.values=B20&productionType.values=B15&productionType." \
"values=B16&productionType.values=B17&productionType.values=B18&productionType.values=B19&dateTime.timezone=CET_CEST&dateTime." \
"timezone_input=CET+(UTC%2B1)+%2F+CEST+(UTC%2B2)&dataItem=ALL&timeRange=YEAR&exportType=CSV"
self.__actual_total_load_url = "https://transparency.entsoe.eu/load-domain/r2/totalLoadR2/export?name=&defaultValue=false&viewType=TABLE&areaType=BZN&atch=false&dateTime" \
".dateTime="+self.date.strftime(
"%d.%m.%Y")+"+00%3A00%7CCET%7CDAY&biddingZone.values=CTY%7C10Y1001A1001A83F!BZN%7C10Y1001A1001A82H&dateTime.timezone=CET_CEST&dateTime" \
".timezone_input=CET+(UTC%2B1)+%2F+CEST+(UTC%2B2)&dataItem=ALL&timeRange=YEAR&exportType=CSV"
self.__generation_forecasted_url = "https://transparency.entsoe.eu/generation/r2/dayAheadGenerationForecastWindAndSolar/export?" \
"name=&defaultValue=false&viewType=TABLE&areaType=BZN&atch=false&dateTime.dateTime="+self.date.strftime(
"%d.%m.%Y") +"+00%3A00%7CCET%7CDAYTIMERANGE&dateTime" \
".endDateTime="+ self.date.strftime(
"%d.%m.%Y") + "+00%3A00%7CCET%7CDAYTIMERANGE&area.values=CTY%7C10Y1001A1001A83F!BZN%7C10Y1001A1001A82H&productionType" \
".values=B16&productionType.values=B18&productionType.values=B19&processType.values=A18&processType.values=A01&processType" \
".values=A40&dateTime.timezone=CET_CEST&dateTime.timezone_input=CET+(UTC%2B1)+%2F+CEST+(UTC%2B2)&dataItem=ALL&timeRange=YEAR&exportType=CSV"
def login_and_download(self):
"""
Executes a sequence of tasks to log into ENTSOE and downloads the specified files
"""
if self.__username == "" or self.__password == "":
raise ValueError(
"Pleaser set the credentials for ENTSOE to download")
# Remove "download" directory to and create new one
if len(os.listdir(os.getcwd() + "/download")) > 0:
shutil.rmtree(self.__download_dir)
os.mkdir(self.__download_dir)
# Instantiate the download urls and request the browser for the homepage login url
self.set_download_url()
self.driver.get(self.__login_url)
# Find login form elements and insert ENTSOE account credentials
self.driver.find_element_by_id("email").send_keys(self.__username)
self.driver.find_element_by_id("password").send_keys(self.__password)
# Trigger click event to sign in
self.driver.find_element_by_xpath(
"//div[@class='submit-line']/div/input").click()
# Wait until login is completed and the "user-panel-drop-down" element is agreed
WebDriverWait(self.driver, 10).until(
EC.presence_of_element_located((By.ID, "close-button")))
# Agree to the terms and conditions
loader_div = self.driver.find_element_by_xpath("//*[@id='loading']")
self.driver.execute_script(
"arguments[0].setAttribute('style','visibility:hidden;');",
loader_div)
self.driver.find_element_by_id("close-button").click()
# Open download url in new tab
urls = [
self.__generation_per_product_url, self.__day_ahead_price_url,
self.__actual_total_load_url, self.__generation_forecasted_url
]
for url, index in zip(urls, range(len(urls))):
if self.check_download(index):
print("Opening url: ",
re.search("/r2/(.*)/export", url).group(1))
self.driver.execute_script("window.open('" + url + "');")
# Download CSV file and exit the browser
time_counter = 0
time_to_wait = 150 # the last CSV file which is the one holding forecast information on solar, wind and total
# takes at least 40 seconds to download
while len(os.listdir(os.path.join(os.getcwd(), "download"))) is not 4:
if time_counter > time_to_wait:
raise FileNotFoundError(
"Necessary files not found in directory"
" (150 seconds timeout reached)!")
time.sleep(1)
time_counter += 1
self.driver.quit()
import boto3
from cv2 import imwrite
from cv2 import addWeighted
from PIL import Image, ImageFilter
import cv2
from scipy.ndimage.filters import median_filter
options = Options()
options.headless = False
browser = Firefox(options=options)
browser.get('https://parivahan.gov.in/rcdlstatus/')
browser.execute_script("document.body.style.zoom='10%'")
elem = browser.find_element_by_id("form_rcdl:j_idt36:j_idt41")
loc = elem.location
size = elem.size
x = int(loc['x'])
y = int(loc['y'])
w = int(size['width'])
h = int(size['height'])
# print(x,y,w,h)
browser.save_screenshot("snap.png")
img = cv2.imread("snap.png")
crop_img = img[y:y + h, x:x + w]
imwrite("Image.png", crop_img)
# print(pytesseract.image_to_string("Image.png"))
client = boto3.client('s3', region_name='us-west-2')
# load what you want to test with (change to phantomjs later)
driver = Firefox()
driver.set_window_size(1000,500)
# uncomment to reparse args
# my_opt()
# args-url location
# driver.get(my_opt.URL)
#Testing Delete later
#Testing Auth____ Comment IN TO autologin Bwap
driver.get("http://192.168.127.144/bWAPP/login.php")
select = Select(driver.find_element_by_name('security_level'))
select.select_by_visible_text("medium")
username = driver.find_element_by_id("login")
password = driver.find_element_by_name("password")
submit = driver.find_element_by_xpath("//button[@type='submit']")
username.send_keys("bee")
password.send_keys("bug"),
# click on the submit button
submit.click()
#BWAPP TEST Start
# XXS GET TEST
driver.get("http://192.168.127.144/bWAPP/xss_get.php?")
# Testing Auth____ Comment IN TO autologin DVWA
driver.request('GET',"http://192.168.127.178/dvwa/login.php")
username = driver.find_element_by_name("username")
class Sunny(object):
def __init__(self, login, password):
self.start_display()
profile = webdriver.FirefoxProfile()
profile.set_preference('browser.download.folderList', 2) # custom location
profile.set_preference('browser.download.manager.showWhenStarting', False)
profile.set_preference('browser.download.dir', current_dir)
profile.set_preference('browser.helperApps.neverAsk.saveToDisk', "text/csv,application/vnd.ms-excel")
#profile.set_preference('browser.helperApps.neverAsk.saveToDisk', "text/plain")
self.driver = Firefox(profile)
self.login(login, password)
self._login = login
self._password = password
def start_display(self):
self.display = Display(visible=0, size=(800, 600))
self.display.start()
def close(self):
self.driver.close()
self.display.stop()
def login(self, login=None, password=None):
"""Login on the Sunny portal website using the credentials
Parameters
----------
login: str
The login credential to sunnyportal
password: str
The password credential of sunnyportal
"""
if not login:
login = self._login
password = self._password
self.driver.get("https://www.sunnyportal.com/Templates/Start.aspx?ReturnUrl=%2f")
self.driver.find_element_by_id("txtUserName").clear()
self.driver.find_element_by_id("txtUserName").send_keys(login)
self.driver.find_element_by_id("txtPassword").clear()
self.driver.find_element_by_id("txtPassword").send_keys(password)
self.driver.find_element_by_id("ctl00_ContentPlaceHolder1_Logincontrol1_LoginBtn").click()
#time.sleep(0.5)
def wait_n_get(self, element_type, value):
""" Wait for an element to be present and get it
Paramters
---------
element_type: By.ID | By.LINK_TEXT...
The type of value to identify the element to get
value: str
the value describing the element to get
Returns
-------
el: element
The driver element requested
"""
return WebDriverWait(self.driver, TIME_DELAY).until(EC.presence_of_element_located((element_type, value)))
def goto(self, n_house):
"""Go to the page of an house given it's number, from the plant list page
Parameters
----------
n_house: int
The number of the house to go to
"""
el = self.wait_n_get(By.LINK_TEXT, houses[n_house])
el.click()
def goto_2(self, n_house):
"""Go to a house from the plant pannel on the Dashboard page
Parameters
----------
n_house: int
The number of the house to go to
"""
self.wait_n_get(By.CLASS_NAME, 'plantselect').click()
self.wait_n_get(By.LINK_TEXT, houses[n_house]).click()
def hover_over(self, id):
"""Hover over an element of the page given its id
Parameter
---------
id: str
The id of the element to hover over
"""
el = self.wait_n_get(By.ID, id)
hover = ActionChains(self.driver).move_to_element(el)
hover.perform()
def click(self, id):
"""Click on an element of the page given its id
Parameter
---------
id: str
The id of the element to click on
"""
el = self.wait_n_get(By.ID, id)
el.click()
def select_date(self, day, month, year):
id_date = 'ctl00_ContentPlaceHolder1_UserControlShowDashboard1_UserControlShowEnergyAndPower1__datePicker_textBox'
id_before = 'ctl00_ContentPlaceHolder1_UserControlShowDashboard1_UserControlShowEnergyAndPower1_btn_prev'
id_after = 'ctl00_ContentPlaceHolder1_UserControlShowDashboard1_UserControlShowEnergyAndPower1_btn_next'
try:
el = self.wait_n_get(By.ID, id_date)
self.driver.execute_script('$("#%s").val("%d/%d/%d")'%(id_date, month, day, year))
sleep(0.2)
self.click(id_before)
sleep(0.2)
self.click(id_after)
sleep(0.2)
except Exception as e:
if "Element is not clickable at point" in str(e):
print(e)
print('trying again!')
self.select_date(day, month, year)
def download(self, day=None, month=None, year=None):
"""Download the CSV file
"""
# Make sure we see the "Day" pannel
tabactive = self.wait_n_get(By.CLASS_NAME, 'tabactive')
if not tabactive.text == 'Day':
self.click(id_day)
# Select the right day
if day:
self.select_date(day, month, year)
# Hover over the download button
try:
self.hover_over(id_hover)
self.click(id_click)
except Exception as e_1:
# Check if the data is available for that day by looking for the info bubble
try:
el = self.wait_n_get(By.ID, id_info)
if 'info.png' in el.get_attribute('src'):
print('no data available for this day')
return None
else:
# Not sure what just happen there
raise(e_1)
except Exception as e_2:
if 'Unable to locate element' in str(e_2):
# The info icon isn't available
print(e_2)
raise(e_1)
else:
# Not sure what just happen there
print(e_1)
print(e_2)
#raise (e1, e2)
# Download the data for the day
res = self.driver.request('GET', url_data_graph)
if res.status_code == 200:
print('sucess')
else:
raise Exception('Error:', res.text)
return res
def download_house(self, n, day=None, month=None, year=None):
""" Download the house power production of the day
Parameters
----------
driver: WebDriver
The WebDriver instance to action
n_house: int
The number of the house to go to
Return
------
df: pandas.DataFrame | None
A dataframe containing the house day power production, or None if there isn't any data available
"""
try:
# Check what is the starting point
if 'Start.aspx' in self.driver.current_url:
# We are on the login screen, we first need to login
print('-- login in main screen')
self.login()
print('-- accessing house', n)
self.goto(n)
elif 'sunnyportal.com/Plants' in self.driver.current_url:
# We are on the plant list, lets
self.goto(n)
elif 'sunnyportal.com/FixedPages/Dashboard.aspx' in self.driver.current_url:
# We are on a dashboard, so we should be able to click on the left hand pannel to go to the new house
self.goto_2(n)
else:
# No idea where we are
raise Exception('I dont know where we are:', self.driver.current_url)
print('-- downloading house', n, 'power data')
res = self.download(day, month, year)
self.date = self.wait_n_get(By.ID, id_date).get_attribute('value')
if day:
if not self.date == "%d/%d/%d"%(month, day, year):
print('Error the date wasnt fixed correctly: '+self.date)
if res:
# There seems to be a positive response, so let's put it in a pandas dataframe
df = pd.read_csv(StringIO(res.text), sep=';', names=['power', 'avg'], skiprows=1)
print('-- download sucessful')
return df
else:
print('-- download failed')
# No response, we return a None object
return res
except Exception as e_1:
# Something whent wrong
try:
# Check if sunny portal has banned us for some time
text = self.wait_n_get(By.ID, 'ctl00_ContentPlaceHolder1_Logincontrol1_DivLogin').text
if 'Login failed! Login will be blocked for' in text:
# It does seem like we have been banned for some time
print(text)
n_sec = int(text.split('for')[1].split(' seconds')[0])
print('going to sleep for %d sec'%(n_sec))
time.sleep(n_sec)
print('retrying this house')
return self.download_house(n, day, month, year)
except Exception as e_2:
# I don't know what went wrong
print(e_1)
print(e_2)
raise(e_1)
def img(self):
"""A simple screenshot function to show on the notebook"""
return Image(self.driver.get_screenshot_as_png())
def download_all(self, day=None, month=None, year=None):
df_dict = {}
for k, v in houses.items():
print(k)
df = self.download_house(k, day, month, year)
if isinstance(df, pd.DataFrame):
df_dict['House %d'%(k)] = df
# Save the data into a DataFrame
self.data = pd.DataFrame({k:v.power for k, v in df_dict.items() if isinstance(v, pd.DataFrame)}, index=df.index)
# Save the data into a file
m,d,y = self.date.split('/')
self.data.to_csv('svalin_%s_%s_%s.csv'%(d,m,y))
return self.data