def bssid_locate(bssid, out_tempfile = module_location + r'\tempdata\bssid_locate.json'):
wget_download('http://api.mylnikov.org/geolocation/wifi?bssid=' + bssid, bar = None, out = out_tempfile)
with open(out_tempfile, "r", encoding = "utf8", errors = 'ignore') as tempfile:
bssid_data = json.load(tempfile)
try: os.remove(out_tempfile)
except: pass
if bssid_data['result'] == 200:
return bssid_data['data']
else:
raise ConnectionError('Could not connect to server')
def geoplugin(ip = '', out_tempfile = module_location + r'\tempdata\geoplugin.json'):
wget_download('http://www.geoplugin.net/json.gp?ip=' + ip, bar = None, out = out_tempfile)
with open(out_tempfile, "r", encoding = "utf8", errors = 'ignore') as tempfile:
geoplugin_data = json.load(tempfile)
try:
os.remove(out_tempfile)
except:
pass
if geoplugin_data.get('geoplugin_status') == 200:
return geoplugin_data
else:
raise ConnectionError('Could not connect to server')
def whois(ip = '', out_tempfile = module_location + r'\tempdata\whois.json'):
wget_download('http://ip-api.com/json/' + ip, bar = None, out = out_tempfile)
with open(out_tempfile, "r", encoding = "utf8", errors = 'ignore') as tempfile:
whois_data = json.load(tempfile)
try:
os.remove(out_tempfile)
except:
pass
if whois_data.get('status') == 'success':
return whois_data
else:
raise ConnectionError('Status: ' + ip_data.get('status') + ', Message: ' + ip_data.get('message'))
def download_file_from_url(input_url,
outfolder=constants.temp_files_outdir,
return_filename=True,
timeout=False,
filename_preffix=None,
try_till_download=False):
'''
download a file specifying some conditions, the "try_till_download" parameter will override 'timeout' parameter
'''
#thx: https://stackoverflow.com/a/18727481/4436950
#thx: https://is.gd/FkH1td
# the url as a path
url_path = urlparse(input_url).path
filename = os.path.basename(url_path)
if filename_preffix:
filename = filename_preffix + filename
outpath = os.path.join(outfolder, filename)
#with the tailored outpath, we can download the file
if not try_till_download:
if timeout:
wget_download_with_timeout(input_url, outpath)
else:
wget_download(input_url, outpath)
else:
while True:
try:
print()
wget_download_with_timeout(input_url, outpath)
print()
time.sleep(1)
break
except:
print('\ntimeout reached, trying again!!')
if return_filename:
return filename
else:
return outpath
def wget_download_with_timeout(input_url, outpath):
wget_download(input_url, outpath)
def download(caller: str): ## {{{
global should_break
if file_type in ['v', 's', 'vs', 'a', 't']: ## {{{
## useful links {{{
## https://stackoverflow.com/questions/18054500/how-to-use-youtube-dl-from-a-python-program
## https://github.com/ytdl-org/youtube-dl/blob/master/README.md#embedding-youtube-dl
## https://github.com/ytdl-org/youtube-dl/blob/3e4cedf9e8cd3157df2457df7274d0c842421945/youtube_dl/YoutubeDL.py#L137-L312 (available options)
## https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L128-L278 (available options)
## https://www.programcreek.com/python/example/98358/youtube_dl.YoutubeDL
## }}}
## FIXME does not display for s and t [14000223000000]
def hook(
response
): ## https://stackoverflow.com/questions/23727943/how-to-get-information-from-youtube-dl-in-python
status = response['status']
## try is used just to make sure nothing goes wrong
try:
## we have to use the if statement and drop else and finally
## otherwise the download_info will be screwed when the exception swoops
## with a message like {'ERROR': "KeyError('speed')"} when download is finished
if status == 'downloading':
speed = response['speed']
speed = convert_byte(speed)
elapsed = response['elapsed']
elapsed = duration(int(elapsed))
eta = response['eta']
eta = duration(int(eta))
total_bytes = response['total_bytes']
total_conv = convert_byte(total_bytes)
downloaded_bytes = response['downloaded_bytes']
downloaded_conv = convert_byte(downloaded_bytes)
downloaded_percent = (downloaded_bytes * 100) / total_bytes
download_info = {
f'{downloaded_conv}/{total_conv}':
f'%{downloaded_percent:.2f}',
'speed': speed,
'elapsed': elapsed,
'ETA': eta
}
print(Color().purple_dim(download_info), end='\r')
except Exception as exc:
## TODO analyze erros locally, like the one we do for o down below. We just need more examples of exc to use in analyze [14000223000000]
download_info = {'ERROR': f'{status} {exc!r}'}
options = {
# 'outtmpl': '%(title)s--%(width)sx%(height)s-f%(format_id)s.%(ext)s',
'outtmpl': f'{outputname}.%(ext)s',
'no_color':
True, ## better be uncommented, otherwise analyze can't display error messages properly
'proxy': proxy,
'nooverwrites': True,
'progress_hooks': [hook],
'logger': Ydl_W_Logger(),
}
if downloader:
options = {
**options, 'external_downloader': 'curl'
} ## <--,-- wget throws DownloadError('ERROR: wget exited with code 8') for v, vs and a but wrorks well for s and t
## '-- axel throws DownloadError('ERROR: axel exited with code 1') for v, vs and a but wrorks well for s and t
langs = ['en', 'en-AU', 'en-CA', 'en-GB', 'en-IE', 'en-NZ', 'en-US']
if file_type == 'v': options = {**options, 'format': video_quality}
elif file_type == 's':
options = {
**options, 'writesubtitles': True,
'writeautomaticsub': True,
'subtitleslangs': langs,
'skip_download': True
}
elif file_type == 'vs':
options = {
**options, 'writesubtitles': True,
'writeautomaticsub': True,
'subtitleslangs': langs,
'format': video_quality
}
elif file_type == 'a':
options = {
**options, 'format':
'bestaudio',
'postprocessors': [{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3'
}]
}
elif file_type == 't':
options = {
**options, 'write_all_thumbnails': True,
'writethumbnail': True,
'skip_download': True
}
try:
with YoutubeDL(options) as Y_DL:
Y_DL.download([url])
print() ## to prevent the removal of download_info
should_break = True
except Exception as exc:
analyze(f'{exc!r}', caller=caller)
## }}}
elif file_type == 'o': ## {{{
def check_raw_size_validity():
global total_bytes
total_bytes = int(raw_size) ## make sure raw_size is an int
_ = 1 / raw_size ## make sure raw_size is not 0
def analyze_locally(local_cmd_stderr: str):
'''it is called "local" because it is meant for within download function o section only'''
# global reason
if 'ERROR' in local_cmd_stderr:
reason = 'ERROR'
elif 'IGNORED' in local_cmd_stderr:
reason = 'IGNORED'
elif 'ZeroDivisionError' in local_cmd_stderr:
reason = '0 raw size'
else:
reason = f'UNKNOWN: {local_cmd_stderr}'
print(Color().orange({'BAR ERROR': reason}))
try:
if downloader:
def wget_bar(
downloaded,
total_bytes,
width=80
): ## https://www.itersdesktop.com/2020/09/06/downloading-files-in-python-using-wget-module/
downloaded_perc = downloaded / total_bytes * 100
downloaded_conv = convert_byte(downloaded)
total_conv = convert_byte(total_bytes)
download_info = {
f'{downloaded_conv}/{total_conv}':
f'%{downloaded_perc:.2f}'
}
print(Color().purple_dim(download_info), end='\r')
if torsocks:
## FIXME find how to set proxy for wget [14000314155813]
continue_without_proxy = get_single_input(
'Setting torsocks for downloader for o is not possible at the moment. Continue without proxy?'
)
if not continue_without_proxy == 'y':
exit()
## File.get_info() may send raw_size as IGNORED (if requested so), ERROR or 0, therefore try is needed here
## wget is able to get downloaded and total_bytes on the go anyway, but it is better to respct the info already present
try:
check_raw_size_validity()
wget_download(url=url, out=outputname, bar=wget_bar)
except Exception as exc:
analyze_locally(f'{exc!r}')
wget_download(url=url,
out=outputname) #, bar=wget_error_bar)
else:
if torsocks:
s.proxies = {'http': proxy, 'https': proxy}
## https://stackoverflow.com/questions/16694907/download-large-file-in-python-with-requests
with s.get(url, headers=hdrs, timeout=20,
stream=True) as SESSION:
SESSION.raise_for_status()
with open(outputname, 'wb') as OUTPUTNAME:
chunksize = 8192 ## 8192 is 8KB
downloaded_bytes = 0
## File.get_info() may send raw_size as IGNORED (if requested so), ERROR or 0, therefore try is needed here
try:
check_raw_size_validity()
total_conv = convert_byte(total_bytes)
for chunk in SESSION.iter_content(
chunk_size=chunksize):
OUTPUTNAME.write(chunk)
downloaded_bytes += chunksize
downloaded_conv = convert_byte(
downloaded_bytes)
downloaded_percent = (
downloaded_bytes * 100
) / total_bytes ## FIXME <--,-- exceeds 100 [14000223000000]
## |-- also, adding if chunk: before OUTPUTNAME.write(chunk)
## '-- prevents downloaded_percent from reaching 100
download_info = {
f'{downloaded_conv}/{total_conv}':
f'%{downloaded_percent:.2f}'
}
print(
Color().purple_dim(download_info),
end='\r'
) ## we can't move this line to finally because bytes are written chunk by chunk
except Exception as exc:
analyze_locally(f'{exc!r}')
for chunk in SESSION.iter_content(
chunk_size=chunksize):
OUTPUTNAME.write(chunk)
print() ## to prevent the removal of download_info
should_break = True
except Exception as exc:
analyze(f'{exc!r}', caller=caller)
def wget(url, output = None, bar = None):
return wget_download(url, bar = bar, out = output)
def start(self, username, password, page_number=None, download=False):
self.driver.get(
'https://0752539c.index-education.net/pronote/eleve.html')
print("[log] Waiting")
# sleep(100)
print(f"username : `{username}`")
title = "Saisissez votre identifiant."
while 1:
try:
username_entry = self.driver.find_element_by_css_selector(
"[title^='" + title + "']")
except NoSuchElementException:
sleep(1)
continue
break
print("[log] Writing Username")
username_entry.send_keys(username)
print("[log] Writing Password")
title = "Saisissez votre mot de passe."
self.driver.find_element_by_css_selector(
"[title^='" + title + "']").send_keys(
str(b64decode(password.encode("UTF-8")).decode("UTF-8")))
print("[log] Clicking on connect")
title = 'Cliquez sur le bouton "Se connecter".'
self.driver.find_element_by_css_selector("[title^='" + title +
"']").click()
sleep(2)
if page_number:
# Connected to pronote
print("[log] Clicking on physique-chimie book")
content = 'Physique chimie 1re, éd. 2019 - Manuel numérique PREMIUM élève'
script = self.driver.find_element_by_xpath(
"//span[.='" + content +
"']").find_element_by_xpath("../..").get_attribute("onclick")
print("Opening educhadoc window with script : `" + script + "`")
self.driver.execute_script(script)
sleep(8)
# Switch to educadhoc tab
tabs = self.driver.window_handles
# self.driver.switch_to.window(tabs[0])
# self.driver.close()
self.driver.switch_to.window(tabs[1])
sleep(4)
# Educhadoc opened
current_url = self.driver.current_url
print("current url : `" + current_url + "`")
page = make_page(current_url, page_number)
if page != current_url:
print("Going to page : `" + page + "`")
self.driver.get(page)
# self.driver.execute_script('window.open("'+page+'")')
if download:
# Clicking on Vie Scolaire
for b in self.driver.find_elements_by_tag_name("div"):
try:
attr_id = b.get_attribute("id")
except:
print("Invalid div, skipping...")
if attr_id[::-1][:6][::-1] == 'Combo5':
print("Found : `" + attr_id + "`")
b.click()
sleep(1)
ActionChains(self.driver).move_to_element_with_offset(
b, 200, 0).perform()
sleep(1)
break
# Clicking on pdf
for b in self.driver.find_elements_by_tag_name("i"):
try:
attr_id = b.get_attribute("id")
except:
print("Invalid div, skipping...")
if attr_id[::-1][:6][::-1] == 'Bouton':
print("Found : `" + attr_id + "`")
screenWidth, screenHeight = pyautogui.size()
pyautogui.moveTo(screenWidth - 30, 90)
pyautogui.click()
sleep(1)
break
# Fill download form
self.driver.find_elements_by_name(
"55_1_rbChoixAnnuel")[1].find_element_by_xpath(
"..").find_elements_by_tag_name("span")[0].click()
self.driver.find_elements_by_name(
"55_1_rbPortrait")[1].find_element_by_xpath(
"..").find_elements_by_tag_name("span")[0].click()
self.driver.find_elements_by_name(
"55_1_rbRenvois")[1].find_element_by_xpath(
"..").find_elements_by_tag_name("span")[0].click()
sleep(0.5)
for b in self.driver.find_elements_by_tag_name("button"):
try:
attr_id = b.get_attribute("id")
except:
print("Invalid div, skipping...")
if attr_id[::-1][:6][::-1] == 'btns_1':
print("Found : `" + attr_id + "`")
b.click()
sleep(1)
break
# Download pdf with direct link
self.driver.switch_to_window(self.driver.window_handles[1])
sleep(1)
current_url = self.driver.current_url
while current_url == 'about:blank':
sleep(0.5)
current_url = self.driver.current_url
print("Trying current url : `" + current_url + "`")
print("current url : `" + current_url + "`")
filename = wget_download(current_url)
print("\nPdf downloaded at : `" + filename + "`")