def __set_cookies(self, data: Data, browser: Browser):
"""
This function sets the specified cookies to the browser.
@param data: The data object of the program.
@type data: Classes.Data
@param browser: The webdriver browser.
@type browser: Browser
@return: None
"""
if data.cookies:
# If user specified cookies.
if Methods.enter_cookies(data, browser, data.url):
# Success.
self.__non_session_browser[0] = Methods.new_browser(
data) # setting up secondary browser object.
print(
f"\t[{COLOR_MANAGER.YELLOW}!{COLOR_MANAGER.ENDC}] {COLOR_MANAGER.YELLOW}"
f"Cookies were added to the session.{COLOR_MANAGER.ENDC}")
else:
# Failure.
print(
f"\t[{COLOR_MANAGER.YELLOW}!{COLOR_MANAGER.ENDC}] {COLOR_MANAGER.YELLOW}"
f"Invalid cookies, check your syntax and try again. {COLOR_MANAGER.ENDC}"
)
data.cookies = None # There is no need to use them again.
def __set_chromedriver(data: Data):
"""
This function sets a browser web driver object for the first time,
and downloads chromedriver file in case of first time running the program.
@param data: The data object of the program.
@type data: Classes.Data
@return: None
"""
driver_file = "chromedriver"
pl = sys.platform
# Get OS.
if pl == "linux" or pl == "linux2":
operating_system = "linux64"
elif pl == "darwin":
operating_system = "mac64"
else:
operating_system = "win32"
driver_file += ".exe"
if driver_file not in os.listdir(
"."): # There is no chromedriver in the folder.
# Getting zip file.
print(
f"\t[{COLOR_MANAGER.YELLOW}?{COLOR_MANAGER.ENDC}] {COLOR_MANAGER.YELLOW}"
f"Downloading Chromedriver...{COLOR_MANAGER.ENDC}")
try:
# Get latest version.
version = requests.get(
"http://chromedriver.storage.googleapis.com/LATEST_RELEASE"
).text
# Get zip link.
link = f"https://chromedriver.storage.googleapis.com/" \
f"{version}/chromedriver_{operating_system}.zip"
zip_content = io.BytesIO(requests.get(link).content)
with zipfile.ZipFile(zip_content) as zip_ref:
# Extracting the executable file.
zip_ref.extractall(".")
except Exception:
raise Exception(
"Download failed, please check your internet connection.")
# There is a chromedriver in the folder.
data.driver = os.getcwd() + "\\" + driver_file # Full path.
try:
print(
f"\t[{COLOR_MANAGER.YELLOW}?{COLOR_MANAGER.ENDC}] {COLOR_MANAGER.YELLOW}"
f"Setting up the Chromedriver...{COLOR_MANAGER.ENDC}")
except Exception:
raise Exception(
"Setting up the web driver failed, please try again.")
def __init__(self,
updateTime,
errorLog,
bufferLog,
updateInterfaceFunctions,
updateCounterMax=[0, 0, 0, 0]):
self.updateTime = updateTime
self.data = Data()
self.dataFile = File()
# self.lapTimeFile = File()
self.lastBuffers = bufferLog
self.errorLog = errorLog
# Dicionario de funcoes: permite chamar funcoes fazendo updateInterfaceFunctions[key], onde key é 1,2,3 ou 4
self.updateInterfaceFunctions = updateInterfaceFunctions
self.packIndexes = [1, 2, 3, 4]
self.packSizes = self.data.pSizes
self.updateCounterMax = updateCounterMax # numero de pacotes recebidos at atualizar a interface
self.updateCounter = [0, 0, 0, 0]
self.updateInterfaceEnabled = True
def logic(self, data: Data):
"""
This function gets the page list of the specified website.
@param data: The data object of the program.
@type data: Classes.Data
@return: None
"""
print(
f"{COLOR_MANAGER.BLUE + COLOR_MANAGER.HEADER}Scraping pages:{COLOR_MANAGER.ENDC}"
)
# Setting environment.
self.__non_session_browser.append(None)
try:
self.__set_lists(data) # Setting white and black list.
self.__set_chromedriver(data) # Setting web browser driver.
browser = Methods.new_browser(data) # Setting up browser object.
self.__set_cookies(data, browser) # Setting up specified cookies.
print(COLOR_MANAGER.ENDC)
except Exception as e:
raise Exception(e, "\t")
try:
# Getting the pages.
self.__get_pages(data, data.url, browser, None)
# We need to clear them in case of session pages.
self.__already_checked.clear()
except Exception:
raise Exception("Unknown problem occurred.", "\t")
# In case of empty website.
if len(data.pages) == 0:
raise Exception("Your website doesn't have any valid web pages.",
"\t")
if len(data.pages) != data.max_pages:
# Getting the session pages.
self.__get_session_pages(data, browser)
# Printing the results.
self.__print_result(data)
# Transferring only the valid pages.
data.pages = [
page for page in data.pages if self.__valid_in_list(page)
]
browser.quit()
# In case of empty website.
if len(data.pages) == 0:
raise Exception(
"Your website doesn't have any web pages that fit your requirements.\n",
"\t")
def main():
"""
This function connects the different managers together.
@return: None
"""
try:
# Initiate manager classes instances.
data = Data()
function_order = [
FlagManager.FlagManager(
).logic, # Get arguments from command line.
AddressManager.AddressManager().logic, # Check specified address.
print_data, # Print given arguments.
PageManager.PageManager(
).logic, # Get all the pages from the website.
PluginManager.PluginManager(
).logic, # Generate the `Check Device` in our directory.
VulnerabilityManager.VulnerabilityManager().
logic # Run plugins with the `Check Device`.
]
# Starting the process.
for function in function_order:
# Executing every function.
function(data)
print(COLOR_MANAGER.ENDC)
except KeyboardInterrupt:
# The user pressed ctrl+c.
COLOR_MANAGER.print_warning(
"You have decided to close the process, please wait few seconds...\n",
"\n\t")
except Exception as e:
if len(e.args) == 2:
COLOR_MANAGER.print_error(str(e.args[0]), str(e.args[1]))
else:
COLOR_MANAGER.print_error(str(e))
finally:
# Every time the program has finished it's run we clean up.
finishing_up()
def __get_session_pages(self, data: Data, browser: Browser):
"""
This function looking for login forms and scraping session pages through them.
@param data: The data object of the program.
@type data: Classes.Data
@param browser: The webdriver browser.
@type browser: Browser
@return: None
"""
if not (data.username and data.password):
# If there are no username or password.
return
if not self.__non_session_browser[0]:
# If the instance is not already set, set up a new one.
self.__non_session_browser[0] = Methods.new_browser(data)
non_session_pages = list(data.pages)
pages_backup = list(data.pages)
login_pages = list()
for page in non_session_pages:
if "html" not in page.type:
continue
# Checking if the page has a login form.
if not self.__get_login_form(data, Methods.get_forms(
page.content)):
continue
# Setting browser for current page.
browser.get(page.url)
# Getting updated form.
form_details = self.__get_login_form(
data, Methods.get_forms(browser.page_source))
try:
Methods.submit_form(data, browser, form_details["inputs"])
except Exception:
continue
new_url = browser.current_url
content = browser.page_source
if new_url in login_pages:
# The new url is already in the list.
continue
login_pages.append(new_url)
login = True
for p in data.pages:
if new_url == p.url:
# Have the same URL.
if Methods.remove_forms(content) == Methods.remove_forms(
p.content):
# Same URL and content.
login = False
break # We do not need to check anymore.
if not login:
# Login attempt failed.
continue
# Setting login flags.
self.__logged_out = True
self.__logged_in = True
while self.__logged_out:
# Until it won't encounter a logout page.
self.__logged_out = False
# Attempting to achieve data from page.
self.__get_pages(data, new_url, browser, page)
if self.__logged_out:
# If the session has encountered a logout page.
self.__already_checked.clear(
) # The function needs to go through all the session pages.
for checked_page in data.pages:
if "html" not in checked_page.type and checked_page not in pages_backup:
pages_backup.append(checked_page)
self.__already_checked.append(checked_page)
data.pages = list(
pages_backup) # Restoring the pages list.
# Updating the session.
browser.delete_all_cookies()
browser.get(page.url)
Methods.enter_cookies(data, browser, page.url)
Methods.submit_form(data, browser, form_details["inputs"])
# Doing the loop all over again, without the logout page.
break
# Closing the non session browser.
self.__non_session_browser[0].quit()
if len(data.pages) == len(non_session_pages):
# Session never append.
if login_pages:
# Found at least one login page.
raise Exception(
"The login attempt failed, your username or password might be invalid.\n",
"\n\t")
else:
# No login pages were found.
raise Exception(
"The program did not find any login form in the listed pages.\n",
"\n\t")
def __get_pages(self,
data: Data,
curr_url: str,
browser: Browser,
previous: Page = None,
recursive: bool = True):
"""
This function gets the list of pages to the data object.
@param data: The data object of the program.
@type data: Classes.Data
@param curr_url: The current URL the function checks.
@type curr_url: str
@param browser: The web driver that gets the rendered content.
@type browser: Browser
@param previous: The previous page.
@type previous: bool
@param recursive: True - check all website pages, False - only the first reachable one.
@type recursive: bool
@return: None
"""
if len(data.pages) == data.max_pages:
# In case of specified amount of pages, the function will stop.
return
if curr_url in self.__logout_list:
# Do not open logout pages.
return
if self.__logged_out:
# If the session already logged out.
if self.__non_session_browser[0]:
# Remove non session browser.
self.__non_session_browser[0].quit()
self.__non_session_browser[0] = None
if self.__logged_in:
# If we logged in (-L) we need to return.
# If we used cookies (-c) we can keep going.
return
try:
# Trying to get the current URL.
browser.get(curr_url)
request = None
for req in browser.requests[::-1]:
if req.url == curr_url or req.url == browser.current_url:
# If we found the right URL.
request = req
if req.response.headers.get("Content-Type"):
# Only if the page has content type.
break
if not request or request.response.status_code != 200:
# Did not find the request.
raise Exception() # The request is not found.
except Exception:
self.__troublesome.append(curr_url)
return
page = Page(browser.current_url, request.response.status_code,
request.response.headers.get("Content-Type").split(";")[0],
browser.page_source, request, browser.get_cookies(),
previous)
color = COLOR_MANAGER.BLUE
if self.__non_session_browser[0]:
# Session page.
try:
same_content = False
same_url = False
a_page = None
for a_page in data.pages:
if not self.__get_links([a_page.url], browser.current_url):
# Have the same URL.
same_url = True
if "html" in a_page.type and \
Methods.remove_forms(a_page.content) == Methods.remove_forms(browser.page_source):
# Have the same content of another page.
same_content = True
if same_url or same_content:
# Already found what we were looking for.
break
if "logout" in curr_url.lower() or same_content:
if not self.__is_session_alive(data, browser):
# It redirected to a non-session page, and have the same content or logout in name.
print(
f"\t[{COLOR_MANAGER.RED}-{COLOR_MANAGER.ENDC}]"
f" {COLOR_MANAGER.RED}{curr_url}{COLOR_MANAGER.ENDC}"
)
self.__logout_list.append(curr_url)
self.__logged_out = True
if not self.__logged_in:
# In case of cookies (-c), the logout action makes the cookies invalid.
removed_list = list()
for page in data.pages:
if not page.is_session:
removed_list.append(page)
data.pages = removed_list
print(
f"\t[{COLOR_MANAGER.RED}!{COLOR_MANAGER.ENDC}]"
f" {COLOR_MANAGER.RED}Cookies are invalid anymore,"
f" all session pages were removed.{COLOR_MANAGER.ENDC}"
)
return
else:
browser.get(request.url)
if same_content and same_url and a_page.is_session:
# Redirected to another session page with the same URL or content.
self.__troublesome.append(curr_url) # No need to check.
return
self.__non_session_browser[0].get(browser.current_url)
if self.__non_session_browser[0].current_url != browser.current_url or \
Methods.remove_forms(self.__non_session_browser[0].page_source) !=\
Methods.remove_forms(browser.page_source):
# If the URL can be reachable from non-session point the session has logged out.
# Session page.
page.is_session = True
color = COLOR_MANAGER.ORANGE
except Exception:
self.__troublesome.append(curr_url)
return
soup = None
if "html" in page.type:
# Only if the page is html.
try:
# Creating a BeautifulSoup object.
soup = BeautifulSoup(page.content, "html.parser")
except Exception:
# Couldn't parse, might be non-html format, like pdf or docx.
self.__troublesome.append(page.url)
return
if page.url != curr_url:
# If the current URL is redirecting to another URL.
self.__troublesome.append(curr_url)
if previous and not self.__get_links([previous.url], page.url):
# The Redirected link is out of the website.
return
# Checking if the page was already printed.
in_list = False
for printed_page in self.__already_printed:
if not self.__get_links([printed_page.url], page.url) and\
(printed_page.content == page.content or printed_page.is_session == page.is_session):
# Same URL (without the http://) and content or both are session.
in_list = True
if not in_list:
# If the page was not printed.
if not soup:
# If it is a non-html page.
color = self.__type_colors["Other"]
for key in self.__type_colors.keys():
if str(key).lower() in page.type:
color = self.__type_colors[key]
break
# Printing the page.
sign = "+"
if not self.__valid_in_list(page):
sign = "-" # Sign of not checking.
print(
f"\t[{color}{sign}{COLOR_MANAGER.ENDC}] {color}{page.url}{COLOR_MANAGER.ENDC}"
)
self.__already_printed.append(page)
# Checking if the page was already checked.
in_list = False
for a_page in data.pages:
if not self.__get_links([a_page.url], page.url) and\
(a_page.content == page.content or a_page.is_session == page.is_session):
# Same URL (without the http://) and (content or both are session).
in_list = True
if not in_list:
# Adding to the page list.
data.pages.append(page)
# Adding to the already-checked list.
self.__already_checked.append(page)
if not soup:
# There is no reason check non-html page.
return
# Getting every application script in the page.
links = self.__get_links(
[script.get("src") for script in soup.find_all("script")],
page.url)
# Getting every css style in the page.
links.extend(
self.__get_links([
script.get("href") for script in soup.find_all(type="text/css")
], page.url))
for link in links:
# Checking only scripts and style file.
for req in browser.requests:
if link == req.url:
# Were already requested with the current page.
self.__get_pages(data, link, browser, page, data.recursive)
links.remove(link)
break
del browser.requests # We do not need the previous requests anymore.
if recursive:
# If the function is recursive.
# Getting every link in the page.
links.extend(
self.__get_links(
[link.get("href") for link in soup.find_all("a")],
page.url))
for link in links:
if (self.__logged_out and self.__logged_in) or len(
data.pages) == data.max_pages:
# More efficient to check every time.
# If the session logged out or the pages amount is at its maximum.
return
if all(link != page.url
for page in data.pages) or self.__non_session_browser[0]:
# If the page is not in the page list.
if (not any(link == checked_page.url
for checked_page in self.__already_checked)
and link not in self.__troublesome):
# Page was not checked, it is not troublesome or in the black list.
self.__get_pages(data, link, browser, page, data.recursive)
def __valid_address(self, data: Data):
"""
This function checks if the URL, port and IP are valid
@param data: The data object of the program.
@type data: Data
@return: None
"""
port: int = data.port
url: str = data.url
ip: str = data.ip
scheme: str = "http"
path: str = "/"
query: str = ""
if url:
# Start by checking the URL.
if not url.startswith(scheme):
# Does not start with http or https.
raise Exception(
"The URL is not in the right format of 'scheme://domain:port/path'.",
"\t")
parse = urlparse(url=data.url) # Parsing the URL.
try:
port = parse.port if parse.port else port # Select a port to use (from a url or from data).
scheme = parse.scheme # Must have a scheme (http or https).
path = parse.path # Must have a path (even "/" counts).
query = "?" + parse.query if parse.query else query
except Exception:
print(
f"\t[{COLOR_MANAGER.YELLOW}%{COLOR_MANAGER.ENDC}]{COLOR_MANAGER.YELLOW}"
f" The port which was specified in the URL is invalid.{COLOR_MANAGER.ENDC}"
)
try:
ip = socket.gethostbyname(parse.hostname)
except Exception:
pass
if ip:
# Found IP in the URL or was already specified in data.
if ip.count(".") != 3 or not all(
field.isnumeric() and 0 <= int(field) <= 255
for field in ip.split(".")):
# If the number of fields is not 4 or any of them are not in range of 0-255.
raise Exception(
f"The IP is not in the right of format of [0-255].[0-255].[0-255].[0-255].",
"\t")
else:
# The IP is not specified and the URL was not found.
raise Exception("No IP was specified or found through the URL.",
"\t")
if port:
# A port was specified in the user's input.
if self.__MAXIMUM_PORT <= port or port <= 0:
# Port out of range.
raise Exception(
f"Port is out of range ('{port}' is not between 0-{self.__MAXIMUM_PORT})",
"\t")
elif data.port == 0:
# -P was specified.
port = 0
else:
# No port specified and -P was not specified.
print(
f"\t[{COLOR_MANAGER.YELLOW}%{COLOR_MANAGER.ENDC}]{COLOR_MANAGER.YELLOW} "
f"Using default port 80.{COLOR_MANAGER.ENDC}")
port = 80
# Setting address into the data object.
data.url = f"{scheme}://{ip}:{port}{path}{query}"
data.port = port
data.ip = ip
def __get_final_args(self, data: Data, args: argparse.Namespace):
"""
This function gets the arguments from the argparse namespace and inserts
them into a Data object which is returned to the main program.
@param data: The data object of the program.
@type data: Data
@param args: All the command line arguments.
@type args: argparse.Namespace
@return: The returned data object, will be processed furthermore in the Main Core.
@rtype: Data
"""
# Set the `Username and Password`.
if type(args.login) is not None:
if len(args.login) == 2:
data.username = self.__char_arr_to_string(args.login[0])
data.password = self.__char_arr_to_string(args.login[1])
# Set the `cookies`.
data.cookies = args.cookies
# Set the `Host IP` Address.
data.ip = args.ip
# Set the `Website URL`.
data.url = args.url
# Check if `all_ports` flag is set.
if args.all_ports:
data.port = 0
else:
# Set the `Host Port`.
data.port = args.port
# Set the `maximum number of pages`.
if args.number_of_pages and args.number_of_pages <= 0:
# If the given number is invalid.
COLOR_MANAGER.print_error(
"Invalid number of pages! Running with unlimited pages.")
data.max_pages = None
else:
# If the number wasn't specified or it was specified and is valid.
data.max_pages = args.number_of_pages
# Set the `output file` name and path.
data.output = args.output
# Set `blacklist` file path.
if args.blacklist is not None:
if args.blacklist.endswith(".txt"):
data.blacklist = args.blacklist
else:
data.blacklist = args.blacklist + ".txt"
else:
data.blacklist = args.blacklist
# Set `whitelist` file path.
if args.whitelist is not None:
if args.whitelist.endswith(".txt"):
data.whitelist = args.whitelist
else:
data.whitelist = args.whitelist + ".txt"
else:
data.whitelist = args.whitelist
# Set `recursive` flag.
data.recursive = args.recursive
# Set `verbose` flag.
data.verbose = args.verbose
if args.verbose:
# Print startup logo and current time.
print(startup())
print(
f"{COLOR_MANAGER.GREEN}Started on: {datetime.datetime.now()}{COLOR_MANAGER.ENDC}"
)
# Set `aggressive` flag.
data.aggressive = args.aggressive
class Program():
def __init__(self,
updateTime,
errorLog,
bufferLog,
updateInterfaceFunctions,
updateCounterMax=[0, 0, 0, 0]):
self.updateTime = updateTime
self.data = Data()
self.dataFile = File()
# self.lapTimeFile = File()
self.lastBuffers = bufferLog
self.errorLog = errorLog
# Dicionario de funcoes: permite chamar funcoes fazendo updateInterfaceFunctions[key], onde key é 1,2,3 ou 4
self.updateInterfaceFunctions = updateInterfaceFunctions
self.packIndexes = [1, 2, 3, 4]
self.packSizes = self.data.pSizes
self.updateCounterMax = updateCounterMax # numero de pacotes recebidos at atualizar a interface
self.updateCounter = [0, 0, 0, 0]
self.updateInterfaceEnabled = True
def openSerialPort(self, port, baudrate, timeout):
self.porta = serial.Serial()
self.porta.baudrate = baudrate
self.porta.port = port
self.porta.timeout = timeout
self.porta.open()
def stopProgram(self):
self.stop = 1 # atualiza o valor da variavel stop, a qual é usada para verificar o funcionamento da interface
# self.lapTimeFile.stopDataSave()
# Fecha arquivo file e porta serial
self.dataFile.stopDataSave()
if self.porta.isOpen():
self.porta.flushInput()
self.porta.close()
else:
pass
# program() roda em loop
def program(self):
if (self.stop == 0):
# Le dados da porta serial
self.buffer = self.readFromSerialPort(self.packSizes,
self.packIndexes)
if len(self.buffer) != 0:
# chamada da função updateDataAndInterface para analisar os dados recebidos atualizar os mostradores da interface
self.updateData(self.buffer, int(self.buffer[0]))
if self.dataFile.save == 1:
self.saveLine(self.buffer, int(self.buffer[0]))
if self.updateInterfaceEnabled:
self.updateInterface(self.buffer, int(self.buffer[0]))
# Apos updateTime segundos, chama funcao program() novamente
QtCore.QTimer.singleShot(self.updateTime, lambda: self.program())
def updateData(self, buffer, packID):
# Atualiza dados em Data e atualiza campos respectivos na interface
if (self.data.updateDataFunctions[packID](buffer) == 0):
self.errorLog.writeLog(" updateData: Pacote " + str(packID) +
"com tamanho diferente do configurado")
# Desloca vetores e chama funcao de atualizar graficos da interface
if packID == 2 or packID == 3:
self.data.rollArrays()
def updateInterface(self, buffer, packID):
# Chama funcao updatePxInterface, atribuida no dicionario updateInterfaceFunctions, para a chave x = packID
if self.updateCounter[packID - 1] >= self.updateCounterMax[packID - 1]:
self.updateInterfaceFunctions[packID](self.data)
self.updateCounter[packID - 1] = 0
else:
self.updateCounter[packID - 1] += 1
# Atualiza o mostrador textBrowser_Buffer com as ultimas listas de dados recebidas.
self.lastBuffers.writeLog(vectorToString(buffer, ' ',
addNewLine=False))
def saveLine(self, buffer, packID):
# Grava linha buffer no arquivo
string = self.data.createPackString(packID)
self.dataFile.writeRow(string)
# Le buffer da porta serial. bufferSize é uma lista com os tamanhos dos pacotes e firstByteValues
# é uma lista com os numeros dos pacotes (1,2,3,4)
def readFromSerialPort(self, bufferSize, firstByteValues):
while True:
# Espera receber algo na porta serial
while (self.porta.inWaiting() == 0):
pass
read_buffer = b''
# Le primeiro e segundo bytes
firstByte = self.porta.read()
if int.from_bytes(firstByte, byteorder='big') in firstByteValues:
read_buffer += firstByte
# Le o segundo byte de inicio
a = self.porta.read()
if int.from_bytes(a, byteorder='big') == 5:
read_buffer += a
break
else:
self.errorLog.writeLog(
"Leitura: segundo byte com valor inesperado. Leu-se " +
str(firstByte) + ", esperava-se 5")
# Se o byte lido nao for 1, 2 3 ou 4,, quer dizer que perdeu algum dado.
else:
self.errorLog.writeLog(
"Leitura: primeiro byte com valor inesperado. Leu-se " +
str(firstByte) + ", esperava-se de 1 a 4")
while True:
# Le resto do buffer
index = int.from_bytes(firstByte, byteorder='big') - 1
byte = self.porta.read(size=int(bufferSize[index] - 2))
read_buffer += byte
if (len(read_buffer) == bufferSize[index]):
if int(read_buffer[bufferSize[index] - 2]) == 9:
# Chegou no fim do pacote
if int(read_buffer[bufferSize[index] - 1]) == 10:
break
else:
self.errorLog.writeLog(
"Leitura: ultimo dado diferente de byte 10" +
str(read_buffer))
return []
else:
self.errorLog.writeLog(
"Leitura: penultimo dado diferente de byte 9")
return []
return read_buffer