def scrape_all(self):
"""Scraps all Ligues in the URLS-Constant"""
self.clear()
for league in TipicoScraper.LEAGUES:
print(f"scraping: {league.country}: {league.name}")
self.__read_site_soup(league.url)
home_teams, away_teams = self.__scrape_teams()
home_win_quotes, draw_quotes, away_win_quotes = self.__scrape_quotes(
)
dates, times = self.__scrape_match_date_and_time()
self.__create_matches(home_teams, away_teams, home_win_quotes,
draw_quotes, away_win_quotes, league, dates,
times)
# we could use away_teams or quotes here... all have the same len()
amount_matches = len(home_teams)
text = f"found {amount_matches} matches \n"
# Print the amount of founded matches in red if no match was found
if amount_matches == 0:
print_red(text)
else:
print(text)
# When we click on a new league on tipico the new league opens above the old league so we would receive
# matches multiple times. The easiest way to avoid this is to get a new browser. Better way is to delete the
# cookies
self.driver.delete_all_cookies()
# close the driver (browser)
self.driver.quit()
class check_version():
print("Checking postgres version.")
global cur
global version_number
db = connect()
cur = db.cur
last_version = get_last_version()
version_number = last_version.version_number
def get_last_version_of_postgres():
cur.execute("select version()")
try:
get_last_version_of_postgres()
versionArr = cur.fetchall()
except psycopg2.Error as err:
print("Error: Fetching last version")
versionStr = ""
for versionA in version_number:
for versionText in versionA:
versionStr += str(versionText)
if versionStr != version_number:
print_red("Warning: Please update your postgresql. " + version_number +
" is avaliable.")
else:
print_green("OK")
class check_public_priveleges():
print("Checking privileges on granted to PUBLIC")
global cur
db = connect()
cur = db.cur
def get_public_privileges():
cur.execute("""SELECT *
FROM information_schema.role_table_grants
where grantee = 'PUBLIC'
""")
try:
get_public_privileges()
privileges = cur.fetchall()
except psycopg2.Error as err:
public("Error: Fetching public privileges failed ", err)
if len(privileges) > 0:
print_red(
"Critical Error: there is some privileges are granted to PUBLIC")
# print(Fore.RED + "Critical Error: there is some privileges are granted to PUBLIC")
# print("Critical Error: there is some privileges are granted to PUBLIC")
else:
print_green("Ok")
class check_privileges_pg_catalog():
print("Checking privileges on pg_catalog.pg_authid.")
global cur
db = connect()
cur = db.cur
def get_privileges_on_pg_catalog(): #get users who have privileges on pg_catalog.pg_authid
cur.execute("""SELECT grantee
FROM information_schema.role_table_grants
where table_schema = 'pg_catalog'
and table_name = 'pg_authid'
group by grantee
""")
def get_superusers():
# cur.execute("""SELECT usename
# from pg_user
# where usesuper = true
# """)
cur.execute("""SELECT *
FROM pg_catalog.pg_authid
where rolsuper = true
""")
try:
get_privileges_on_pg_catalog()
users_on_pg_catalog = cur.fetchall()
except psycopg2.Error as err:
print("Error: Fetching privileges on pg_catalog failed ",err)
try:
get_superusers()
superusers = cur.fetchall()
except psycopg2.Error as err:
print("Error: Fetching superusers failed",err)
superList = []
for super in superusers: #creating super user list.
for superuser in super:
superList.append(superuser)
for users in users_on_pg_catalog:
for user in users:
if user not in superList: # if that user not super user print an error.
print_red("Critical Error: " + user + " has privileges on pg_catalog.pg_authid. "
+ user + " must be a superuser")
else:
print_green("OK")
class check_ssl_force():
print("Checking if ssl is forced")
global lines
is_ssl_forced = True
conf = open_conf_file()
lines = conf.lines
for line in lines: #get line by line
method = line[:7].strip() #get last 6 character to check if it is trust or not
if method != "hostssl":
is_ssl_forced = False
if is_ssl_forced :
print_red("Error: SSL is not forced")
else:
print_green("OK")
class check_trust_auth():
print("Checking if any user have trust authentication except local users")
is_trusted = False
global lines
conf = open_conf_file()
lines = conf.lines
for line in lines: #get line by line
method = line[-6:].strip(
) #get last 6 character to check if it is trust or not
if method == "trust":
type = line[:5].strip() # is it local
if type != "local" and type[:4] != "host":
is_trusted = True
if is_trusted:
print_red(
"Error: There are some users have trust authentication in pg_hba.conf file"
)
else:
print_green("OK")
class security_definer_checker():
global cur
db = connect()
cur = db.cur
def get_database_functions():
cur.execute("""SELECT p.proname
FROM pg_catalog.pg_namespace n
JOIN pg_catalog.pg_proc p
ON p.pronamespace = n.oid
WHERE n.nspname = 'public'""")
def get_function_content(func_name):
cur.execute("""SELECT routine_definition
FROM
information_schema.routines
WHERE specific_schema
LIKE 'public'
AND routine_name
LIKE """ + "'" + str(func_name) + "'")
def check_function(function_declaration):
if "security definer" in function_declaration.lower():
return True
return False
print("Security Definer checking...")
try:
get_database_functions() #return declared functions in db
rows = cur.fetchall() # get all functions into list
except psycopg2.Error as err:
print("there is an error", err)
#After getting functions from database
#We need to look at function defenitons one by one
#Rows returns 2 dimensions array
#With for loops we get the function name
#And getting function definition by get_function_content function
for row in rows: #rows return 2 dimension array
for ro in row:
if ro:
try:
get_function_content(ro)
func_content = cur.fetchall()
except psycopg2.Error as err:
print("func content error", err)
#Check for if there is security definer in function content
for func in func_content:
for single_func in func:
try:
if check_function(
single_func
): # if there is a security definer returns true
print_red(
"Critical issue: There should be no SQL function with SECURITY DEFINER. "
)
else:
print_green("OK")
except psycopg2.Error as err:
print("function checking error")