def run(self):
self.signalConsole.emit(f"Starting map of {self.url}...")
self.subSystem = Scrap(self.url, 0, self.driver, self.targetDevice,
self.configuration, False)
self.getXPaths(self.url, self.filename)
self.signalConsole.emit(f"Map complete.")
self.signalFinished.emit(True)
def scan(self, filename):
self.subSystem = Scrap(self.url, 0, self.driver, self.targetDevice,
self.configuration, False)
paths = self.subSystem.loadXPaths(filename)
#print(paths)
scans = self.subSystemDatabase.getScanTable()
#print(scans)
id = 0
if len(scans) >= 1:
for x in scans:
id += 1
#table = "site_" + str(id)
#print(table)
time = datetime.now()
dt = time.strftime("%d/%m/%Y %H:%M")
self.subSystemDatabase.addScan(id, ".com", "Chrome", dt)
#self.subSystemDatabase.createSiteTable(table)
self.subSystem.openSite(self.url)
try:
count = int(self.subSystemDatabase.getMaxElemID()[0][0]) + 1
except:
count = 0
#print(count)
for item in paths:
#print(item)
ats = self.subSystem.getAllAttributes(self.url, item)
if ats['type'] != "":
self.subSystemDatabase.insertSiteTable(
id, count, ats['type'], self.url, item, ats['nonText'],
ats['decorative'], ats['class'], ats['name'], ats['id'],
ats['alt'], ats['aria-label'], ats['src'])
self.subSystemDatabase.conn.commit()
count += 1
def operationInit(self) -> bool:
trigger = True
try:
#print(self.driver)
self.subSytemScrap = Scrap(self.url, self.navdepth, self.driver, self.targetDevice, self.configuration, self.screenshots)
for linkData,\
linkError,\
imageData,\
imageError,\
urlError,\
invalid,\
external,\
duplicate in self.subSytemScrap.scrapSite():
a = 0
#print("************************************************************************************" + str(len(imageData)))
for item in imageData:
if trigger:
self.signalConsole.emit("Saving data into databases...")
trigger = False
a = item
imageDatas = asdict(a)
'''if imageDatas['image'] == None:
imageDatas['image'] = "Blank"'''
self.subSytemDatabase.insertImageTableData(self.imgTable, self.count , imageDatas['url'], imageDatas['id'], imageDatas['name'], imageDatas['src'], imageDatas['text'], imageDatas['altText'],
imageDatas['width'], imageDatas['height'], imageDatas['x'], imageDatas['y'], str(imageDatas['displayed']), imageDatas['download'], imageDatas['image'])
self.count+=1
self.signalData.emit(
{
"linkData":len(linkData),
"linkError": linkError,
"imageData": len(imageData),
"imageError": imageError,
"urlError": urlError,
"invalid": invalid,
"external": external,
"duplicate": duplicate
}
)
self.signalConsole.emit("Done Saving...")
except Exception as e:
logger.error(f"{e}")
logger.error(e, exc_info=True)
return False
else:
return True
def testing(self, siteID, url):
#self.subsystem.driver.get("https://www.jackboxgames.com/party-pack-five/")
self.subsystem = Scrap(self.url, 0, self.driver, self.targetDevice,
self.configuration, False)
data = self.subSystemDatabase.getSiteData('elements', siteID)
#print(data)
#print(f"Data: {data}")
back = self.subsystem.nonText(self.url, data)
#print(f"Back: {back}")
for item in back:
#print(item["xPath"])
self.subSystemDatabase.addNotesToSite('elements', "wcag11",
item["xPath"],
str(item["errors"]), siteID)
def sanityCheck(args):
import requests
from urllib.parse import urlparse
if not Scrap.checkUrlValid(args.url):
raise requests.ConnectionError(f"invalid url {args.url}")
if args.navdepth < -1:
raise OverflowError("depth range should be [-1, ... ).")
main(args=args)
def _checkValidAddr(self, a0: str) -> bool:
if not a0:
self.ui.btn_addrPrefix.setStyleSheet(
"background-color: white; color: black")
return False
else:
if not Scrap.checkUrlValid(a0):
logger.error(f"{a0} not valid url")
self.ui.btn_addrPrefix.setStyleSheet(
"background-color: red; color: black")
return False
else:
self.ui.btn_addrPrefix.setStyleSheet(
"background-color: green; color: black")
return True
class siteMap(QThread):
signalConsole = pyqtSignal(str)
signalFinished = pyqtSignal(bool)
def __init__(self,
url: str,
filename: str,
driver: str,
targetDevice: Union[str, None],
configuration: Optional[Dict] = None) -> None:
super().__init__()
self.targetDevice = targetDevice
self.url = url
self.configuration = configuration
self.driver = driver
self.filename = filename
self.subSystemDatabase = WACDatabase("sqlite/database.db")
def _setupScrap(self) -> bool:
self.setupDriver()
if not self.isDriverInitialized():
print("driver not initialized!")
return False
return True
def __getattribute__(self, name, *args, **kwargs):
def make_interceptor(callble):
def func(*args, **kwargs):
logger.info(f"{name} {args} {kwargs}")
return callble(*args, **kwargs)
return func
returned = object.__getattribute__(self, name)
if inspect.isfunction(returned) or inspect.ismethod(returned):
return make_interceptor(returned)
return returned
def run(self):
self.signalConsole.emit(f"Starting map of {self.url}...")
self.subSystem = Scrap(self.url, 0, self.driver, self.targetDevice,
self.configuration, False)
self.getXPaths(self.url, self.filename)
self.signalConsole.emit(f"Map complete.")
self.signalFinished.emit(True)
def getXPaths(self, url, filename):
if not self.subSystem._setupScrap():
logger.critical(f"driver setup failed")
self.subSystem.driver.get(url)
#self.subSystem.driver.mana manage().timeouts().implicitlyWait(10)
try:
WebDriverWait(self.subSystem.driver, 10).until(
EC.presence_of_all_elements_located((By.CSS_SELECTOR, '*')))
except:
print("webwait failed")
#time.sleep(5)
'''EC.element_to_be_clickable((By.CLASS_NAME, 'playButton'))
self.subSystem.driver.find_element(By.CLASS_NAME,"ule-onboarding-continue-button").click()
self.subSystem.driver.find_element(By.CLASS_NAME,"playButton").click()'''
elements = self.subSystem.driver.find_elements(By.TAG_NAME, 'html')
tags = []
self.subSystem.driver.implicitly_wait(0.00001)
self.subSystem.paths = []
#self.subSystem.index = 0
self.subSystem.paths.append('/html')
self.trigger = True
for x in elements:
#print(x.get_attribute("innerHTML"))
subElems = x.find_elements_by_xpath(".//*")
self.elemSize = len(subElems)
print(f'Final Total: {self.xPath(subElems,"/html", 1)}')
allpaths = ""
for x in self.subSystem.paths:
#print(x)
#x = x.replace('[1]', '')
#print(x)
allpaths += f"\n{x}"
file = open(filename + ".txt", 'w')
file.write(allpaths)
file.close()
return self.subSystem.paths
def xPath(self, elems, path, size):
index = 0
if self.trigger:
#self.signalConsole.emit(f"Elements to map: {len(elems)}")
self.trigger = False
while len(elems) >= index + 1:
if len(self.subSystem.paths) % (int(
self.elemSize / 20)) == 0 and index != 0:
#print(f"Completed {index} of {len(elems)}" )
perc = int((len(self.subSystem.paths) / self.elemSize) * 100)
self.signalConsole.emit(
f"Completed {len(self.subSystem.paths)} elements. {perc}%"
)
i = 1
while True:
if not (path + '/' + elems[index].tag_name +
f'[{i}]') in self.subSystem.paths:
new = path + '/' + elems[index].tag_name + f'[{i}]'
if elems[index].tag_name == 'svg':
clas = elems[index].get_attribute('class')
new = f'svg {clas}' #f'//svg[@class="{clas}"][@viewbox="{viewbox}"]'
#print(str(at))
if 'path' not in new:
self.subSystem.paths.append(new)
#print(path + '/' + elems[index].tag_name + f'[{i}]')
#print(path + '/' + elems[index].tag_name + f'[{i}] Index: {index}')
break
else:
i += 1
try:
subElems = elems[index].find_elements_by_xpath(".//*")
except:
subElems = []
subSize = self.xPath(subElems,
path + '/' + elems[index].tag_name + f'[{i}]',
size + 1)
index += subSize + 1
return index
def run(self):
self.signalConsole.emit(f"Starting scan of {self.url}")
self.subSystem = Scrap(self.url, 0, self.driver, self.targetDevice,
self.configuration, False)
paths = self.subSystem.loadXPaths(self.filename)
if paths == False:
self.signalConsole.emit(f"Unable to find path file.")
else:
scans = self.subSystemDatabase.getScanTable()
id = 0
if len(scans) >= 1:
for x in scans:
#print(f"x: {x}")
if x[0] > id:
id = x[0]
id += 1
time = datetime.now()
dt = time.strftime("%d/%m/%Y %H:%M")
self.subSystemDatabase.addScan(id, ".com", "Chrome", dt)
self.signalConsole.emit(f"New scanID created: {id}")
self.subSystem.openSite(self.url)
self.signalConsole.emit(f"Chrome driver started...")
try:
count = int(self.subSystemDatabase.getMaxElemID()[0][0]) + 1
except:
count = 0
#print(f'count: {count}')
total = len(paths)
completed = 0
#self.subSystemDatabase.conn.cursor.fast_executemany = True
elements = []
self.scanSignal.emit(str(id))
for item in paths[1:]:
#print(completed)
if not ((('header' in item) or
('svg' in item)) and self.headers):
#time.sleep(1)
#print('inside')
ats = self.subSystem.getAllAttributes(self.url, item)
#print('mid')
if ats['type'] != "":
#print(ats)
#self.subSystemDatabase.insertSiteTable(id, count, ats['type'], self.url, item, ats['alt'], ats['aria-current'], ats['aria-describedby'], ats['aria-hidden'],ats['aria-label'],ats['aria-required'],
# ats['class'], ats['id'], ats['name'], ats['role'], ats['src'], ats['tabindex'], ats['target'],ats['title'], ats['href'], ats['text'], ats['type id'])
elements.append(
(id, count, ats['type'], self.url, item,
ats['alt'], ats['aria-current'],
ats['aria-describedby'], ats['aria-hidden'],
ats['aria-label'], ats['aria-required'],
ats['class'], ats['id'], ats['name'], ats['role'],
ats['src'], ats['tabindex'], ats['target'],
ats['title'], ats['href'], ats['text'],
ats['type id'], ats['scope'], ats['imgText'],
ats['imgTable'],
ats['imgIcon'], ats['srcset'], ats['lang'],
str('html'), ats['fontSize'], ats['data-src']))
count += 1
#print('next')
#print(elements)
completed += 1
if completed == 1:
self.signalConsole.emit(
f"First Element scanned of {total} elements.")
if completed % (int(len(paths) / 20)) == 0:
perc = int((completed / len(paths)) * 100)
self.signalConsole.emit(
f'Completed scan of {completed}/{total} elements. {perc}%'
)
#print('before')
self.subSystemDatabase.insertLargeSiteTable(elements)
self.subSystemDatabase.conn.commit()
elements = []
#print('after')
#print('done')
ret1, ret2 = self.checkTabbing()
print('got back')
if len(ret1) > 0:
err = "Tab progression respects a logical sequence. Error found in tab numbers: "
for item in ret1:
err = err + f'{item},'
self.subSystemDatabase.addNotesToSite("elements", "Rule 1",
'/html', err, id)
if len(ret2) > 0:
err = "Each tab should have h2 element even if blank. These did not "
for item in ret2:
err = err + f'{item},'
self.subSystemDatabase.addNotesToSite("elements", "Rule 2",
'/html', err, id)
#self.subSystemDatabase.conn.cursor.fast_executemany = False
self.signalConsole.emit(f"Scan Complete.")
self.signalFinished.emit(True)
def checkTabbing(self):
self.subsystem = Scrap(self.url, 0, self.driver, self.targetDevice,
self.configuration, False)
return self.subsystem.checkTabbing()
class wcagScan(QThread):
signalConsole = pyqtSignal(str)
scanSignal = pyqtSignal(str)
signalFinished = pyqtSignal(bool)
def __init__(self,
url: str,
filename: str,
driver: str,
headers: bool,
targetDevice: Union[str, None],
configuration: Optional[Dict] = None) -> None:
super().__init__()
self.targetDevice = targetDevice
self.headers = headers
self.url = url
self.filename = filename
self.configuration = configuration
self.driver = driver
self.subSystemDatabase = WACDatabase("sqlite/database.db")
self.numxpath = 4
print(self.targetDevice)
def _setupScrap(self) -> bool:
self.setupDriver()
if not self.isDriverInitialized():
print("driver not initialized!")
return False
return True
def __getattribute__(self, name, *args, **kwargs):
def make_interceptor(callble):
def func(*args, **kwargs):
logger.info(f"{name} {args} {kwargs}")
return callble(*args, **kwargs)
return func
returned = object.__getattribute__(self, name)
if inspect.isfunction(returned) or inspect.ismethod(returned):
return make_interceptor(returned)
return returned
def checkTabbing(self):
self.subsystem = Scrap(self.url, 0, self.driver, self.targetDevice,
self.configuration, False)
return self.subsystem.checkTabbing()
def run(self):
self.signalConsole.emit(f"Starting scan of {self.url}")
self.subSystem = Scrap(self.url, 0, self.driver, self.targetDevice,
self.configuration, False)
paths = self.subSystem.loadXPaths(self.filename)
if paths == False:
self.signalConsole.emit(f"Unable to find path file.")
else:
scans = self.subSystemDatabase.getScanTable()
id = 0
if len(scans) >= 1:
for x in scans:
#print(f"x: {x}")
if x[0] > id:
id = x[0]
id += 1
time = datetime.now()
dt = time.strftime("%d/%m/%Y %H:%M")
self.subSystemDatabase.addScan(id, ".com", "Chrome", dt)
self.signalConsole.emit(f"New scanID created: {id}")
self.subSystem.openSite(self.url)
self.signalConsole.emit(f"Chrome driver started...")
try:
count = int(self.subSystemDatabase.getMaxElemID()[0][0]) + 1
except:
count = 0
#print(f'count: {count}')
total = len(paths)
completed = 0
#self.subSystemDatabase.conn.cursor.fast_executemany = True
elements = []
self.scanSignal.emit(str(id))
for item in paths[1:]:
#print(completed)
if not ((('header' in item) or
('svg' in item)) and self.headers):
#time.sleep(1)
#print('inside')
ats = self.subSystem.getAllAttributes(self.url, item)
#print('mid')
if ats['type'] != "":
#print(ats)
#self.subSystemDatabase.insertSiteTable(id, count, ats['type'], self.url, item, ats['alt'], ats['aria-current'], ats['aria-describedby'], ats['aria-hidden'],ats['aria-label'],ats['aria-required'],
# ats['class'], ats['id'], ats['name'], ats['role'], ats['src'], ats['tabindex'], ats['target'],ats['title'], ats['href'], ats['text'], ats['type id'])
elements.append(
(id, count, ats['type'], self.url, item,
ats['alt'], ats['aria-current'],
ats['aria-describedby'], ats['aria-hidden'],
ats['aria-label'], ats['aria-required'],
ats['class'], ats['id'], ats['name'], ats['role'],
ats['src'], ats['tabindex'], ats['target'],
ats['title'], ats['href'], ats['text'],
ats['type id'], ats['scope'], ats['imgText'],
ats['imgTable'],
ats['imgIcon'], ats['srcset'], ats['lang'],
str('html'), ats['fontSize'], ats['data-src']))
count += 1
#print('next')
#print(elements)
completed += 1
if completed == 1:
self.signalConsole.emit(
f"First Element scanned of {total} elements.")
if completed % (int(len(paths) / 20)) == 0:
perc = int((completed / len(paths)) * 100)
self.signalConsole.emit(
f'Completed scan of {completed}/{total} elements. {perc}%'
)
#print('before')
self.subSystemDatabase.insertLargeSiteTable(elements)
self.subSystemDatabase.conn.commit()
elements = []
#print('after')
#print('done')
ret1, ret2 = self.checkTabbing()
print('got back')
if len(ret1) > 0:
err = "Tab progression respects a logical sequence. Error found in tab numbers: "
for item in ret1:
err = err + f'{item},'
self.subSystemDatabase.addNotesToSite("elements", "Rule 1",
'/html', err, id)
if len(ret2) > 0:
err = "Each tab should have h2 element even if blank. These did not "
for item in ret2:
err = err + f'{item},'
self.subSystemDatabase.addNotesToSite("elements", "Rule 2",
'/html', err, id)
#self.subSystemDatabase.conn.cursor.fast_executemany = False
self.signalConsole.emit(f"Scan Complete.")
self.signalFinished.emit(True)
def mapSite(self, filename):
self.subSystem = Scrap(self.url, 0, self.driver, self.targetDevice,
self.configuration, False)
self.subSystem.getXPaths(self.url, filename)
def openElem(self, xPath):
self.subsystem = Scrap(self.url, 0, self.driver, self.targetDevice,
self.configuration, False)
self.signalConsole.emit("Opening Element")
self.subsystem.openSite(self.url)
self.subsystem.openWebElement(self.url, str(xPath))
def output(self, scanID, filename):
errors = []
print(scanID)
#self.subSystemDatabase2 = WACDatabase("sqlite/database.db")
data = self.subSystemDatabase.getSiteErrors('elements', scanID)
#print(data[0:200])
for item in data:
#print(item[self.numxpath])
add = False
ers = []
ers.append(item[3])
ers.append(item[4])
ers.append(item[2])
for i in range(60):
#print(f'item: {item[i + 31]} i = {i}')
if item[i + 30] != None:
if len(item[i + 30]) > 2:
ers.append(f"Rule {i}, {item[i + 30]}")
print(item[i + 30])
add = True
if add:
print(f"{item[0:4]} + {ers}")
errors.append(ers)
print(f"errors: {errors[0:10]}")
self.ws = Xlsx(filename + '_errors.xlsx', "Sheet1")
for x in errors:
self.ws.addRows(x)
self.ws.set_col_widths_errors(100)
self.ws.saveWorkbook()
def openElem(self, xPath):
self.subsystem = Scrap(self.url, 0, self.driver, self.targetDevice,
self.configuration, False)
self.signalConsole.emit("Opening Element")
self.subsystem.openSite(self.url)
self.subsystem.openWebElement(self.url, str(xPath))
def mapSite(self, filename):
self.subSystem = Scrap(self.url, 0, self.driver, self.targetDevice,
self.configuration, False)
self.subSystem.getXPaths(self.url, filename)
class OperationScrapSite(QThread):
signalConsole = pyqtSignal(str)
signalData = pyqtSignal(dict)
signalTeardown = pyqtSignal(bool)
def __init__(
self,
db: str,
url: str,
navdepth: int,
driver: str,
targetDevice: str,
screenshots: bool,
configuration: Optional[Dict] = None,
) -> None:
super().__init__()
self.count = 0
self.url = url
self.navdepth = navdepth
self.driver = driver
self.targetDevice = targetDevice
self.configuration = configuration
self.db = db
self.screenshots = screenshots
#
self.emid: Optional[int] = None
self.ssid: Optional[int] = None
self.urlTable: str = ""
self.imgTable: str = ""
self.btnTable: str = ""
#
self.countUrlTotal:int = 0
self.countUrlDomain:int = 0
self.countUrlExternal:int = 0
self.countUrlInvalid:int = 0
self.countUrlErrors:int = 0
self.countImagesTotal:int = 0
self.countImagesScrapped:int = 0
self.countImagesDownloaded:int = 0
self.countImagesScreenshot:int = 0
self.countImagesErrors:int = 0
self.subSytemDatabase = WACDatabase(db)
def __getattribute__(self, name, *args, **kwargs):
def make_interceptor(callble):
def func(*args, **kwargs):
logger.info(f"{name} {args} {kwargs}")
return callble(*args, **kwargs)
return func
returned = object.__getattribute__(self, name)
if inspect.isfunction(returned) or inspect.ismethod(returned):
return make_interceptor(returned)
return returned
'''def updateData(self,data:dict):
self.signalData.emit(data)'''
def run(self):
ret = self.operationWarmup()
if ret:
try:
self.subSytemDatabase.conn.commit()
except Exception as e:
logger.error(f"{e}")
logger.error(e, exc_info=True)
return
else:
self.signalConsole.emit(f"warmup complete, scrapping now...")
else:
self.signalConsole.emit(f"error: warmup failed, not scrapping")
return
ret = self.operationInit()
if ret:
self.signalConsole.emit(f"scrap completed, performing post process functions")
else:
self.signalConsole.emit(f"error: scrap process failed!")
return
self.subSytemScrap.driver.close()
def operationWarmup(self) -> bool:
# hash key to match previous configuration entry
hKey = createHashMD5(
[self.url, self.driver, self.targetDevice, self.navdepth]
)
# get id from entry
self.emid = self.subSytemDatabase.selectEMIDInEntry(self.url)
flagScrapSetting = True
print("EMID = " + str(self.emid))
# if id in entry present, search for entries in scrap configs
if self.emid:
self.signalConsole.emit(f"previous entry found for {self.url}")
ret = self.subSytemDatabase.selectAllScrapSetting(EMID=self.emid)
# if configurations present , check for hashkey
if ret:
self.signalConsole.emit(f"{len(ret)} scrap entries found for {self.url}")
for ret_ in ret:
r_ = nt_ScrapSetting._make(ret_)
if r_.HASHKEY == hKey:
self.signalConsole.emit(f"scrap found for current config HASHKEY:{hKey} on date:{r_.date}")
# if a hashkey present for same date skip
if r_.date == str(datetime.datetime.now().date()):
self.signalConsole.emit(f"choose different scrap configuration (for same date)")
return False
# if entry not present , create new
elif not self.emid:
try:
self.emid = self.subSytemDatabase.getMaxId("Entry") + 1
except:
self.emid = 1
self.signalConsole.emit(f"creating new entry for {self.url}")
flag = self.subSytemDatabase.insertInEntry(self.url, status="processing", desc="None")
if not flag:
self.signalConsole.emit(f"error: could not create new entry for {self.url}")
return False
# get id of new entry
# if configuration not present create new
if flagScrapSetting:
self.signalConsole.emit(f"creating new scrap configuration " + str(self.emid))
flag = self.subSytemDatabase.insertInScrapSetting(
self.emid, hKey, self.driver, self.targetDevice, self.navdepth #type:ignore
)
if not flag:
self.signalConsole.emit(f"error: could not create new scrap configuration")
return False
else:
self.signalConsole.emit(f"proceeding to create tables")
else:
print("not flagged")
# get ssid for new configuration entry
self.ssid = self.subSytemDatabase.getMaxId("ScrapSetting")
# create table for elements
self.signalConsole.emit(f"creating element tables entries")
etrid = self.subSytemDatabase.getMaxId("ElementTableRef")
if isinstance(etrid, int):
if etrid == 0: etrid = 1
else: etrid+=1
else:
self.signalConsole.emit(f"error: fetching element table id")
return False
# table names
self.imgTable = prefix_EleImgTable+str(etrid)
self.urlTable = prefix_EleUrlTable+str(etrid)
self.btnTable = prefix_EleBtnTable+str(etrid)
flag = self.subSytemDatabase.insertInElementTableRef(
self.ssid, #type:ignore
self.urlTable, self.imgTable, self.btnTable
)
if not flag:
self.signalConsole.emit(f"error: could not create table entries")
return False
else:
self.signalConsole.emit(f"creating new element tables")
if not self.subSytemDatabase.creatTableImg(self.imgTable):
self.signalConsole.emit(f"error: cant creat new element table {self.imgTable}")
return False
if not self.subSytemDatabase.creatTableUrl(self.urlTable):
self.signalConsole.emit(f"error: cant creat new element table {self.urlTable}")
return False
return True
def operationInit(self) -> bool:
trigger = True
try:
#print(self.driver)
self.subSytemScrap = Scrap(self.url, self.navdepth, self.driver, self.targetDevice, self.configuration, self.screenshots)
for linkData,\
linkError,\
imageData,\
imageError,\
urlError,\
invalid,\
external,\
duplicate in self.subSytemScrap.scrapSite():
a = 0
#print("************************************************************************************" + str(len(imageData)))
for item in imageData:
if trigger:
self.signalConsole.emit("Saving data into databases...")
trigger = False
a = item
imageDatas = asdict(a)
'''if imageDatas['image'] == None:
imageDatas['image'] = "Blank"'''
self.subSytemDatabase.insertImageTableData(self.imgTable, self.count , imageDatas['url'], imageDatas['id'], imageDatas['name'], imageDatas['src'], imageDatas['text'], imageDatas['altText'],
imageDatas['width'], imageDatas['height'], imageDatas['x'], imageDatas['y'], str(imageDatas['displayed']), imageDatas['download'], imageDatas['image'])
self.count+=1
self.signalData.emit(
{
"linkData":len(linkData),
"linkError": linkError,
"imageData": len(imageData),
"imageError": imageError,
"urlError": urlError,
"invalid": invalid,
"external": external,
"duplicate": duplicate
}
)
self.signalConsole.emit("Done Saving...")
except Exception as e:
logger.error(f"{e}")
logger.error(e, exc_info=True)
return False
else:
return True
class wcag11(QThread):
signalConsole = pyqtSignal(str)
def __init__(self,
url: str,
driver: str,
targetDevice: Union[str, None],
configuration: Optional[Dict] = None) -> None:
super().__init__()
self.targetDevice = targetDevice
self.url = url
self.configuration = configuration
self.driver = driver
self.subSystemDatabase = WACDatabase("sqlite/database.db")
def _setupScrap(self) -> bool:
self.setupDriver()
if not self.isDriverInitialized():
print("driver not initialized!")
return False
return True
#def run
def __getattribute__(self, name, *args, **kwargs):
def make_interceptor(callble):
def func(*args, **kwargs):
logger.info(f"{name} {args} {kwargs}")
return callble(*args, **kwargs)
return func
returned = object.__getattribute__(self, name)
if inspect.isfunction(returned) or inspect.ismethod(returned):
return make_interceptor(returned)
return returned
def start(self):
self.subSystemDatabase.createSiteTable("Test")
def testing(self, siteID, url):
#self.subsystem.driver.get("https://www.jackboxgames.com/party-pack-five/")
self.subsystem = Scrap(self.url, 0, self.driver, self.targetDevice,
self.configuration, False)
data = self.subSystemDatabase.getSiteData('elements', siteID)
#print(data)
#print(f"Data: {data}")
back = self.subsystem.nonText(self.url, data)
#print(f"Back: {back}")
for item in back:
#print(item["xPath"])
self.subSystemDatabase.addNotesToSite('elements', "wcag11",
item["xPath"],
str(item["errors"]), siteID)
def scan(self, filename):
self.subSystem = Scrap(self.url, 0, self.driver, self.targetDevice,
self.configuration, False)
paths = self.subSystem.loadXPaths(filename)
#print(paths)
scans = self.subSystemDatabase.getScanTable()
#print(scans)
id = 0
if len(scans) >= 1:
for x in scans:
id += 1
#table = "site_" + str(id)
#print(table)
time = datetime.now()
dt = time.strftime("%d/%m/%Y %H:%M")
self.subSystemDatabase.addScan(id, ".com", "Chrome", dt)
#self.subSystemDatabase.createSiteTable(table)
self.subSystem.openSite(self.url)
try:
count = int(self.subSystemDatabase.getMaxElemID()[0][0]) + 1
except:
count = 0
#print(count)
for item in paths:
#print(item)
ats = self.subSystem.getAllAttributes(self.url, item)
if ats['type'] != "":
self.subSystemDatabase.insertSiteTable(
id, count, ats['type'], self.url, item, ats['nonText'],
ats['decorative'], ats['class'], ats['name'], ats['id'],
ats['alt'], ats['aria-label'], ats['src'])
self.subSystemDatabase.conn.commit()
count += 1
def nonText(self, url, data):
altNum = 10
nameNum = 8
idNum = 9
ariaNum = 11
classNum = 7
nonTextElems = ['img', 'button', 'input', 'video']
elements = []
#print(data)
for item in data:
#print(item)
#if nonText
if item[5] == "True":
xPath = item[4] #.replace('[1]', '')
#elem = self.driver.find_element_by_xpath(xPath)
#elements.append(self.driver.find_element_by_xpath(xPath))
elements.append({
'type': item[2],
'attributes': item,
'element': None,
'errors': [],
'xPath': xPath
})
#print(elements)
#except:
#print(f'Failed Path: {item}')
#print(elements)
for item in elements:
#print(item)
# Get name and alt text
try:
alt = item['attributes'][altNum]
#print(f"alt: {alt}")
except:
item['errors'].append(
"WCAG 1.1.1 - All non-text content should have an alt text."
)
alt = "Broke"
#print(f"alt: {alt}")
try:
name = item['attributes'][nameNum]
#print(f"name: {name}")
except:
item['errors'].append(
"Non-text content should have a name. Controls Must have a name."
)
name = ""
#item['errors'].append(f"Id: {item['attributes'][idNum]}")
try:
if item['type'] == "input" or item['type'] == "button":
aria = item['attributes'][ariaNum]
if not aria == "":
alt = aria
else:
item['errors'].append(
"WCAG 1.1.1 - Control did not have an aria label to replace alt."
)
elif "captcha" in item['attributes'][classNum]:
item['errors'].append(
"WCAG 1.1.1 - Exception: Captcha not tested for alt text."
)
except:
pass
#Is displayed
#Error not displayed - never displayed?
#item['errors'].append(f"alt = {alt}")
item['attributes'] = (f"Name: {name} Alt: {alt}. ")
#print(item['attributes'])
# Test for name and alt text
if not len(str(alt)) >= 1:
item['errors'].append(
"WCAG 1.1.1 - All non-text content should have an alt text."
)
if not len(str(name)) >= 1:
item['errors'].append(
"Non-text content should have a name. Controls Must have a name."
)
# compare textdistance
temp = textdistance.jaro_winkler(
str(name).lower(),
str(alt).lower())
if temp <= .2:
item['errors'].append(
f"WCAG 1.1.1 - Name of element should match alt text or aria-label."
)
#print(elements)
return elements
'''self.ws = Xlsx('WWC.xlsx', "Sheet1")
for x in elems:
self.ws.addRows([x['type'], x['attributes'], x['errors']])
self.ws.set_col_widths(80)
self.ws.saveWorkbook()
self.driver.close()'''
def mapSite(self, filename):
self.subSystem = Scrap(self.url, 0, self.driver, self.targetDevice,
self.configuration, False)
self.subSystem.getXPaths(self.url, filename)
def openElem(self, xPath):
self.subsystem = Scrap(self.url, 0, self.driver, self.targetDevice,
self.configuration, False)
self.signalConsole.emit("Opening Element")
self.subsystem.openWebElement(self.url, str(xPath))