def taskAandB():
print('Aufgabe a)')
print('implementieren eines Scrapers. Siehe ./scraper/ ')
print('\n' + 'Aufgabe b)')
print('Seiten holen mit \'scrapy.exe crawl dataScraper\' und speichern unter ./scraper/')
pages = pa.Pages()
return pages
def getUnusedLangStrings():
lang = xmls.Language('./files/languages/en.xml')
pageData = pages.Pages('./files/pages/')
pageData.loadXML('./files/base/styles.xml')
pageData.loadXML('./files/base/c_page.xml')
pageData.loadXML('./files/base/powerPanel.xml')
pageData.loadXML('./files/ui.xml')
pageData.loadXML('./files/portrait.xml')
pageLang = utils.removeDublicate(pageData.langStrings)
uncommonList = []
flag = 0
for item in lang.langStrings:
for mItem in pageLang:
#print(f'comparing {item.get("langString")} with {mItem}')
if item.get('langString') == mItem:
#print(f'matched {item.get("langString")} with {mItem}')
flag = 1
break
else:
flag = 0
if flag == 0:
uncommonList.append(item)
for item in uncommonList:
print(f'Line {item.get("lineNo")} : {item.get("langString")}')
print(len(uncommonList))
def getUnusedFonts():
styleFile = StyleFile('./files/base/styles.xml')
p = pages.Pages('./files/pages/')
p.loadXML('./files/base/styles.xml')
p.loadXML('./files/base/c_page.xml')
p.loadXML('./files/base/powerPanel.xml')
p.loadXML('./files/ui.xml')
p.loadXML('./files/portrait.xml')
RAW = ""
result = []
for page in p.pageData:
rawTexts = page.get("lines")
for line in rawTexts:
RAW += line
fonts = utils.removeDublicate(styleFile.usedFont)
for font in fonts:
if font not in RAW:
result.append(font)
print("Unused Fonts - ")
for unused in result:
print(unused)
def __init__(self):
''' connect to midi device and set up ros
'''
self.sp = SmartPAD('SmartPAD')
self.axis = Axis()
self.axis.init()
self.web = pages.Pages()
rospy.loginfo("Using input device %s" % self.sp.device)
os.environ['PORT'] = '8081'
def getUnusedImages():
x = Xmlres('./files/base/images.xml')
p = pages.Pages('./files/pages/')
p.loadXML('./files/base/styles.xml')
p.loadXML('./files/base/c_page.xml')
p.loadXML('./files/base/powerPanel.xml')
p.loadXML('./files/ui.xml')
p.loadXML('./files/portrait.xml')
result = []
RAW = p.getRAW()
for var in x.fileVars:
if var.get('varName') not in RAW:
result.append(var.get('varName'))
return result
def main():
global driver
for x in range(0, 2000):
try:
driver = webdriver.Chrome('D:/chromedriver.exe')
time.sleep(2)
runner = pages.Pages(driver, search_name)
runner.route()
time.sleep(2)
print('looking for ', search_name)
runner.goto()
time.sleep(2)
print(x, 'window')
driver.close()
time.sleep(1)
except:
print("something went wrong !ERROR!")
time.sleep(60)
print('you made it ! Congrat')
def getUnusedStyles():
styleFile = StyleFile('./files/base/styles.xml')
p = pages.Pages('./files/pages/')
p.loadXML('./files/base/styles.xml')
p.loadXML('./files/base/c_page.xml')
p.loadXML('./files/base/powerPanel.xml')
p.loadXML('./files/ui.xml')
p.loadXML('./files/portrait.xml')
RAW = ""
result = []
for page in p.pageData:
rawTexts = page.get("lines")
for line in rawTexts:
RAW += line
for style in styleFile.styles:
if style not in RAW:
result.append(style)
print("Unused Styles - ")
for unused in result:
print(unused)
def run(self):
for event in pygame.event.get():
if event.type == pygame.QUIT:
return
elif event.type == pygame.VIDEORESIZE:
pygame.display.set_mode(event.size, pygame.RESIZABLE)
break
g.init()
if not self.journal:
utils.load()
self.pages = pages.Pages()
self.map1 = map1.Map1()
self.ctry = ctry.Ctry()
load_save.retrieve()
self.buttons_setup()
if self.canvas is not None:
self.canvas.grab_focus()
ctrl = False
going = True
answer_input = False
down_event = None
while going:
if self.journal:
# Pump Gtk messages.
while Gtk.events_pending():
Gtk.main_iteration()
# Pump PyGame messages.
for event in pygame.event.get():
if event.type == pygame.QUIT:
if not self.journal:
utils.save()
going = False
elif event.type == pygame.MOUSEMOTION:
g.pos = event.pos
g.redraw = True
if self.canvas is not None:
self.canvas.grab_focus()
elif event.type == pygame.MOUSEBUTTONDOWN:
# Store the latest MOUSEBUTTONDOWN event
if event.button == 1:
down_event = event
elif event.type == pygame.MOUSEBUTTONUP:
g.redraw = True
self.ctry.message = None
g.pic = g.globe
if event.button == 1:
if self.proximity(
event, down_event) and answer_input is False:
if self.do_click():
pass
else:
bu = buttons.check()
if bu != '':
value = self.do_button(bu)
if value == 0:
answer_input = True
elif self.proximity(
event, down_event) and answer_input is True:
res = self.ctry.which_oval()
if res == 'y':
self.check_response()
else:
self.ctry.message = "Sorry, " + self.ctry.answer +\
" is not on my list"
self.ctry.answer = ''
answer_input = False
self.flush_queue()
if event.button == 3:
self.ctry.clear()
elif event.type == pygame.KEYDOWN:
# throttle keyboard repeat
self.ctry.message = None
g.pic = g.globe
if ctrl:
if event.key == pygame.K_q:
if not self.journal:
utils.save()
going = False
break
else:
ctrl = False
if answer_input is False:
if event.key in (pygame.K_LCTRL, pygame.K_RCTRL):
ctrl = True
break
value = self.do_key(event.key)
if value == 0:
answer_input = True
else:
if event.key == g.YES: # Value of 'y'
self.check_response()
else:
self.ctry.message = "Sorry, " + self.ctry.answer +\
" is not on my list"
self.ctry.answer = ''
answer_input = False
g.redraw = True
self.flush_queue()
elif event.type == pygame.KEYUP:
ctrl = False
if not going:
break
if g.redraw:
self.display()
if g.version_display:
utils.version_display()
g.screen.blit(g.pointer, g.pos)
pygame.display.flip()
g.redraw = False
g.clock.tick(40)
def run(self):
g.init()
if not self.journal: utils.load()
self.pages = pages.Pages()
self.map1 = map1.Map1()
self.ctry = ctry.Ctry()
load_save.retrieve()
self.buttons_setup()
if self.canvas <> None: self.canvas.grab_focus()
ctrl = False
pygame.key.set_repeat(600, 120)
key_ms = pygame.time.get_ticks()
going = True
while going:
if self.journal:
# Pump GTK messages.
while gtk.events_pending():
gtk.main_iteration()
# Pump PyGame messages.
for event in pygame.event.get():
if event.type == pygame.QUIT:
if not self.journal: utils.save()
going = False
elif event.type == pygame.MOUSEMOTION:
g.pos = event.pos
g.redraw = True
if self.canvas <> None: self.canvas.grab_focus()
elif event.type == pygame.MOUSEBUTTONDOWN:
g.redraw = True
self.ctry.message = None
g.pic = g.globe
if event.button == 1:
if self.do_click():
pass
else:
bu = buttons.check()
if bu != '': self.do_button(bu)
self.flush_queue()
if event.button == 3: self.ctry.clear()
elif event.type == pygame.KEYDOWN:
# throttle keyboard repeat
if pygame.time.get_ticks() - key_ms > 110:
self.ctry.message = None
g.pic = g.globe
key_ms = pygame.time.get_ticks()
if ctrl:
if event.key == pygame.K_q:
if not self.journal: utils.save()
going = False
break
else:
ctrl = False
if event.key in (pygame.K_LCTRL, pygame.K_RCTRL):
ctrl = True
break
self.do_key(event.key)
g.redraw = True
self.flush_queue()
elif event.type == pygame.KEYUP:
ctrl = False
if not going: break
if g.redraw:
self.display()
if g.version_display: utils.version_display()
g.screen.blit(g.pointer, g.pos)
pygame.display.flip()
g.redraw = False
g.clock.tick(40)
import re
from urllib.parse import urlparse
import extract as ex
import tokenizer
import pages
import timer_debug as td
#16091663,75952015
#SEEDURL = https://www.ics.uci.edu,https://www.cs.uci.edu,https://www.informatics.uci.edu,https://www.stat.uci.edu
my_pages = pages.Pages()
timer = td.TimerDebug()
def print_data():
f = open("HW2output.txt", "w+")
unique_pages = my_pages.get_all_links_visited()
# Question 1: How many unique pages did you find? Uniqueness is established by the URL, but discarding the fragment
# part. So, for example, http://www.ics.uci.edu#aaa and http://www.ics.uci.edu#bbb are the same URL.
# print("Number Of Unique Pages:",len(unique_pages))
f.write("Number Of Unique Pages: {0}\n".format(len(unique_pages)))
# f.write(str("Number Of Unique Pages:", len(unique_pages)))
largest_page, number_of_tokens = my_pages.get_largest_page()
# Question 2: What is the longest page in terms of number of words? (HTML markup doesn’t count as words)
# print("Longest Page:", largest_page, '\t Number Of Words:', number_of_tokens)
