def waitAnswer(self):
time = '[' + str(now()) + ']'
print(c.green(time), self.info(), '| waiting answer')
response = self.queue[TypeQ.MSG_RECEIVER].get(block=True)
print(c.green(time), self.info(), '| response: ',
c.orange(msg.readText(response)))
return response
def get_single_element_BY(self, target):
element = None
assert isinstance(
target,
dict), 'Target must be a dictionary with key:value [type:target]'
try:
c_target = target.copy()
_type, _target = c_target.popitem()
if _type is CLASS:
element = self.__get_element_by_class(_target)
elif _type is ID:
element = self.__get_element_by_id(_target)
elif _type is XPATH:
element = self.__get_element_by_xpath(_target)
elif _type is TAG:
element = self.__get_element_by_tag(_target)
else:
print('Error - Type ( {} ).'.format(_type))
except Exception as e:
print(
'Exception: {}: {} -> {} - is not present in page: {}'.format(
e, c.blue(_type), c.red(_target),
c.orange(self.webBrowser.current_url)))
finally:
if self.log and element is not None:
print('{} ---> {}'.format(c.blue(_type), c.green(_target)))
return element
def find_elements_BY(self, element, target):
nested_element = None
assert isinstance(
target,
dict), 'Target must be a dictionary with key:value [type:target]'
try:
c_target = target.copy()
_type, _target = c_target.popitem()
if _type is CLASS:
nested_element = self.__find_elements_by_class(
element, _target)
if _type is ID:
nested_element = self.__find_elements_by_id(element, _target)
if _type is XPATH:
nested_element = self.__find_elements_by_xpath(
element, _target)
if _type is TAG:
nested_element = self.__find_elements_by_tag(element, _target)
if _type is LINK:
nested_element = self.__find_elements_by_partial_link_text(
element, _target)
except Exception as e:
print(
'Exception: {}: {} -> {} - is not present in page: {}'.format(
e, c.blue(_type), c.red(_target),
c.orange(self.webBrowser.current_url)))
finally:
if self.log and nested_element is not None:
print('{} ---> {}'.format(c.blue(_type), c.green(_target)))
return nested_element
def __init__(self, delay=10, log=False, headless=False, tor=False):
# Binary path
path = pkg_resources.resource_filename(pkg_name, geckodriver_file)
profile = webdriver.FirefoxProfile()
if tor:
print(c.red('Warning '), c.blue(
'Be sure you have Tor browser opened in backgroud!'))
print('Tor Proxy', c.green('Enabled'))
profile.set_preference('network.proxy.type', 1)
profile.set_preference('network.proxy.socks', '127.0.0.1')
profile.set_preference('network.proxy.socks_port', 9150)
profile.set_preference("browser.cache.disk.enable", False)
profile.set_preference("browser.cache.memory.enable", False)
profile.set_preference("browser.cache.offline.enable", False)
profile.set_preference("network.http.use-cache", False)
options = Options()
if headless:
options.add_argument('-headless')
self.webBrowser = webdriver.Firefox(
firefox_profile=profile, executable_path=path, firefox_options=options)
self.delay = delay
self.log = log
self.log_file = 'geckodriver.log'
self.wait = WebDriverWait(self.webBrowser, timeout=delay)
def what(self, text=None):
D = c.green('@what')
if text is None:
text = input('What you want search -> ')
search = Page.scraper.get_element_BY(Target.Search.name)
search.send_keys(text)
print(Page.I + self.I + D, c.underline(text))
def area(self, idx=None):
D = c.green('@area')
area = Page.scraper.get_element_BY(Target.Search.area)
view = Select(area)
options = view.options
print(c.underline('Geographic Area:'))
for i, op in enumerate(options):
print('[{}] - {}'.format(c.orange(str(i)), c.blue(op.text)))
if idx is None:
idx = int(input('insert the number [i] -> '))
chs = options[idx].text
options[idx].click()
print(Page.I + self.I + D, c.underline(chs))
def catagory(self, idx=None):
D = c.green('@catagory')
category = Page.scraper.get_element_BY(Target.Search.category)
view = Select(category)
options = view.options
print(c.underline('Category:'))
for i, op in enumerate(options):
print('[{}] - {}'.format(c.orange(str(i)), c.blue(op.text)))
if idx is None:
idx = int(input('\nInsert number [i] -> '))
chs = options[idx].text
options[idx].click()
print(Page.I + self.I + D, c.underline(chs))
def next(self, check=False):
D = c.green('@next')
next_b = Page.scraper.get_element_BY(Target.ListAd.button_next)
link = Page.scraper.find_elements_BY(next_b, Target.ListAd.link)
if link:
if check:
return True
Page.scraper.openUrl(link)
print(Page.I + self.I + D, c.underline('Next'))
return True
print(Page.I + self.I + D, c.red('Finish'))
return False
def links(self):
D = c.green('@links')
links = []
raw_ads = Page.scraper.get_nested_elements_from_root(
Target.ListAd.list_link_element)
for raw_ad in raw_ads:
raw_link = Page.scraper.find_elements_BY(
raw_ad, Target.ListAd.raw_link)
link = Page.scraper.find_elements_BY(raw_link[0],
Target.ListAd.link)
links.append(link)
print(Page.I + self.I + D, 'Founded links: ',
c.underline(str(len(links))))
return links
def __init__(self, user, queue):
Thread.__init__(self)
self.user = user
self.entityName = self.__class__.__name__
self.daemon = True
self.timeStart = now()
time = '[' + str(self.timeStart) + ']'
print(c.green(time), ' Start ', self.info())
if queue:
self.queue = queue
else:
self.queue = {
TypeQ.MSG_SENDER: Queue(),
TypeQ.MSG_RECEIVER: Queue(),
TypeQ.SWITCH_CONVERSATION: Queue()
}
def sendTextWithKeyboard(self, text, buttonList):
self.addMessageToSenderQ(
msg.createKeyboardMessage(self.user['id'], text, buttonList))
time = '[' + str(now()) + ']'
print(c.green(time), self.info(), '| sent: ', c.light_blue(text))
return self.waitAnswer()
def sendText(self, text):
self.addMessageToSenderQ(msg.createTextMessage(self.user['id'], text))
time = '[' + str(now()) + ']'
print(c.green(time), self.info(), '| sent: ', c.light_blue(text))
def continue_button(self):
D = c.green('@continue_button')
b_continue = Page.scraper.get_element_BY(
Target.Search.button_continue)
b_continue.click()
print(Page.I + self.I + D, c.underline('Continue'))
def goto_url(self):
D = c.green('@goto_url')
url = 'https://www.subito.it'
Page.scraper.openUrl(url)
print(Page.I + self.I + D, c.underline(url))