def fill(value, elements=None, locators_list=None, element_name=None):
if elements:
element = waiting_for_element_is_visible(elements=elements)
if element.is_enabled() == True:
element.send_keys(value)
else:
raise ElementClickInterceptedException(
'AIST-element is not enabled')
elif locators_list and element_name:
element = waiting_for_element_is_visible(locators_list=locators_list,
element_name=element_name)
if element.is_enabled() == True:
element.send_keys(value)
else:
raise ElementClickInterceptedException(
'AIST-element is not enabled')
def click_obscured_link(element):
for i in range(20):
try:
element.click()
break
except ElementClickInterceptedException:
time.sleep(1)
raise ElementClickInterceptedException("Unable to click on element")
def click_on(elements=None, locators_list=None, element_name=None):
time.sleep(2)
if elements:
element = waiting_for_element_is_visible(elements=elements)
if element.is_enabled() == True:
element.click()
else:
raise ElementClickInterceptedException(
'AIST-element is not clickable')
elif locators_list and element_name:
element = waiting_for_element_is_visible(locators_list=locators_list,
element_name=element_name)
if element.is_enabled() == True:
element.click()
else:
raise ElementClickInterceptedException(
'AIST-element is not clickable')
def click(self, locator, wait_condition=WaitCondition.ELEMENTPRESENT):
try:
self.context.driver.execute_script(
'arguments[0].click();',
self.get_web_element(locator, wait_condition))
except ElementClickInterceptedException as e:
raise ElementClickInterceptedException(e)
except Exception as e:
raise Exception("Unable to click element {} as {}".format(
locator, e))
def _click(self, obj, count=0):
try:
if obj.is_displayed():
obj.click()
return self
else:
return self._click(obj)
except ElementClickInterceptedException as e:
if (count < 30):
count = self.wait(count)
return self._click(obj, count)
else:
self.raiser(
ElementClickInterceptedException(
f'Not able to click after {count} seconds'))
def wait_for_click(self, xpath):
element = self.get_element(xpath)
try_count = 25
while try_count > 0:
try:
element.click()
except ElementClickInterceptedException:
self.logger.debug("Can not click on {}".format(xpath))
time.sleep(1)
try_count = try_count - 1
except StaleElementReferenceException:
self.logger.debug(
"Stale element exception for {}".format(xpath))
element = self.get_element(xpath)
try_count = try_count - 1
else:
return True
raise ElementClickInterceptedException(
"Failed to click on {}".format(xpath))
def crawle_apartment_details(self, address):
parsed_url = self.prepare_url_request(address)
self.web_driver.get(parsed_url)
if test_google_recaptcha(self.web_driver):
raise ElementClickInterceptedException(
"Can't continue, user must pass the Google reCAPTCHA")
area_details = AreaDetails.init_from_web_driver(self.web_driver)
sales_details = SalesDetails.init_from_web_driver(self.web_driver)
neighborhood_details = NeighborhoodDetails.init_from_web_driver(
self.web_driver)
great_schools = get_great_schools(self.web_driver)
return {
**asdict(area_details),
**asdict(sales_details),
**asdict(neighborhood_details), 'great_schools': great_schools
}
def mercado_parse(self, response):
print('entro en el parse para mercado')
print('\n', response.url)
try: mercado_n_cat = response.meta['mercado_n_cat']
except: mercado_n_cat = 0
trys = 1
while trys <= 5:
try:
print('\n','*'*10, 'INTENTO # ', trys,'*'*10, '\n')
check_connection()
self.driver.get(response.url)
if trys <= 3:
sleep(3)
else:
sleep(5)
buttons = self.driver.find_elements_by_css_selector('html body div#app div.Quicksilver__SurferBoard-sc-1n5zca7-0.jIISCM div.Container__ContainerTemplate-yuoyfe-0.eqBJUR div.Container__ItemTemplate-yuoyfe-1.hopuig div.SubContentStyled-sc-1gjz1on-0.kEzHZm div.ProductsByCategory__DisplayByProductStyled-art75o-1.bWikfl div.ProductsByCategory__Header-art75o-0.RKWEe button.ant-btn.ant-btn-link')
buttons[mercado_n_cat].click()
break
except:
trys +=1
sleep(3)
pag = 1
while True:
print('\n', '#'*15, 'Estamos en la pagina ', pag,'#'*15, '\n')
mercado_pag_sel = Selector(text= self.driver.page_source)
mercado_products = mercado_pag_sel.xpath('.//*[@class= "ProductDisplay__StyledContainer-sc-5nadct-0 dNEKlU"]/div')
cat_name = mercado_pag_sel.xpath('//h1/text()').extract_first()
print('\n', '#'*15, 'De la categoria ', cat_name,'#'*15, '\n')
print('Vamos en la categoria n ', mercado_n_cat+1, len(buttons))
sleep(3)
try:
for mercado_product in mercado_products:
prod_name = mercado_product.xpath('.//*[@class= "product__content__name"]//text()').extract_first()
disc_price = mercado_product.xpath('.//*[@class= "product__content__price__wrapper"]/p/text()').extract()
print(disc_price, 'este es disc_price de largo', len(disc_price))
if len(disc_price) > 1:
print('ESTE ENTRA AL IF')
normal_price = disc_price[-1]
disc_price = disc_price[1]
else:
print('ESTE ENTRA AL ELSE')
normal_price = disc_price[0]
disc_price = disc_price[0]
image_url = mercado_product.xpath('.//*[@class= "product__figure-wrapper__img"]/@src').extract_first()
print('\n', '#'*15, 'Resultado Producto', '#'*15, '\n')
print('Categoria: ', cat_name, '\n',
'\n\tProducto:\t', prod_name,
'\n\tPrecio norm:\t', normal_price,
'\n\tPrecio desc:\t', disc_price, '\n')
yield {'cat_name': cat_name,
'prod_name': prod_name,
'normal_price': normal_price,
'disc_price': disc_price,
'image_url': image_url}
next_button= self.driver.find_elements_by_xpath('/html/body/div[1]/div/div[2]/div/div/div[2]/ul/li')
button_test = mercado_pag_sel.xpath('/html/body/div[1]/div/div[2]/div/div/div[2]/ul/li')
if button_test == []:
pass
else:
button_test = mercado_pag_sel.xpath('/html/body/div[1]/div/div[2]/div/div/div[2]/ul/li')[-1]
button_test = button_test.xpath('.//button[@class= "ant-pagination-item-link"]/@disabled')
if next_button != [] and button_test == []:
#next_button= self.driver.find_elements_by_xpath('/html/body/div[1]/div/div[2]/div/div/div[2]/ul/li/a')[-1].click()
next_button[-1].click()
sleep(3)
else:
raise ElementClickInterceptedException()
pag += 1
except ElementClickInterceptedException:
if mercado_n_cat < len(buttons)-1:
mercado_n_cat += 1
check_connection()
yield Request(url = response.url,
callback= self.mercado_parse,
meta= {'mercado_n_cat': mercado_n_cat},
dont_filter= True)
break
else:
print('\n', '='*20,'\n', 'TAL PARECE QUE SE EXTRAJO TODA LA INFORMACION DE LA PAGINA !!','\n', '='*20, '\n')
self.driver.quit()
break