def click_and_get_response(cls, url, **kwargs):
# Note the website need to run in USA to be able to have a good result
"""
url -- type is str
return list of SeleniumResponse
"""
try:
from scrapy_balloons.spiders.balloon import config
from scrapy_balloons.selenium_api import slm_config, driver, balloon_spider
slm_response = SeleniumResponse(kwargs.get('request'))
slmStep = SlmStep(slm_config['click'])
driver.get(url)
time.sleep(slmStep.sleep)
slm.find_element(xpath="//select[@id='ddlFOS']/option[@value='1']").click()
time.sleep(slmStep.sleep)
slm.find_element(xpath="//select[@id='ddlCategory']/option[@value='1']").click()
time.sleep(slmStep.sleep)
slm.find_element(id='btnSearch2').click()
time.sleep(slmStep.sleep)
WebDriverWait(driver, 120).until(EC.invisibility_of_element_located((By.XPATH, "//div[@id='UpdateProgress1']")))
time.sleep(slmStep.sleep)
slm_response.add_html_res(url, driver.page_source.encode('utf-8'))
except:
pass
return slm_response.get_html_res()
def click_and_get_response(cls, url, **kwargs):
# Note the website need to run in USA to be able to have a good result
"""
url -- type is str
return list of SeleniumResponse
"""
try:
from scrapy_balloons.spiders.balloon import config
from scrapy_balloons.selenium_api import slm_config, driver, balloon_spider
slmStep = SlmStep(slm_config['click'])
slm_response = SeleniumResponse(kwargs.get('request'))
driver.get(url)
wait = ui.WebDriverWait(driver, 3)
slmStep = SlmStep(slm_config['click'])
print "url %s add response 0 " % (url)
slm_response.add_html_res(url, driver.page_source.encode('utf-8'))
if driver.find_element_by_xpath(slmStep.xpath):
for i in range(1, slmStep.repeat):
target = wait.until(lambda driver: driver.find_element_by_xpath(slmStep.xpath))
target.click()
time.sleep(slmStep.sleep)
print "url %s add response %s " % (url, i)
slm_response.add_html_res(url, driver.page_source.encode('utf-8'))
except:
print "Paging review end"
pass
return slm_response.get_html_res()
def click_multi_and_follow(self, response):
def build_xpath_cat(indice):
return "//a[@href='#%s']" % (indice)
def build_xpath_next(indice):
return "//div[@id='%s']//a[not(contains(@class,'k-state-disabled')) and contains(./span,'arrow-e')]" % (indice)
try:
from scrapy_balloons.spiders.balloon import config
from scrapy_balloons.selenium_api import slm_config, driver, balloon_spider
slmStep = SlmStep(slm_config['click_multi'])
slm_response = SeleniumResponse()
if self.is_click is False:
driver.get(response.url)
indices = ["SearchTabs-1", "SearchTabs-2", "SearchTabs-3"]
# add response when starting
slm_response.add_html_res(response.url, driver.page_source.encode('utf-8'))
for i, indice in enumerate(indices):
try:
if i>0:
# try to find and click
for j in range(0, 10):
try :
time.sleep(slmStep.sleep)
linkCat = driver.find_element_by_xpath(build_xpath_cat(indice))
if linkCat is None:
print "Retry to find the linkcat xpath %s %s times "%(build_xpath_cat(indice), j)
else:
linkCat.click()
time.sleep(slmStep.sleep)
break
except:
traceback.print_exc()
pass
# add response when starting new category
slm_response.add_html_res(response.url, driver.page_source.encode('utf-8'))
for i in range(0,5):
elm = WebDriverWait(driver, slmStep.sleep).until(
EC.presence_of_element_located((By.XPATH, build_xpath_next(indice)))
)
elm.click()
print "Click on the next link"
WebDriverWait(driver, slmStep.timeout).until(EC.invisibility_of_element_located((By.XPATH, "//div[@class='k-loading-mask']")))
#add response when the click is successfully
slm_response.add_html_res(response.url, driver.page_source.encode('utf-8'))
except:
log.msg("learnnowonline.click_multi_and_follow has no more link to click ")
pass
self.is_click = True
return RequestService.requests_to_follow(slm_response.get_html_res())
else:
return None
except:
traceback.print_exc()
pass