def GetRandom(self):
""" select a proxy randomly """
logger.log("Proxies.GetRandom()", "infos", "select a random proxy ...")
index = randint(0,len(self.proxies) - 1)
logger.log("Proxies.GetRandom()", "infos", "selected proxy : " + self.proxies[index]["value"])
return self.proxies[index]
def getFirefoxDriver(proxy):
PROXY_HOST = proxy.getIP()
PROXY_PORT = proxy.getPort()
logger.log(
"getFirefoxDriver()", "infos",
"defining the firefox driver with proxy " + PROXY_HOST + ":" +
PROXY_PORT + " ...")
fp = webdriver.FirefoxProfile()
fp.set_preference("network.proxy.type", 1)
fp.set_preference("network.proxy.http", proxy.getIP())
fp.set_preference("network.proxy.http_port", int(PROXY_PORT))
fp.set_preference("network.proxy.https", PROXY_HOST)
fp.set_preference("network.proxy.https_port", int(PROXY_PORT))
fp.set_preference("network.proxy.ssl", PROXY_HOST)
fp.set_preference("network.proxy.ssl_port", int(PROXY_PORT))
fp.set_preference("network.proxy.ftp", PROXY_HOST)
fp.set_preference("network.proxy.ftp_port", int(PROXY_PORT))
fp.set_preference("network.proxy.socks", PROXY_HOST)
fp.set_preference("network.proxy.socks_port", int(PROXY_PORT))
fp.set_preference("http.response.timeout", 20)
fp.set_preference("dom.max_script_run_time", 20)
# fp.set_preference("general.useragent.override","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A")
fp.update_preferences()
logger.log("getFirefoxDriver()", "success", "firefox driver initialised")
driver = webdriver.Firefox(firefox_profile=fp)
return driver
def findElementById(self, root):
""" fill the given fild with the given value """
logger.log("findElementById()", "infos",
"start with criteria [" + self.action.criteria + "] ...")
self.context.element = root.find_element_by_id(self.action.criteria)
logger.log("findElementById()", "success",
"finished with criteria [" + self.action.criteria + "] ...")
return self.context.element
def start(self):
methodname = self.action.name + " " + self.action.result + " " + self.action.method
logger.log("ACTION [" + methodname + "]", "infos", "start ...")
result = self.methods[methodname](self,
self.getRootFromString(
self.action.root))
logger.log("ACTION [" + methodname + "]", "infos", "finished!")
return result
def waitById(self, root):
""" wait the element specified by his identifier """
logger.log("waitById()", "infos",
"start with criteria [" + self.action.criteria + "] ...")
self.context.element = WebDriverWait(root, 10).until(
EC.presence_of_element_located((By.ID, self.action.criteria)))
logger.log("waitById()", "success",
"finished with criteria [" + self.action.criteria + "] ...")
return self.context.element
def findElementsByTagName(self, root):
""" find all elements that matche with the given tag name """
logger.log("findElementsByTagName()", "infos",
"start with criteria [" + self.action.criteria + "] ...")
self.context.elements = root.find_elements_by_tag_name(
self.action.criteria)
logger.log("findElementsByTagName()", "success",
"finished with criteria [" + self.action.criteria + "] ...")
return self.context.elements
def submitById(self, root):
""" submit the given field """
logger.log("submitById()", "infos",
"submitting the field " + self.action.criteria + " ...")
self.context.element = root.find_element_by_id(self.action.criteria)
self.context.element.submit()
logger.log("submitById()", "success",
"The field " + self.action.criteria + " has been submitted")
return self.context.element
def clickElementByTagName(self, root):
""" click on an element specified by its tag name """
logger.log("clickElementByTagName()", "infos",
"start with criteria [" + self.action.criteria + "] ...")
self.context.element = root.find_element_by_tag_name(
self.action.criteria)
self.context.element.click()
logger.log("findElementsByTagName()", "success",
"finished with criteria [" + self.action.criteria + "] ...")
return self.context.element
def getAttributeValuesForEachElements(self, root):
logger.log("getAttributeForEach()", "infos",
"start with criteria [" + self.action.criteria + "] ...")
results = []
for element in root:
elt = element.get_attribute(self.action.criteria)
if elt != None:
results.append(elt)
self.context.results = results
logger.log("getAttributeForEach()", "success",
"finished with criteria [" + self.action.criteria + "] ...")
return self.context.elements
def findForEachElementsByTagName(self, root):
logger.log("findForEachElementsByTagName()", "infos",
"start with criteria [" + self.action.criteria + "] ...")
results = []
for element in root:
elt = element.find_element_by_tag_name(self.action.criteria)
if elt != None:
results.append(elt)
self.context.elements = results
logger.log("findForEachElementsByTagName()", "success",
"finished with criteria [" + self.action.criteria + "] ...")
return self.context.elements
def do(self):
logger.log("action.do()", "infos", "executing the action " + self.name + " ...")
""" Execute the action """
# initialize a new action maker
amaker = ActionMaker(self, self.context)
# launch the action maker
result = amaker.start()
logger.log("action.do()", "infos", "action " + self.name + " finished!")
return result
def fillById(self, root):
""" fill the given fild with the given value """
logger.log(
"fillById()", "infos",
"filling the field " + self.action.criteria + " with the value " +
self.action.input + " ...")
self.context.element = root.find_element_by_id(self.action.criteria)
self.context.element = self.context.element.send_keys(
self.action.input)
logger.log(
"fillById()", "success", "The field " + self.action.criteria +
" has been filled with the value " + self.action.input)
return self.context.element
def fillField(driver, id, value, submit=False):
""" fill the given fild with the given value """
logger.log("fillField()", "infos",
"filling the field " + id + " with the value " + value + " ...")
inputField = driver.find_element_by_id(id)
inputField.send_keys(value)
logger.log("fillField()", "success",
"The field " + id + " has been filled with the value " + value)
if submit:
return inputField.submit()
else:
return None
def run(self):
"""Code à exécuter pendant l'exécution du thread."""
logger.log("main()", "infos", "worker " + self.identifier + " start ...")
# create a directory for the results
file.mkdir("tmp/worker_" + self.identifier)
try:
for _ in range(len(self.workflows.list)):
w = self.workflows.getNextWorkflow()
for _ in range(len(w.actions)):
action = w.getNextAction()
action.do()
logger.log("main()", "success", "worker " + self.identifier + " finished !")
except NavigationFailed as e:
ctx = self.workflows.context
ctx.proxies.setScoreFails(ctx.proxy)
logger.log("main()", "error", "worker " + self.identifier + ": " + e.message)
finally:
try:
ctx.driver.close()
except e:
logger.log("main()", "error", "driver cannot be closed: " + e.message)
def navigateURL(url, driver):
logger.log("navigateURL()", "infos", "navigate to " + url + " ...")
try:
driver.get(url)
logger.log("navigateURL()", "success", url + " opened.")
except:
logger.log("navigateURL()", "error", url + " nothing opened.")
raise ConnectionTimeoutError("Connection timeout error")
return driver
def setScoreFails(self,proxy):
logger.log("Proxies.setScoreFails()", "infos", "increase the fail score of the proxy " + proxy.value)
# add a success to the proxy
for p in self.proxies:
if p["value"] == proxy.value:
logger.log("Proxies.setScoreFails()", "infos", "updating " + proxy.value + " ...")
p["fails"] = p["fails"] + 1
# save the proxies file updated
with open(self.file,"w") as file:
json.dump(self.proxies, file)
file.close()
logger.log("Proxies.setScoreFails()", "success", "fails score of the proxy " + proxy.value + " has been increased")
def navigateByURL(self, root):
""" fill the given fild with the given value """
logger.log("navigateByURL()", "infos",
"navigates to " + self.action.criteria + " ...")
# navigate to the url
try:
root.get(self.action.criteria)
logger.log("navigateByURL()", "success",
"Page " + self.action.criteria + " opened")
except:
logger.log("navigateByURL()", "error",
"Page " + self.action.criteria + " not openened")
raise NavigationFailed("Failed to navigate to " +
self.action.criteria)
return self.context.driver.page_source
