def GetRandom(self): """ select a proxy randomly """ logger.log("Proxies.GetRandom()", "infos", "select a random proxy ...") index = randint(0,len(self.proxies) - 1) logger.log("Proxies.GetRandom()", "infos", "selected proxy : " + self.proxies[index]["value"]) return self.proxies[index]
def getFirefoxDriver(proxy): PROXY_HOST = proxy.getIP() PROXY_PORT = proxy.getPort() logger.log( "getFirefoxDriver()", "infos", "defining the firefox driver with proxy " + PROXY_HOST + ":" + PROXY_PORT + " ...") fp = webdriver.FirefoxProfile() fp.set_preference("network.proxy.type", 1) fp.set_preference("network.proxy.http", proxy.getIP()) fp.set_preference("network.proxy.http_port", int(PROXY_PORT)) fp.set_preference("network.proxy.https", PROXY_HOST) fp.set_preference("network.proxy.https_port", int(PROXY_PORT)) fp.set_preference("network.proxy.ssl", PROXY_HOST) fp.set_preference("network.proxy.ssl_port", int(PROXY_PORT)) fp.set_preference("network.proxy.ftp", PROXY_HOST) fp.set_preference("network.proxy.ftp_port", int(PROXY_PORT)) fp.set_preference("network.proxy.socks", PROXY_HOST) fp.set_preference("network.proxy.socks_port", int(PROXY_PORT)) fp.set_preference("http.response.timeout", 20) fp.set_preference("dom.max_script_run_time", 20) # fp.set_preference("general.useragent.override","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A") fp.update_preferences() logger.log("getFirefoxDriver()", "success", "firefox driver initialised") driver = webdriver.Firefox(firefox_profile=fp) return driver
def findElementById(self, root): """ fill the given fild with the given value """ logger.log("findElementById()", "infos", "start with criteria [" + self.action.criteria + "] ...") self.context.element = root.find_element_by_id(self.action.criteria) logger.log("findElementById()", "success", "finished with criteria [" + self.action.criteria + "] ...") return self.context.element
def start(self): methodname = self.action.name + " " + self.action.result + " " + self.action.method logger.log("ACTION [" + methodname + "]", "infos", "start ...") result = self.methods[methodname](self, self.getRootFromString( self.action.root)) logger.log("ACTION [" + methodname + "]", "infos", "finished!") return result
def waitById(self, root): """ wait the element specified by his identifier """ logger.log("waitById()", "infos", "start with criteria [" + self.action.criteria + "] ...") self.context.element = WebDriverWait(root, 10).until( EC.presence_of_element_located((By.ID, self.action.criteria))) logger.log("waitById()", "success", "finished with criteria [" + self.action.criteria + "] ...") return self.context.element
def findElementsByTagName(self, root): """ find all elements that matche with the given tag name """ logger.log("findElementsByTagName()", "infos", "start with criteria [" + self.action.criteria + "] ...") self.context.elements = root.find_elements_by_tag_name( self.action.criteria) logger.log("findElementsByTagName()", "success", "finished with criteria [" + self.action.criteria + "] ...") return self.context.elements
def submitById(self, root): """ submit the given field """ logger.log("submitById()", "infos", "submitting the field " + self.action.criteria + " ...") self.context.element = root.find_element_by_id(self.action.criteria) self.context.element.submit() logger.log("submitById()", "success", "The field " + self.action.criteria + " has been submitted") return self.context.element
def clickElementByTagName(self, root): """ click on an element specified by its tag name """ logger.log("clickElementByTagName()", "infos", "start with criteria [" + self.action.criteria + "] ...") self.context.element = root.find_element_by_tag_name( self.action.criteria) self.context.element.click() logger.log("findElementsByTagName()", "success", "finished with criteria [" + self.action.criteria + "] ...") return self.context.element
def getAttributeValuesForEachElements(self, root): logger.log("getAttributeForEach()", "infos", "start with criteria [" + self.action.criteria + "] ...") results = [] for element in root: elt = element.get_attribute(self.action.criteria) if elt != None: results.append(elt) self.context.results = results logger.log("getAttributeForEach()", "success", "finished with criteria [" + self.action.criteria + "] ...") return self.context.elements
def findForEachElementsByTagName(self, root): logger.log("findForEachElementsByTagName()", "infos", "start with criteria [" + self.action.criteria + "] ...") results = [] for element in root: elt = element.find_element_by_tag_name(self.action.criteria) if elt != None: results.append(elt) self.context.elements = results logger.log("findForEachElementsByTagName()", "success", "finished with criteria [" + self.action.criteria + "] ...") return self.context.elements
def do(self): logger.log("action.do()", "infos", "executing the action " + self.name + " ...") """ Execute the action """ # initialize a new action maker amaker = ActionMaker(self, self.context) # launch the action maker result = amaker.start() logger.log("action.do()", "infos", "action " + self.name + " finished!") return result
def fillById(self, root): """ fill the given fild with the given value """ logger.log( "fillById()", "infos", "filling the field " + self.action.criteria + " with the value " + self.action.input + " ...") self.context.element = root.find_element_by_id(self.action.criteria) self.context.element = self.context.element.send_keys( self.action.input) logger.log( "fillById()", "success", "The field " + self.action.criteria + " has been filled with the value " + self.action.input) return self.context.element
def fillField(driver, id, value, submit=False): """ fill the given fild with the given value """ logger.log("fillField()", "infos", "filling the field " + id + " with the value " + value + " ...") inputField = driver.find_element_by_id(id) inputField.send_keys(value) logger.log("fillField()", "success", "The field " + id + " has been filled with the value " + value) if submit: return inputField.submit() else: return None
def run(self): """Code à exécuter pendant l'exécution du thread.""" logger.log("main()", "infos", "worker " + self.identifier + " start ...") # create a directory for the results file.mkdir("tmp/worker_" + self.identifier) try: for _ in range(len(self.workflows.list)): w = self.workflows.getNextWorkflow() for _ in range(len(w.actions)): action = w.getNextAction() action.do() logger.log("main()", "success", "worker " + self.identifier + " finished !") except NavigationFailed as e: ctx = self.workflows.context ctx.proxies.setScoreFails(ctx.proxy) logger.log("main()", "error", "worker " + self.identifier + ": " + e.message) finally: try: ctx.driver.close() except e: logger.log("main()", "error", "driver cannot be closed: " + e.message)
def navigateURL(url, driver): logger.log("navigateURL()", "infos", "navigate to " + url + " ...") try: driver.get(url) logger.log("navigateURL()", "success", url + " opened.") except: logger.log("navigateURL()", "error", url + " nothing opened.") raise ConnectionTimeoutError("Connection timeout error") return driver
def setScoreFails(self,proxy): logger.log("Proxies.setScoreFails()", "infos", "increase the fail score of the proxy " + proxy.value) # add a success to the proxy for p in self.proxies: if p["value"] == proxy.value: logger.log("Proxies.setScoreFails()", "infos", "updating " + proxy.value + " ...") p["fails"] = p["fails"] + 1 # save the proxies file updated with open(self.file,"w") as file: json.dump(self.proxies, file) file.close() logger.log("Proxies.setScoreFails()", "success", "fails score of the proxy " + proxy.value + " has been increased")
def navigateByURL(self, root): """ fill the given fild with the given value """ logger.log("navigateByURL()", "infos", "navigates to " + self.action.criteria + " ...") # navigate to the url try: root.get(self.action.criteria) logger.log("navigateByURL()", "success", "Page " + self.action.criteria + " opened") except: logger.log("navigateByURL()", "error", "Page " + self.action.criteria + " not openened") raise NavigationFailed("Failed to navigate to " + self.action.criteria) return self.context.driver.page_source