def __fetch_requests(self, task_entry, shell_groups): p = Pattern(task_entry, self.__getCurrentShell(task_entry, shell_groups), self.global_data) timeout = task_entry.get('timeout', 120) urls = p.convertPattern('url') s = requests.session() headers = task_entry.get('headers', []) task_entry['datas'] = [] if not urls: return task_entry for url in urls: self.logger.info("fetching " + url) data = "" if not url: # do not fetch null url continue try: response = s.get(url, timeout=timeout, headers=headers) if 200 != response.status_code: self.logger.error("fetch " + url + " failed with code " + (str)(response.status_code)) data = response.text except: self.logger.error("fetch " + url + " failed in sockets") task_entry['datas'].append(data) return task_entry
def __fetch_requests(self, task_entry): p = Pattern(task_entry, self.__getCurrentShell(task_entry), self.global_data) timeout = task_entry.get('timeout', 120) urls = p.convertPattern('url') s = requests.session() headers = task_entry.get('headers', []) task_entry['datas'] = [] if not urls: return task_entry for url in urls: self.logger.info("fetching " + url) data = "" if not url: # do not fetch null url continue try: response = s.get(url, timeout=timeout, headers=headers) if 200 != response.status_code: self.logger.error("fetch " + url + " failed with code " + (str)(response.status_code)) data = response.text except: self.logger.error("fetch " + url + " failed in sockets") task_entry['datas'].append(data) return task_entry
def __fetch(self, task_entry): p = Pattern(task_entry, self.__getCurrentShell(task_entry), self.config_data) urls = p.convertPattern('url') s = requests.session() task_entry['datas'] = [] for url in urls: self.logger.info("fetching " + url) if "" == url: # do not fetch null url continue response = s.get(url) if 200 != response.status_code: self.logger.error("fetch " + url + " failed with code" + response.status_code) data = response.text task_entry['datas'].append(data) return task_entry
def __fetch_webkit(self, task_entry): p = Pattern(task_entry, self.__getCurrentShell(task_entry), self.global_data) import cwebbrowser task_entry['datas'] = [] urls = p.convertPattern('url') timeout = task_entry.get('timeout', 120) delay = task_entry.get('delay', 0) for url in urls: self.logger.info("fetching " + url) data = "" if not url: # do not fetch null url continue browser = cwebbrowser.CWebBrowser() browser.setHeaders(task_entry.get('headers', [])) #browser.show(); try: browser.load(url=url, load_timeout=timeout, delay=delay) except cwebbrowser.Timeout: self.logger.error("fetch " + url + " timeout ") except Exception, exception: self.logger.error("fetch " + url + " error ") print "Exception message:", exception else: html = browser.html() if html: html = html.encode('utf-8') data = html else: self.logger.error("fetch " + url + " failed with no response") task_entry['datas'].append(data) browser.close()