def urllib3_test(): from urllib3 import PoolManager import time, random url = ['http://sist.swjtu.edu.cn:8080/MicroElec/feng/index.asp', 'http://sist.swjtu.edu.cn:8080/MicroElec/feng/contact/contact.asp,' 'http://sist.swjtu.edu.cn:8080/MicroElec/feng/Templates/%E5%A4%8D%E4%BB%B6%20CSS/', 'http://sist.swjtu.edu.cn:8080/MicroElec/index.asp', 'http://sist.swjtu.edu.cn:8080/My%20Documents/Documents%20and%20Settingsjiangtigang%E6%A1%8C%E9%9D%A2ImcGroupHomepageNewImcWebYuan%20Ding"%20l'] errors = 0 oks = 0 p = PoolManager(20) for a in url: try: p.request(method="GET", url=a) s = random.randint(0,10) time.sleep(s) print s oks += 1 except: errors += 1 continue print ".", print print "Errors: %s | Oks: %s." % (str(errors), str(oks))
class APISent(object): """Sentiment features using API tools. Interacts with web and therefore needs urllib3. Might be _very_ slow, use with caution and prefrably store features. Parameters ---------- mode : string, optional, default 'deep' Can be either 'deep' for Twitter-based neural sentiment (py2, boots local server instance), or 'nltk' for the text-processing.com API. Examples -------- >>> sent = APISent() >>> sent.transform("you're gonna have a bad time") ... 0.030120761495050809 >>> sent = APISent(mode='nltk') >>> sent.transform("you're gonna have a bad time") ... Notes ----- Implemented by: Chris Emmery Deep sentiment: https://github.com/xiaohan2012/twitter-sent-dnn NLTK API: http://text-processing.com """ def __init__(self, mode='deep'): """Load poolmanager and set API location.""" from urllib3 import PoolManager self.name = 'apisent' self.mode = mode self.pool = PoolManager() def __str__(self): """String representation for APISent.""" return ''' feature: {0} mode: {1} '''.format(self.name, self.mode) def transform(self, raw, _): """Return a dictionary of feature values.""" if self.mode == 'deep': jsf = json.dumps({'text': raw}) header = {'content-type': 'application/json'} request = "http://localhost:6667/api" r = self.pool.request('POST', request, headers=header, body=jsf) out = {'deepsent': float(r.data.decode('utf-8'))} elif self.mode == 'nltk': qf = urlencode({'text': raw}) request = "http://text-processing.com/api/sentiment/" r = self.pool.request('POST', request, body=qf) try: out = json.loads(r.data.decode('utf-8'))["probability"] except ValueError: exit("SentAPI threw unexpected response, " + "probably reached rate limit.") return out
class gamefetcher: def __init__(self): from urllib3 import PoolManager self.pool = PoolManager() def getfreegames(self): from json import loads response = self.pool.request('GET', config.fetch_games_url, preload_content=False) response.release_conn() return response.data def filtergames(self, text, games_owned): from bs4 import BeautifulSoup from re import compile apps = [] soup = BeautifulSoup(text, 'html.parser') links = soup.find_all("a", href=True) regexraw = r"^https:\/\/store.steampowered.com\/app\/[0-9]{1,}" regex = compile(regexraw) for link in links: result = regex.match(link["href"]) if (result != None): app_id = result.group(0).replace( "https://store.steampowered.com/app/", "") if app_id not in games_owned: apps.append(app_id) return apps
def download_sig(opts, sig, version=None): """Download signature from hostname""" code = None downloaded = False useagent = 'ClamAV/0.101.1 (OS: linux-gnu, ARCH: x86_64, CPU: x86_64)' manager = PoolManager(headers=make_headers(user_agent=useagent), cert_reqs='CERT_REQUIRED', ca_certs=certifi.where(), timeout=Timeout(connect=10.0, read=60.0)) if version: path = '/%s.cvd' % sig filename = os.path.join(opts.workdir, '%s.cvd' % sig) else: path = '/%s.cdiff' % sig filename = os.path.join(opts.workdir, '%s.cdiff' % sig) try: req = manager.request('GET', 'http://%s%s' % (opts.hostname, path)) except BaseException as msg: error("Request error: %s" % msg) data = req.data code = req.status if req.status == 200: with open(filename, 'wb') as handle: handle.write(data) downloaded = os.path.exists(filename) return downloaded, code
def __downloadPackage(self, url: str, filew: str, http: urllib3.PoolManager) -> None: """ Download package installation file Parameters ---------- url : str Url of installation file filew : str Name of installation file http : urllib3.PoolManager Urllib PoolManager web connection >>> __downloadPackage("https://domain.com/package.whl", "package.whl", urllib3.PoolManager()) None """ if os.path.exists(filew): return files = http.request("GET", url) with open(filew, "wb") as f: f.write(files.data)
def get_resultados(name='', price_max=''): http = PoolManager() url_api = 'http://es.wallapop.com/rest/items?minPrice=&maxPrice={price_max}&dist=&order=creationDate-des&lat=41.398077&lng=2.170432&kws={kws}'.format( kws=urllib.parse.quote(name, safe=''), price_max=price_max) results = http.request('GET', url_api) results = json.loads(results.data.decode('utf-8')) return results['items']
def filerecv(servloca, attrdata): """ Receives file from the provided Veritas server :param servloca: :param attrdata: :return: """ try: httpobjc = PoolManager() rgetfild = {"tokniden": attrdata} rqstobjc = httpobjc.request("GET", servloca + "filerecv", fields=rgetfild) respdata = json.loads(rqstobjc.data.decode()) if respdata["retnmesg"] == "FAIL": click.echo(click.style("Transfer failed!", fg="red")) elif respdata["retnmesg"] == "DONE": filename = respdata["filename"] b64etext = respdata["contents"] contents = b64decode(b64etext.encode()).decode() with open(filename, "w") as fileobjc: fileobjc.write(contents) click.echo(click.style("Transfer successful!", fg="green")) except Exception as expt: click.echo(" * " + click.style("Error occurred : " + str(expt), fg="red"))
def get_html(self, url): """ Функция принимает url страницы и собирает его html код, возвращая в get_all_links :param url: :return: html-код """ http = PoolManager(1) # берётся один случайный прокси proxy = self.get_proxy() # Если страницу с таким url уже парсили, то не повторяем if url in self.parsed_pages: return '' try: # запрашиваем страницу, получаем объект HTTPResponse resp = http.request('GET', url, proxy[0], proxy[1]) # обработка ошибки при слишком большом количестве перенаправлений except urllib3.exceptions.MaxRetryError: print(f'Достигнут лимит перенаправлений со страницы: {url}') return MAX_RETRY page = resp.data.decode('utf-8', 'ignore') # приводим response в читаемый вид soup = BeautifulSoup(page, "html.parser") # Регистрируем, что эту страницу обошли self.parsed_pages.add(url) # Возвращаем html код страницы return soup
def _load_internet(self): """ Description ----------- boot sequence 3. check internet connection Notes ----- use urllib3 to set connection timeout manually and reduce check time Return ------ result 0 success -1 fail """ self.net_connected = False try: from urllib3 import PoolManager, Timeout , Retry http = PoolManager( timeout=Timeout(connect=1.0, read=2.0), retries=Retry(0, redirect=0) ) response = http.request("HEAD", "https://status.cloud.google.com/") if response.status == 200: # if internet ok. self.log.info("pino_boot_loader.py: internet not connected!") self.net_connected = True except Exception as E: self.log.error("pino_boot_loader.py: _load_internet(), " + repr(E)) return -1 else: return 0
def show_incomes(): """ docstring here """ access_token = session.get('access_token') if access_token is None: return redirect(url_for('login')) access_token = access_token[0] _http = PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where()) headers = {'Authorization': 'OAuth '+access_token} _request = _http.request( 'GET', 'https://www.googleapis.com/oauth2/v1/userinfo', None, headers) if _request.status == 401: session.pop('access_token', None) return redirect(url_for('login')) userinfo = json.loads(_request.data.decode('utf-8', 'strict')) if not userinfo['email'] in ['*****@*****.**', '*****@*****.**']: session.pop('access_token', None) return redirect(url_for('login')) else: session['email'] = userinfo['email'] session['logged_in'] = True return render_template( 'show_entries.html', entries=list( map(lambda x: x.to_dict(), incomes_get(id=0))), currencies=currency_get(id=0), periods=intervals_get(id=0), transactions=list( map(lambda x: x.to_dict(), transactions_get(id=0))))
def export_template(t_id, output='string', file_path=None): """ Exports a template as a string of xml :param t_id: ID of the template to export :param output: string of return type of template, 'str' or 'file' :param file_path: if file output type, the path and filename to write to :return: basestring of the xml template or file path written """ # TemplatesAPI.export template is broken in swagger definition of NiFi1.2 # return TemplateDTO is replaced by return string in a later version valid_output_types = ['file', 'string'] if output not in valid_output_types: raise ValueError("Output type {0} not valid for ({1})".format( output, valid_output_types)) con = PoolManager() url = nifi_config.host + '/templates/' + t_id + '/download' response = con.request('GET', url, preload_content=False) template_xml = etree.fromstring(response.data) if output == 'string': return etree.tostring(template_xml, encoding='utf8', method='xml') if output == 'file': assert access(dirname(file_path), W_OK), \ "File_path {0} is inaccessible or not writable".format(file_path) xml_tree = etree.ElementTree(template_xml) xml_tree.write(file_path) return file_path
class Do: def __init__(self, url, **kwargs): self.url = url self.http = PoolManager() self.connect_timeout = kwargs['connect_timeout'] if kwargs['connect_timeout'] else 2.0 self.read_timeout = kwargs['read_timeout'] if kwargs['read_timeout'] else 2.0 self.available_status = kwargs['available_status'] if kwargs['available_status'] else '200,' self.alert_mail = kwargs['alert_mail'] self._get() def _get(self): r = None try: start = time.perf_counter() r = self.http.request('GET', self.url, timeout=Timeout(connect=self.connect_timeout, read=self.read_timeout)) end = time.perf_counter() except Exception as ex: send_mail(self.alert_mail, 'Request {} failed'.format(self.url), str(ex)) available_status = self.available_status.split(',') if not r: return None if not hasattr(r, 'status'): return None if str(r.status) in available_status: logger.info(self.url, end-start) else: send_mail(self.alert_mail, '{} :{}'.format(self.url, r.status), '{}\r\n{}\r\n{}'.format(r.status, r.data, end-start))
def download_url(urlpath, output_dir=".", binary=False): http = PoolManager() req = http.request("GET", urlpath) if req.status != 200: raise Exception("Could not get file from " + urlpath) parsed = urlparse(urlpath) filename = os.path.basename(parsed.path) writemod = "wb" if binary else "w" contents = req.data if output_dir != ".": if not os.path.exists(output_dir): log.error("{0} does not exist".format(output_dir)) log.error("Writing file to {0}".format(os.getcwd())) else: filename = "/".join([output_dir, filename]) with open(filename, writemod) as downloaded: try: downloaded.write(contents) except TypeError: with open(filename, "wb") as downloaded: downloaded.write(contents) if not os.path.exists(filename): raise Exception("Could not write to {}".format(filename)) return filename
async def parseqr(qr_e): """ .decode komutu cevap verilen fotoğraftan QR kodu / Barkod içeriğini alır """ downloaded_file_name = await qr_e.client.download_media( await qr_e.get_reply_message()) # QR kodunu çözmek için resmi ZXing web sayfasını ayrıştır files = {'f': open(downloaded_file_name, 'rb').read()} t_response = None try: http = PoolManager() t_response = http.request('POST', "https://zxing.org/w/decode", fields=files) t_response = t_response.data http.clear() except: pass os.remove(downloaded_file_name) if not t_response: await qr_e.edit("decode başarısız oldu.") return soup = BeautifulSoup(t_response, "html.parser") qr_contents = soup.find_all("pre")[0].text await qr_e.edit(qr_contents)
def pool_request(self, pool: urllib3.PoolManager, data: dict = None): """ 池化http请求方式发起远程调用 :param pool: urllib3连接池实例 :param data:请求参数 :return: """ try: # 轮询方式访问实例 instance = self.__instances[self.__instance_index] # 检查服务注册信息(防止调用自身) if self.__interface_app.service_name() == self.__service_name: print('警告::(有风险)使用远程调用单元调用自身服务(', self.__service_name, ',', self.__interface, ')') try: if instance['host'] is socket.gethostbyname(socket.getfqdn(socket.gethostname())) \ and \ instance['port'] is self.__interface_app.port(): # 检查请求路径 self.__instances.pop() else: self.__instance_index = self.__instance_index + 1 instance = self.__instances[self.__instance_index] except Exception as e: print(e) url = str('http://' + instance['host'] + ':' + instance['port'] + self.__interface) res = pool.request(method=self.__req_obj.get_method(), url=url, fields=data) self.__id = instance['id'] self.__instance_index = (self.__instance_index + 1) if self.__instance_index < len(self.__instances) - 2 else 0 url = str('http://' + instance['host'] + ':' + instance['port'] + self.__interface) setattr(self.__req_obj, 'full_url', url) return eval(res.data.decode('utf8')) if res.status == 200 else {'event': res.status, 'msg': '服务调用异常'} except: return {'event': 500, 'msg': '服务调用异常'}
class __ProviderManager(QtCore.QObject): sendError = QtCore.pyqtSignal(str) def __init__(self): QtCore.QObject.__init__(self) self.providers = {} self.poolManager = PoolManager(timeout=Timeout(10), headers={'Accept-Encoding': 'gzip,deflate'}) self.engine = Engine() def loadProviderFromFile(self,path): try: providerFile = open(path,mode="r") provider = json.loads(providerFile.read()) providerFile.close() self.providers[provider["name"]] = provider except Exception as e: self.sendError.emit("cannot load provider at '"+path+"' <br/><b>Reason:</b> "+str(e)) def loadProviderFromUrl(self,url): try: req = self.poolManager.request("GET", url) provider = json.loads(req.data.decode('utf-8')) self.providers[provider["name"]] = provider del req except Exception as e: self.sendError.emit("cannot load provider at '"+url+"' <br/><b>Reason:</b> "+str(e)) def queryProvider(self,text,category,pages,providerName,perPageCallback=None,whenDoneCallback=None):#Exceptions here should be managed by the caller return self.engine.makeQuery(self.providers[providerName], text, category, pages,perPageCallback,whenDoneCallback) def reset(self): if self.providers: self.providers.clear()
def __request_api(self, options={}) -> dict: """Sent request to diferent data point Args: options (dict, optional): options to sent req. Defaults to {}. Returns: dict: response of api """ url = options.get('url') headers = options.get('headers') method = options.get('method') # Set default timeout to 5 second and retries to 3 using urllib3 PoolManager retries = Retry(connect=3, read=3, redirect=3) timeout = Timeout(connect=5.0, read=5.0) http = PoolManager(retries=retries, timeout=timeout) # Sent req to api especific try: response = http.request(url=url, headers=headers, method=method) except Exception as err: return {'Error': err} return json.loads(response.data.decode('utf-8'))
def getfreegames(s): url = s from urllib3 import PoolManager, exceptions from bs4 import BeautifulSoup import certifi try: https = PoolManager(ca_certs=certifi.where()) response = https.request('GET', url, headers=config.headers).data.decode('utf-8') logwrite('Got url: {}'.format(url)) except exceptions.ConnectionError: pp.pprint('Cant connect to {}'.format(url)) exit() soup = BeautifulSoup(response, 'html.parser') filterapps = soup.findAll('td') text = '{}'.format(filterapps) soup = BeautifulSoup(text, 'html.parser') from re import compile appidfinder = compile('^[0-9]{2,}$') link = compile('^/') for _ in soup.findAll('a', attrs={'href': link}): appid = returnappid(_.get('href')) appid = appidfinder.match(appid) if appid: appids.append(appid.string) else: break
async def covid(event): try: url = 'https://quiec.tech/corona.php' http = PoolManager() request = http.request('GET', url) result = jsloads(request.data.decode('utf-8')) http.clear() except: await event.edit("`Bir hata oluştu.`") return sonuclar = ("** Koronavirüs Verileri **\n" + "\n**Dünya geneli**\n" + f"**🌎 Vaka:** `{result['tum']}`\n" + f"**🌎 Ölüm:** `{result['tumolum']}`\n" + f"**🌎 İyileşen:** `{result['tumk']}`\n" + "\n**Türkiye**\n" + f"**🇹🇷 Vaka (toplam):** `{result['trtum']}`\n" + f"**🇹🇷 Vaka (bugün):** `{result['trbtum']}`\n" + f"**🇹🇷 Vaka (aktif):** `{result['tra']}`\n" + f"**🇹🇷 Ölüm (toplam):** `{result['trolum']}`\n" + f"**🇹🇷 Ölüm (bugün):** `{result['trbolum']}`\n" + f"**🇹🇷 İyileşen:** `{result['trk']}`") await event.edit(sonuclar)
def filesend(servloca, attrdata): """ Sends file to the provided Veritas server :param servloca: :param attrdata: :return: """ try: with open(attrdata, "r") as fileobjc: contents = fileobjc.read() b64etext = b64encode(contents.encode()).decode() rgetfild = {"filename": attrdata, "contents": b64etext} httpobjc = PoolManager() rqstobjc = httpobjc.request("GET", servloca + "filesend", fields=rgetfild) respdata = json.loads(rqstobjc.data.decode()) if respdata["retnmesg"] == "FAIL": click.echo(click.style("Transfer failed!", fg="red")) elif respdata["retnmesg"] == "DONE": click.echo( click.style("Store this token safely -> " + respdata["tokniden"], fg="green")) except Exception as expt: click.echo(" * " + click.style("Error occurred : " + str(expt), fg="red"))
def getPharmaciesData(): pharmaciesUrl = r'https://raw.githubusercontent.com/kiang/pharmacies/master/json/points.json' http = PoolManager() r = http.request('GET', pharmaciesUrl) pharmaciesData = json.loads(r.data)['features'] allData = [] for phd in pharmaciesData: data = {} geo = phd['geometry']['coordinates'] name = phd['properties']['name'] address = phd['properties']['address'] mask_adult = phd['properties']['mask_adult'] mask_child = phd['properties']['mask_child'] available = phd['properties']['available'] note = phd['properties']['note'] data = { 'name': name, 'address': address, 'mask_adult': mask_adult, 'mask_child': mask_child, 'available': available, 'note': note, 'geometry': geo } allData.append(data) return allData
def get_entries_for_menu(menu): url = menu.url parent_element = menu.menu_element meal_element = menu.meal_element price_element = menu.price_element meal_selector = f"{parent_element} > * {meal_element}" price_selector = f"{parent_element} > * {price_element}" conn_pool = PoolManager() downloaded_page = conn_pool.request('GET', menu.url) print(downloaded_page._body) file_html = downloaded_page._body """ file_html = f.readlines() file_html = "".join(file_html) """ soup = BeautifulSoup(file_html, 'html.parser') "div#day-selection-tab-1 > * div.single-food > strong" meals = soup.select(meal_selector) prices = soup.select(price_selector) menu_entries = [] for meal, price in zip(meals, prices): menu_entries.append((meal, price)) return menu_entries
async def parseqr(qr_e): """ .decode """ downloaded_file_name = await qr_e.client.download_media( await qr_e.get_reply_message()) # QR files = {'f': open(downloaded_file_name, 'rb').read()} t_response = None try: http = PoolManager() t_response = http.request( 'POST', "https://zxing.org/w/decode", fields=files) t_response = t_response.data http.clear() except: pass os.remove(downloaded_file_name) if not t_response: await qr_e.edit(LANG['ERROR']) return soup = BeautifulSoup(t_response, "html.parser") qr_contents = soup.find_all("pre")[0].text await qr_e.edit(qr_contents)
def getAmqpStats(helper): from urllib3 import PoolManager, util try: username = "******" password = "******" headers = util.make_headers(basic_auth = username + ":" + password) http = PoolManager() r = http.request("GET", "upsilon:15672/api/channels", headers = headers) channels = json.loads(r.data) tbl = PrettyTable(["Connection", "username", "Unacked", "Publish"]) if helper.args.debug: print json.dumps(channels, indent = 4); for conn in channels: tbl.add_row([conn['name'], conn['user'], conn['messages_unacknowledged'], 0]) print tbl except Exception as e: print str(e)
def check_stock_proxy_manager(url, proxy=None, count=0): if proxy is None: manager = PoolManager(timeout=5, cert_reqs='CERT_REQUIRED', ca_certs=certifi.where()) else: proxy_url = "%s://%s:%s" % (proxy[0], proxy[1], proxy[2]) manager = ProxyManager(proxy_url, timeout=5, cert_reqs='CERT_REQUIRED', ca_certs=certifi.where()) headers = util.make_headers(accept_encoding='gzip, deflate', keep_alive=True, user_agent="Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:47.0) Gecko/20100101 Firefox/47.0") headers['Accept-Language'] = "en-US,en;q=0.5" headers['Connection'] = 'keep-alive' headers['Accept'] = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" try: response = manager.request('GET', url, preload_content=False, headers=headers) content = json.loads(response.data) print("%s - Connect Success!" % count) return content['hasStock'] except Exception as ex: print("%s - Connect Error!" % count) return False
def downloadPDF(self, filename='tdnet.zip', limit=3): try: df = self.df.head(limit) except: df = self.df df = df[df['datetime'] > self.date_min] with ZipFile(filename, 'w', compression=ZIP_DEFLATED) as new_zip: for pdf, code, date, title in zip(df['pdf'], df['code'], df['date'], df['title']): url = pdf filename = str(code) + '_' + str(date)[:10].replace( '-', '') + '_' + title + '.pdf' #download request_methods = PoolManager() response = request_methods.request('GET', url) f = open(filename, 'wb') f.write(response.data) f.close() new_zip.write(filename) os.remove(filename) # 時間調整 sleep(3)
def thingspeakgetfloat(chan, field, trace): """Retrieve a floating point value from a ThingSpeak field""" try: if chan == TS_BASEMENT_CHAN: key = TS_BASEMENT_API_READKEY nresults = "2" chanstr = "basement" else: wg_error_print("thingspeakgetfloat", "Invalid channel arg: " + str(chan)) return THINGSPEAK_FLOAT_ERROR pman = PoolManager() # Get the requested field from the specified ThingSpeak channel retstruct = pman.request( 'GET', 'http://api.thingspeak.com/channels/' + chan + '/fields/' + field + '.json?api_key=' + key + '&results=' + nresults) decodestruct = json.loads(retstruct.data.decode('utf-8')) retval = decodestruct['feeds'][0]['field' + field] wg_trace_print( "field " + field + " of " + chanstr + " channel = " + retval, trace) except: wg_error_print( "thingspeakgetfloat", "Exception getting field " + field + " from " + chanstr + " channel") return THINGSPEAK_FLOAT_ERROR return float(retval)
class Downloader: def __init__(self): self.pool = PoolManager() self.htot = html2text.HTML2Text() self.htot.body_width = 0 self.htot.ignore_links = True def _get_page(self, url): return self.pool.request('GET', url).data.decode() def _tag_visible(self, elem): if elem.parent.name in [ 'style', 'script', 'head', 'title', 'meta', '[document]', 'article', 'img', 'aside', 'iframe', 'footer' ]: return 0 if isinstance(elem, element.Comment): return 0 if elem == "\n" or elem.strip() == "": return 0 return 1 def _apply_formatting(self, elem): # formatting elem = re.sub(" +", " ", elem) return elem def _tag_to_text(self, tag): return self.htot.handle(str(tag)) def _text_to_html(self, body): soup = BeautifulSoup(body, 'html.parser') texts = soup.findAll(text=True) visible_texts = list(filter(self._tag_visible, texts)) return visible_texts
def _url2dictlist(self, mined_url: str) -> List[Dict]: """ Generates a json file with all the metadata. Parameters ---------- mined_url: str URL of the GH Archive json.gz file. Returns ------- List(Dict): Returns a list of the JSON dictionaries corresponding to the mined data. """ https = PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where()) dict_list = [] try: response = https.request('GET', mined_url) compressed_json = BytesIO(response.data) with gzip.GzipFile(fileobj=compressed_json) as json_bytes: json_str = json_bytes.read().decode('utf-8') for json_value in json_str.split('\n'): if self._is_valid_json(json_value): data = json.loads(json_value) if self.filter_by_event(data, self.event_set): dict_list.append(data) except OSError: pass return dict_list
def get_url(url, encoding=''): if encoding == '': encoding = GBK pool = PoolManager() res = pool.request('get', url) res_dict = res.data.decode(encoding) return res_dict
def check_vul(url): """ Test if a GET to a URL is successful :param url: The URL to test :return: A dict with the exploit type as the keys, and the HTTP status code as the value """ if gl_args.mode == 'auto-scan' or gl_args.mode == 'file-scan': timeout = Timeout(connect=1.0, read=3.0) pool = PoolManager(timeout=timeout, retries=1, cert_reqs='CERT_NONE') else: timeout = Timeout(connect=3.0, read=6.0) pool = PoolManager(timeout=timeout, cert_reqs='CERT_NONE') url_check = parse_url(url) if '443' in str(url_check.port) and url_check.scheme != 'https': url = "https://"+str(url_check.host)+":"+str(url_check.port) print(GREEN + "\n ** Checking Host: %s **\n" % url) headers = {"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Connection": "keep-alive", "User-Agent": user_agents[randint(0, len(user_agents) - 1)]} paths = {"jmx-console": "/jmx-console/HtmlAdaptor?action=inspectMBean&name=jboss.system:type=ServerInfo", "web-console" : "/web-console/ServerInfo.jsp", "JMXInvokerServlet": "/invoker/JMXInvokerServlet", "admin-console" : "/admin-console/"} for i in paths.keys(): if gl_interrupted: break try: print(GREEN + " * Checking %s: \t" % i + ENDC), r = pool.request('HEAD', url +str(paths[i]), redirect=False, headers=headers) paths[i] = r.status # check if it's false positive if len(r.getheaders()) == 0: print(RED + "[ ERROR ]\n * The server %s is not an HTTP server.\n" % url + ENDC) paths = {"jmx-console": 505, "web-console": 505, "JMXInvokerServlet": 505, "admin-console": 505} break if paths[i] in (301, 302, 303, 307, 308): url_redirect = r.get_redirect_location() print(GREEN + "[ REDIRECT ]\n * The server sent a redirect to: %s\n" % url_redirect) elif paths[i] == 200 or paths[i] == 500: if i == "admin-console": print(RED + "[ EXPOSED ]" + ENDC) else: print(RED + "[ VULNERABLE ]" + ENDC) else: print(GREEN + "[ OK ]") except: print(RED + "\n * An error occurred while connecting to the host %s\n" % url + ENDC) paths[i] = 505 return paths
def getwylie(tibtxt): http = PoolManager() url = WYCONVURL + urlparse.quote(tibtxt) req = http.request('GET', url) if req.status == 200: return req.data.decode('utf-8') return False
def gettib(wytxt): http = PoolManager() url = TIBCONVURL + wytxt.replace(' ', '%20') req = http.request('GET', url) if req.status == 200: return req.data.decode('utf-8') return False
def fetch_images_with_urls(word_net_id_list): ''' Download the images with url and store them on the local folder. :param word_net_id_list: list of wordnet ids :return: none ''' image_db_dir = os.path.dirname(os.path.realpath(__file__)) db_conn = sqlite3.connect(os.path.join(str(image_db_dir), "ImageNet.db")) c = db_conn.cursor() http_conn = PoolManager() for word_net_id in word_net_id_list: # Create a list of image ids that will be fetched image_id_list = [] image_file_dir = os.path.dirname(os.path.realpath(__file__)) image_id_file_path = os.path.join(str(image_file_dir), 'ImageID.txt') with open(image_id_file_path, 'r', encoding='utf8') as out_file: for line in out_file: if line.startswith(word_net_id): image_id_list = line.split("\t")[1].strip().split(',') break # Check how many images are downloaded for the target noun object file_path = os.path.dirname(os.path.realpath(__file__)) image_folder_path = os.path.join(file_path, 'Images', word_net_id) if not os.path.exists(image_folder_path): os.makedirs(image_folder_path) url_count = len(os.listdir(image_folder_path)) # Randomly download the images for the target noun object random.Random(time.time()).shuffle(image_id_list) start_time = time.time() for image_id in image_id_list: if url_count < 10 and image_id.startswith(word_net_id): # First, find the url of the image from the database url = c.execute( 'SELECT URL from ImageNetURLs WHERE ImageID=(?) ORDER BY RANDOM() LIMIT 10', (image_id, )).fetchone() if url is not None: if check_url(url[0]) is not False: # Second, fetch the image timeout = Timeout(connect=5.0, read=10.0) response = http_conn.request('GET', url[0], preload_content=False, timeout=timeout) file_name = os.path.normpath(image_folder_path + "/" + image_id + ".png") # Last, save the image on the local directory out_file = open(file_name, 'wb') shutil.copyfileobj(response, out_file) url_count = url_count + 1 # print("Image directory: " + image_folder_path) # print("--- %s seconds (Get Images) ---" % (time.time() - start_time)) c.close()
def check_url(url): ''' Verify the url before fetching the image. :param url: url string :return: TRUE or FALSE ''' try: timeout = Timeout(connect=1.0, read=1.0) conn = PoolManager() response = conn.request('HEAD', url, timeout=timeout) status = int(response.status) is_url_valid = False # Check the HTTP response status code and whether the url redirects to another page if status in range(200, 209) and response.get_redirect_location() is False: # Check the content length of the response(if present) to verify whether the url # contains an image which can be used in scene creation content_length = int(response.headers.get('Content-Length', 0)) if content_length != 0 and content_length > 2100: is_url_valid = True conn.clear() return is_url_valid except Exception: return False
def radtherm_get_todays_highest_setting(trace): """Figure out the highest temp setting in today's program.""" days = ["mon", "tue", "wed", "thu", "fri", "sat", "sun"] num_tries = 1 retval = {} prog = RADTHERM_FLOAT_ERROR try: pman = PoolManager() wkdy = datetime.datetime.today().weekday() url = 'http://' + TSTAT_IP +'/tstat/program/heat/' + days[wkdy] while num_tries < 6 and retval.get(str(wkdy), 'error') == 'error': ret = pman.request('GET', url) retval = json.loads(ret.data.decode('utf-8')) if trace: pprt = pprint.PrettyPrinter(indent=4) pprt.pprint(retval) if retval.get(str(wkdy), 'error') != 'error': prog = max((retval[str(wkdy)])[1::2]) # 1,3,5, etc. elements are the temps num_tries += 1 return prog except Exception as err: #pylint: disable=W0703 wg_error_print("radtherm_get_todays_highest_setting", str(err)) if hasattr('err', 'response'): wg_error_print(err.response.text) return prog
def urllib3_test(): from urllib3 import PoolManager errors = 0 oks = 0 p = PoolManager(20) for a in url: try: p.request(method="GET", url=a) oks += 1 except: errors += 1 continue print ".", print print "Errors: %s | Oks: %s." % (str(errors), str(oks))
def run(self): try: response = self.pool.request("GET", self.url, headers=self.headers) except HostChangedError, e: # Redirect, give up on managing resources ourselves, just get the # file managed_pool = PoolManager(1) response = managed_pool.request("GET", e.url, headers=self.headers)
def getUrl(url): """Возвращает содержимое страницы по переданном url""" try: http = PoolManager() r = http.request('GET', url, timeout=Timeout(connect=2.0, read=5.0), retries=Retry(5, redirect=False) ) return html.fromstring(r.data) except urllib3.exceptions.MaxRetryError: print('Превышено максимальное число попыток (5):', url) return None
def getSequential(self, urls_headers): conn_pool = connection_from_url(urls_headers[0][0], maxsize=CONNECTIONS_PER_HOST) responses = [] for url, headers in urls_headers: try: response = conn_pool.request("GET", url, headers=headers) except HostChangedError, e: # Redirect, give up on managing resources ourselves, just get the # file managed_pool = PoolManager(1) response = managed_pool.request("GET", e.url, headers=headers) responses.append((url, response))
class PlainUtility(): def __init__(self): user_agent = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7' self.headers={'User-Agent':user_agent} self.ip_url = 'http://icanhazip.com/' self.logger = logging.getLogger('gkp') retries = Retry(connect=5, read=5, redirect=5) self.agent = PoolManager(10,retries=retries, timeout=Timeout(total=30.0)) def current_ip(self): return self.request(self.ip_url) def request(self,url): r = self.agent.request('GET',url) if r.status == 200: return r.data else: self.logger.error('status %s'%r.status)
def download(self): if self.url is None or self.url == '': return DOWNLOAD_RESULT['FAIL'] ##################### Start Download Web Page ##################### printState(hint="Connecting", msg=self.url) parse_url = parseURL(self.url) scheme = parse_url.scheme (filename, filetype) = getFileInURL(parse_url.path) timeout = Timeout(connect=2., read=7.) if scheme.lower() is 'https': http = PoolManager( cert_reqs='CERT_REQUIRED', ca_certs=certifi.where(), timeout=timeout ) else: http = PoolManager(timeout=timeout) try: r = http.request('GET', self.url) printState(hint='Establish', msg=self.url) except SSLError as e: printFail(hint="SSL Error", msg=self.url) return DOWNLOAD_RESULT['FAIL'] except MaxRetryError as e: printFail(hint="Resolve Error", msg=self.url) return DOWNLOAD_RESULT['FAIL'] ##################### End ##################### ##################### Start Save Web Page ##################### if isNormalConn(r.status): try: file_name = save(data=r.data,filename=filename, dir=DOWNLOAD_DIR) except AttributeError as e: printFail(hint="Save file fail in", msg=self.url) return DOWNLOAD_RESULT['FAIL'] URL_VISITED_FILE_LIST.put(file_name) URL_VISITED_LIST.append(self.url) printSuccess(hint="Finish", msg=self.url) self.url = None self.fail_time = 0 return DOWNLOAD_RESULT['SUCCESS']
def _get_report_file(self, url): # Ensure file url matches the hostname in settings, # workaround for Canvas bug help.instructure.com/tickets/362386 url = re.sub(r'^https://[^/]+', settings.RESTCLIENTS_CANVAS_HOST, url) timeout = getattr(settings, "RESTCLIENTS_TIMEOUT", 15.0) cafile = getattr(settings, "RESTCLIENTS_CA_BUNDLE", "/etc/ssl/certs/ca-bundle.crt") pool_manager = PoolManager(cert_reqs="CERT_REQUIRED", ca_certs=cafile, socket_timeout=timeout, retries=5) response = pool_manager.request("GET", url) if response.status != 200: raise DataFailureException(url, response.status, response.data) return response.data
def test_urllib3_basic(self): ''' Basic ''' # {{{ http = PoolManager() for n in self.numbers: r = http.request('GET', self.prefix_url+n) l = len(r.data)
using = 0 notusing = 0 broke = 0 http = urllib3.PoolManager( cert_reqs='CERT_REQUIRED', # Force certificate check. ca_certs=certifi.where(), # Path to the Certifi bundle. ) while i<len(domains): domain = domains[i].rstrip('\n') url=urlstart + domains[i].rstrip('\r\n') http = PoolManager(timeout=Timeout(read=2.0)) try: check = http.request('GET', url, headers={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1'},timeout=2) response=check.headers if 'strict-transport-security' in response: print "[+] " +site + ': is using HSTS!!!' if 'preload' in str(response): print " [+] Preload enabled" else: print " [Warning!] Preload is not configured" if 'includeSubDomains' in str(response): print " [+] includeSubdomains is present" else: print " [Warning!] includeSubDomains is not configured" if 'max-age=31536000' in str(response): print " [+] max-age is set to two years - well done" else: print " [Warning!] max-age should really be set to two years (31536000)"
class Client(object): """Interface to KISSmetrics tracking service""" def __init__(self, key, trk_host=KISSmetrics.TRACKING_HOSTNAME, trk_scheme=KISSmetrics.TRACKING_SCHEME): """Initialize client for use with KISSmetrics API key. :param key: API key for product, found on the "KISSmetrics Settings". :type key: str :param trk_host: tracking host for requests; defaults production tracking service. :param trk_proto: the protocol for requests; either be `'http'` or `'https'`. """ self.key = key if trk_scheme not in ('http', 'https'): raise ValueError('trk_scheme must be one of (http, https)') self.http = PoolManager() self.trk_host = trk_host self.trk_scheme = trk_scheme def record(self, person, event, properties=None, timestamp=None, path=KISSmetrics.RECORD_PATH): """Record `event` for `person` with any `properties`. :param person: the individual performing the `event` :param event: the `event` name that was performed :param properties: any additional data to include :type properties: dict :param timestamp: when the `event` was performed; optional for back-dating :param path: HTTP endpoint to use; defaults to ``KISSmetrics.RECORD_PATH`` :returns: an HTTP response for the request :rtype: `urllib3.response.HTTPResponse` """ this_request = request.record(self.key, person, event, timestamp=timestamp, properties=properties, scheme=self.trk_scheme, host=self.trk_host, path=path) return self._request(this_request) def set(self, person, properties=None, timestamp=None, path=KISSmetrics.SET_PATH): """Set a property (or properties) for a `person`. :param person: individual to associate properties with :param properties: key-value pairs to associate with `person` :type properties: dict :param timestamp: when the `event` was performed; optional for back-dating :param path: HTTP endpoint to use; defaults to ``KISSmetrics.SET_PATH`` :returns: an HTTP response for the request :rtype: `urllib3.response.HTTPResponse` """ this_request = request.set(self.key, person, timestamp=timestamp, properties=properties, scheme=self.trk_scheme, host=self.trk_host, path=path) return self._request(this_request) def alias(self, person, identity, path=KISSmetrics.ALIAS_PATH): """Map `person` to `identity`; actions done by one resolve to other. :param person: consider as same individual ``identity``; the source of the alias operation :type person: str or unicode :param identity: consider as an alias of ``person``; the target of the alias operation :type identity: str or unicode :param path: HTTP endpoint to use; defaults to ``KISSmetrics.ALIAS_PATH`` :returns: an HTTP response for the request :rtype: `urllib3.response.HTTPResponse` Note the direction of the mapping is ``person`` to ``identity`` (so "``person`` is also known as ``identity``" or "``person`` => ``identity``" when looking at it as "``<source>`` => ``<target>``") When consulting the Aliasing documentation, `person` corresponds to ``query_string.PERSON_PARAM`` and `identity` corresponds to ``query_string.ALIAS_PARAM``. Aliasing is not a reversible operation. When aliasing to an identity, take care not to use a session identifier or any other value that is not relatively stable (a value that will not change per request or per session). For more information see the API Specifications on `Aliasing <http://support.kissmetrics.com/apis/specifications.html#aliasing-users>`_. """ this_request = request.alias(self.key, person, identity, scheme=self.trk_scheme, host=self.trk_host, path=path) return self._request(this_request) def _request(self, uri, method='GET'): return self.http.request(method, uri)
class ProxyService(object): def __init__(self, error_dict): self.redis_service = RedisService() self.error_proxy_dict = error_dict self.connection_pool = PoolManager() def get_proxy(self, protocol): proxy = self.redis_service.read_set(PROXY_URL_KEY.format(protocol=protocol)) if not proxy: return None return {protocol: proxy} def get_valid_size(self, protocol): return self.redis_service.get_set_size(PROXY_URL_KEY.format(protocol=protocol)) def process(self): logger.info('Start load proxy.') #content = self._scrape_http_proxy() #parser_proxy_url_set = self._parser_http_proxy(content) #self._save('http', self._check('http', parser_proxy_url_set)) content = self._scrape_https_proxy() parser_proxy_url_set = self._parser_https_proxy(content) self._save('https', self._check('https', parser_proxy_url_set)) def manage(self, proxy, error): if not proxy: return protocol, proxy_url = proxy.items()[0] if error: if proxy_url in self.error_proxy_dict: self.error_proxy_dict[proxy_url] += 1 if self.error_proxy_dict[proxy_url] > DEFAULT_ERROR_TIMES: self.redis_service.remove_set(PROXY_URL_KEY.format(protocol=protocol), proxy_url) self.error_proxy_dict.pop(proxy_url) logger.info('Invalid proxy: {}'.format(proxy_url)) print 'Invalid proxy' else: self.error_proxy_dict[proxy_url] = 1 else: if proxy_url in self.error_proxy_dict: self.error_proxy_dict[proxy_url] -= 1 if self.error_proxy_dict[proxy_url] < 1: self.error_proxy_dict.pop(proxy_url) logger.info(self.error_proxy_dict) @retry(2) def _scrape_http_proxy(self): scrape_url = 'http://www.xicidaili.com/nn' header = {'content-type': 'text/html', 'User-Agent': user_agents[random.randint(0, len(user_agents)-1)]} try: response = self.request.get(scrape_url, headers=header, proxies=None) return response.content except: raise Exception('Failed scrape proxies.') @retry(2) def _scrape_https_proxy(self): #scrape_url = 'http://www.nianshao.me/?stype=2' scrape_url = 'http://proxy.moo.jp/zh/?c=&pt=&pr=HTTPS&a%5B%5D=0&a%5B%5D=1&a%5B%5D=2&u=60' header = {'content-type': 'text/html', 'Accept-Language': 'zh-CN,zh;q=0.8', 'User-Agent': user_agents[random.randint(0, len(user_agents)-1)]} response = self.connection_pool.request('GET', scrape_url, timeout=60, headers=header) return response.data def _parser_http_proxy(self, content): soup = BeautifulSoup(content, 'html.parser') proxy_tag = soup.find(id='ip_list').select('tr') parser_proxy_url_set = set() for i in range(1, 21): proxy_url = PROXY_URL.format(protocol='http', ip=proxy_tag[i].find_all('td')[1].string, port=proxy_tag[i].find_all('td')[2].string) parser_proxy_url_set.add(proxy_url) return parser_proxy_url_set def _parser_https_proxy(self, content): soup = BeautifulSoup(content, 'html.parser') proxy_tag = soup.find_all('tr', {'class': 'Odd'}) res = re.compile('%(%|\w)+') parser_proxy_url_set = set() for i in range(0, 25): tds = proxy_tag[i].find_all('td') if not tds[0].string: continue ip_res = res.search(tds[0].string) if ip_res: ip = urllib.unquote(ip_res.group(0)) port = tds[1].string proxy_url = PROXY_URL.format(protocol='https', ip=ip, port=port) parser_proxy_url_set.add(proxy_url) return parser_proxy_url_set def _check(self, protocol, proxy_url_set): valid_proxy_url_set = set() for url in proxy_url_set: header = {'content-type': 'text/html', 'User-Agent': user_agents[random.randint(0, len(user_agents)-1)]} proxy = {protocol: url} conection_pool = ProxyManager(url) try: response = conection_pool.request('GET', CHECK_URL[protocol], timeout=60, headers=header) if response.status == 200: valid_proxy_url_set.add(url) print 'Valid proxy url', url else: print 'Invalid ', url except Exception as ex: print ex print 'Invalid ', url return valid_proxy_url_set def _save(self, protocol, parser_proxy_url_set): for url in parser_proxy_url_set: self.redis_service.add_set(PROXY_URL_KEY.format(protocol=protocol), url)