def getRemoteFile(urlOrPath, destPath, proxy={}): ''' Fetches URL to local path or just returns absolute path. :param urlOrPath: resource locator, generally URL or path :param destPath: path to store the resource, usually a path on file system :return: tuple having (path, 'local'/'remote') ''' urlp = urlparse(urlOrPath) if urlp.scheme == '': return (os.path.abspath(urlOrPath), 'local') elif urlp.scheme not in ('http', 'https'): return (urlOrPath, 'local') else: filename = toFilename(urlOrPath) destPath = destPath + '/' + filename log.info('Retrieving %s to %s.' % (urlOrPath, destPath)) try: proxy = urllibreq.ProxyHandler(proxy) opener = urllibreq.build_opener(proxy) urllibreq.install_opener(opener) urlretrieve(urlOrPath, destPath) except IOError: # monkey patch fix for SSL/Windows per Tika-Python #54 # https://github.com/chrismattmann/tika-python/issues/54 import ssl if hasattr(ssl, '_create_unverified_context'): ssl._create_default_https_context = ssl._create_unverified_context # delete whatever we had there if os.path.exists(destPath) and os.path.isfile(destPath): os.remove(destPath) urlretrieve(urlOrPath, destPath) return (destPath, 'remote')
def crawler(): while not q.empty(): # 循环 path = q.get() #将line从队列 q 中取出来 url = "%s%s" % (domain_name, path) # 组合url地址,用于下一步提交 random_proxy = random.choice(proxy_list) # 随机使用一个代理服务器 proxy_support = urllib.ProxyHandler(random_proxy) opener = urllib.build_opener(proxy_support) urllib.install_opener(opener) headers = {} headers['User-Agent'] = Baidu_spider # 蜘蛛的头部信息 # 玩蛇网 www.iplaypython.com request = urllib.Request(url, headers=headers) try: response = urllib.urlopen(request) content = response.read() if len(content): # 内容不为空的情况下返回状态码、路径 print ("Status [%s] - path: %s" % (response.code, path)) response.close() time.sleep(1) # 休息一会儿,防止速度过快连接数过大被封掉IP except urllib.HTTPError as e: # print e.code, path pass # 异常处理,先暂时pass掉
def run(self): if self.proxyHost!=None and self.ProxyPort!=None: QgsMessageLog.logMessage('Proxy? '+str(self.proxyHost), MESSAGE_CATEGORY, Qgis.Info) proxy = urllib.ProxyHandler({'http': proxyHost}) opener = urllib.build_opener(proxy) urllib.install_opener(opener) QgsMessageLog.logMessage('Started task "{}"'.format(self.description()),MESSAGE_CATEGORY, Qgis.Info) self.graph = Graph() try: if self.filename.startswith("http"): self.graph.load(self.filename) else: filepath=self.filename.split(".") result = self.graph.parse(self.filename, format=filepath[len(filepath)-1]) except Exception as e: QgsMessageLog.logMessage('Failed "{}"'.format(self.description()),MESSAGE_CATEGORY, Qgis.Info) self.exception=str(e) return False self.geoconcepts=[] if self.graph!=None: print("WE HAVE A GRAPH") results = self.graph.query(self.query) for row in results: self.geoconcepts.append(str(row[0])) return True
def __init__(self, proxy): self.register_in_service_locator() self.db_config = self.get_component("db_config") self.target = self.get_component("target") self.transaction = self.get_component("transaction") self.url_manager = self.get_component("url_manager") self.error_handler = self.get_component("error_handler") self.plugin_handler = self.get_component("plugin_handler") self.timer = self.get_component("timer") self.http_transaction = None self.headers = {'User-Agent': self.db_config.get('USER_AGENT')} self.req_count_refused = 0 self.req_count_total = 0 self.log_transactions = False self.proxy = proxy if proxy is None: logging.debug( "WARNING: No outbound proxy selected. It is recommended to " "use an outbound proxy for tactical fuzzing later") self.opener = build_opener(MyHTTPHandler, MyHTTPSHandler, SmartRedirectHandler) else: # All requests must use the outbound proxy. logging.debug("Setting up proxy(inbound) for OWTF requests..") ip, port = proxy proxy_conf = { 'http': 'http://%s:%s' % (ip, port), 'https': 'http://%s:%s' % (ip, port) } proxy_handler = ProxyHandler(proxy_conf) # FIXME: Works except no raw request on https. self.opener = build_opener(proxy_handler, MyHTTPHandler, MyHTTPSHandler, SmartRedirectHandler) install_opener(self.opener)
def testTripleStoreConnection( self, query="SELECT ?a ?b ?c WHERE { ?a ?b ?c .} LIMIT 1"): if self.proxyHost != None and self.ProxyPort != None: proxy = urllib.ProxyHandler({'http': proxyHost}) opener = urllib.build_opener(proxy) urllib.install_opener(opener) sparql = SPARQLWrapper( self.triplestoreurl, agent= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11" ) sparql.setQuery(query) sparql.setReturnFormat(JSON) print("now sending query") try: results = sparql.query().convert() if self.testURL and not self.testConfiguration: self.message = "URL depicts a valid SPARQL Endpoint!" if "ASK" in query: return results["boolean"] self.feasibleConfiguration = True return True except: self.message = "URL does not depict a valid SPARQL Endpoint!" self.feasibleConfiguration = False return False
def post_data(self, data, username, password): utc_datetime = datetime.datetime.utcnow() url_parameters = { 'cluster': self.elasticMonitoringCluster, 'index': self.elasticIndex, 'index_period': utc_datetime.strftime("%Y.%m.%d"), } url = "%(cluster)s/%(index)s-%(index_period)s/hrbdata" % url_parameters headers = {'content-type': 'application/json'} try: req = Request(url) req.add_header('Content-Type', 'application/json; charset=utf-8') jsondata = json.dumps(data) jsondataasbytes = jsondata.encode('utf-8') req.add_header('Content-Length', len(jsondataasbytes)) data = urllib.parse.urlencode(data).encode("utf-8") # req = urllib.request.urlopen(url, headers=headers, data=json.dumps(data)) if self.write_es_security_enable: password_mgr = urllib.HTTPPasswordMgrWithDefaultRealm() password_mgr.add_password(None, url, username, password) handler = urllib.HTTPBasicAuthHandler(password_mgr) opener = urllib.build_opener(handler) urllib.install_opener(opener) response = urllib.request.urlopen(req, jsondataasbytes) else: response = urllib.request.urlopen(req, jsondataasbytes) except Exception as e: print("Error: {}".format(str(e)))
def auth_as_app(self) -> str: logger.info("Getting access token for app") self.request_time = datetime.datetime.now() data = {'grant_type': 'client_credentials'} pw_manager = urllib.HTTPPasswordMgrWithDefaultRealm() pw_manager.add_password(None, constants.OAUTH_URL, self.app_id, self.app_secret) urllib.install_opener( urllib.build_opener(urllib.HTTPBasicAuthHandler(pw_manager))) request = urllib.Request(constants.OAUTH_URL, data=data, headers=constants.OAUTH_REQUEST_HEADERS) try: with urllib.request.urlopen(request) as response: self.full_response = json.loads(response.read()) self.jwt = self.full_response["access_token"] self.expires_in_secs = datetime.timedelta( seconds=self.full_response["expires_in"]) self.expiry_time = self.request_time + self.expires_in_secs - constants.JWT_TOKEN_REFRESH_BUFFER logger.log( 5, "refreshed auth token at %s - token will expire in %s, with a buffer next refresh will be at %s", self.request_time, self.expires_in_secs, self.expiry_time) return self.full_response except urllib.error.HTTPError as err: logger.critical( "Tried to authenticate an app, got error code %s with reason %s", err.code, err.reason) raise UnauthorizedRequestError()
def run(self): QgsMessageLog.logMessage( 'Started task "{}"'.format(self.description()), MESSAGE_CATEGORY, Qgis.Info) if self.proxyHost != None and self.ProxyPort != None: QgsMessageLog.logMessage('Proxy? ' + str(self.proxyHost), MESSAGE_CATEGORY, Qgis.Info) proxy = urllib.ProxyHandler({'http': proxyHost}) opener = urllib.build_opener(proxy) urllib.install_opener(opener) sparql = SPARQLWrapper( self.triplestoreurl, agent= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11" ) sparql.setQuery(self.query) print("now sending query") sparql.setReturnFormat(JSON) results = sparql.query().convert() for result in results["results"]["bindings"]: self.viewlist.append(str(result[self.queryvar]["value"])) print(self.viewlist) #self.layercount.setText("["+str(len(viewlist))+"]") if self.getlabels and "classlabelquery" in self.triplestoreconf and self.triplestoreconf[ "classlabelquery"] != "": labels = self.getLabelsForClasses( self.viewlist, self.triplestoreconf["classlabelquery"]) print(labels) self.amountoflabels = len(labels) i = 0 sorted_labels = sorted(labels.items(), key=lambda x: x[1]) for lab in sorted_labels: self.resultlist.append(labels[lab[0]] + " (" + lab[0] + ")") i = i + 1 return True
def crawler(): while not q.empty(): # 循环 path = q.get() #将line从队列 q 中取出来 url = "%s%s" % (domain_name, path) # 组合url地址,用于下一步提交 random_proxy = random.choice(proxy_list) # 随机使用一个代理服务器 proxy_support = urllib.ProxyHandler(random_proxy) opener = urllib.build_opener(proxy_support) urllib.install_opener(opener) headers = {} headers['User-Agent'] = Baidu_spider # 蜘蛛的头部信息 # 玩蛇网 www.iplaypython.com request = urllib.Request(url, headers=headers) try: response = urllib.urlopen(request) content = response.read() if len(content): # 内容不为空的情况下返回状态码、路径 print("Status [%s] - path: %s" % (response.code, path)) response.close() time.sleep(1) # 休息一会儿,防止速度过快连接数过大被封掉IP except urllib.HTTPError as e: # print e.code, path pass # 异常处理,先暂时pass掉
def run(self): QgsMessageLog.logMessage('Started task "{}"'.format( self.description()), MESSAGE_CATEGORY, Qgis.Info) if self.proxyHost!=None and self.ProxyPort!=None: QgsMessageLog.logMessage('Proxy? '+str(self.proxyHost), MESSAGE_CATEGORY, Qgis.Info) proxy = urllib.ProxyHandler({'http': proxyHost}) opener = urllib.build_opener(proxy) urllib.install_opener(opener) sparql = SPARQLWrapper(self.triplestoreurl, agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11") sparql.setQuery(self.query) sparql.setMethod(POST) sparql.setReturnFormat(JSON) try: results = sparql.query().convert() except Exception as e: try: sparql = SPARQLWrapper(self.triplestoreurl, agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11") sparql.setQuery(self.query) sparql.setMethod(GET) sparql.setReturnFormat(JSON) results = sparql.query().convert() except Exception as e: self.exception=e return #print(results) # geojson stuff self.geojson=self.processResults(results,(self.triplestoreconf["crs"] if "crs" in self.triplestoreconf else ""),self.triplestoreconf["mandatoryvariables"][1:],self.allownongeo) return True
def detectNamespaces(self, subpredobj): if subpredobj < 0 or subpredobj == None: query = "select distinct ?ns where { ?s ?p ?o . bind( replace( str(?s), \"(#|/)[^#/]*$\", \"$1\" ) as ?ns )} limit 10" elif subpredobj == 0: query = "select distinct ?ns where { ?s ?p ?o . bind( replace( str(?p), \"(#|/)[^#/]*$\", \"$1\" ) as ?ns )} limit 10" else: query = "select distinct ?ns where { ?s ?p ?o . bind( replace( str(?o), \"(#|/)[^#/]*$\", \"$1\" ) as ?ns )} limit 10" if self.proxyHost != None and self.ProxyPort != None: proxy = urllib.ProxyHandler({'http': proxyHost}) opener = urllib.build_opener(proxy) urllib.install_opener(opener) sparql = SPARQLWrapper( self.triplestoreurl, agent= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11" ) sparql.setQuery(query) sparql.setReturnFormat(JSON) print("now sending query") try: results = sparql.query().convert() reslist = [] for nss in results["results"]["bindings"]: if "ns" in nss: reslist.append(nss["ns"]["value"]) return reslist except: return []
def fetion(msg): cj = cookielib.LWPCookieJar() opener = urllib.build_opener(urllib.HTTPCookieProcessor(cj)) urllib.install_opener(opener) args = {'pass': password, 'm': user, 'loginstatus': loginstatus} print('Logining...') req = urllib.Request(url_login, urllib.urlencode(args)) jump = opener.open(req) page = jump.read() url = re.compile(r'<card id="start".*?ontimer="(.*?);').findall(page)[ 0] #��ȡ��ת���� arg_t = re.compile(r't=(\d*)').findall(page)[0] if url == '/im/login/login.action': #��¼ʧ�� print('Login Failed!') input('Press any key to exit.') return else: print('Login Successfully!') sendmsg = urllib.Request( url_msg, urllib.urlencode({'msg': msg.decode('gbk').encode('utf-8')})) finish = urllib.urlopen(sendmsg) if finish.geturl == 'http://f.10086.cn/im/user/sendMsgToMyself.action': print('Send Failed!') else: print('Send Successfully') logout = urllib.Request(url_logout + arg_t) response = urllib.urlopen(logout) #ע�� print('Logout Successfully!')
def __init__(self, column, row, triplestoreconf, prefixes, interlinkOrEnrich, table, propOrClass=False, bothOptions=False, currentprefixes=None, addVocab=None): super(QDialog, self).__init__() self.setupUi(self) self.currentcol = column self.currentrow = row self.table = table self.prefixes = prefixes self.currentprefixes = currentprefixes self.bothOptions = bothOptions self.triplestoreconf = triplestoreconf self.interlinkOrEnrich = interlinkOrEnrich self.addVocab = addVocab if column != 4: self.findConcept.setChecked(True) if column == 4 or (not interlinkOrEnrich and column != 4) or (not interlinkOrEnrich and propOrClass): self.findProperty.setChecked(True) if not bothOptions: self.findProperty.setEnabled(False) self.findConcept.setEnabled(False) self.tripleStoreEdit.setEnabled(False) for triplestore in self.triplestoreconf: if not "File" == triplestore["name"]: self.tripleStoreEdit.addItem(triplestore["name"]) if addVocab != None: for cov in addVocab: self.tripleStoreEdit.addItem(addVocab[cov]["label"]) self.searchButton.clicked.connect(self.getClassesFromLabel) urlregex = QRegExp( "http[s]?://(?:[a-zA-Z#]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+" ) urlvalidator = QRegExpValidator(urlregex, self) self.costumproperty.setValidator(urlvalidator) self.costumproperty.textChanged.connect(self.check_state3) self.costumproperty.textChanged.emit(self.costumproperty.text()) self.costumpropertyButton.clicked.connect(self.applyConceptToColumn2) self.applyButton.clicked.connect(self.applyConceptToColumn) s = QSettings() #getting proxy from qgis options settings self.proxyEnabled = s.value("proxy/proxyEnabled") self.proxyType = s.value("proxy/proxyType") self.proxyHost = s.value("proxy/proxyHost") self.proxyPort = s.value("proxy/proxyPort") self.proxyUser = s.value("proxy/proxyUser") self.proxyPassword = s.value("proxy/proxyPassword") if self.proxyHost != None and self.ProxyPort != None: proxy = urllib.ProxyHandler({'http': proxyHost}) opener = urllib.build_opener(proxy) urllib.install_opener(opener)
def getHtml(url): USER_AGENTS = [ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1", "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6", "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5", "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10", "Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/533.17.8 (KHTML, like Gecko) Version/5.0.1 Safari/533.17.8", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.1.17) Gecko/20110123 (like Firefox/3.x) SeaMonkey/2.0.12", "Mozilla/5.0 (Windows NT 5.2; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 SeaMonkey/2.7.1", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/532.8 (KHTML, like Gecko) Chrome/4.0.302.2 Safari/532.8", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.464.0 Safari/534.3", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.15 Safari/534.13", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.54 Safari/535.2", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7", "Mozilla/5.0 (Macintosh; U; Mac OS X Mach-O; en-US; rv:2.0a) Gecko/20040614 Firefox/3.0.0 ", "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10.5; en-US; rv:1.9.0.3) Gecko/2008092414 Firefox/3.0.3", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-US; rv:1.9.1) Gecko/20090624 Firefox/3.5", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2.14) Gecko/20110218 AlexaToolbar/alxf-2.0 Firefox/3.6.14", "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10.5; en-US; rv:1.9.2.15) Gecko/20110303 Firefox/3.6.15", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1" ] # 代理的IP设置 proxies = ['114.215.95.188:3128', '218.14.115.211:3128'] req = urllib.Request(url) # 设置消息头 req.add_header('User-Agent', random.choice(USER_AGENTS)) # 设置代理ip地址 proxy_support = urllib.ProxyHandler({"http": random.choice(proxies)}) opener = urllib.build_opener(proxy_support) urllib.install_opener(opener) # 访问并获取服务端返回的对象 times = 0 try: res = urllib.urlopen(req) html = res.read() return html except: times += 1 getHtml(url)
def get_unread_msgs(user, passwd): auth_handler = urllib.HTTPBasicAuthHandler() auth_handler.add_password(realm='New mail feed', uri='https://mail.google.com', user='******'.format(user=user), passwd=passwd) opener = urllib.build_opener(auth_handler) urllib.install_opener(opener) feed = urllib.request.urlopen(FEED_URL) return feed.read()
def handle_urlopen(self, urlData, username, password): if self.read_es_security_enable: password_mgr = urllib.HTTPPasswordMgrWithDefaultRealm() password_mgr.add_password(None, urlData, username, password) handler = urllib.HTTPBasicAuthHandler(password_mgr) opener = urllib.build_opener(handler) urllib.install_opener(opener) response = urllib.request.urlopen(urlData) else: response = urllib.request.urlopen(urlData) return response.read().decode('utf-8')
def setupProxy(): proxies = {} if _proxyhttp != None: proxies['http'] = 'http://' + _proxyhttp os.environ['http'] = _proxyhttp if _proxyhttps != None: proxies['https'] = _proxyhttps os.environ['https'] = 'http://' + _proxyhttps if proxies != {}: proxy = urllib.ProxyHandler(proxies) opener = urllib.build_opener(proxy) urllib.install_opener(opener)
def __init__(self, ip_address, port, username="******", password="******"): PowerDevice.__init__(self, ip_address, username, password) self._ip_address = ip_address self.port = port # create a password manager password_mgr = _urllib.HTTPPasswordMgrWithDefaultRealm() password_mgr.add_password(None, 'http://' + ip_address, username, password) handler = _urllib.HTTPBasicAuthHandler(password_mgr) opener = _urllib.build_opener(handler) # Now all calls to _urllib.urlopen use our opener. _urllib.install_opener(opener)
def http_auth_request(url, host, user, passwd, user_agent=USER_AGENT): """Call an HTTP server with authorization credentials using urllib. """ if DEBUG: http.client.HTTPConnection.debuglevel = 1 # Hook up handler/opener to urllib password_manager = urllib.HTTPPasswordMgrWithDefaultRealm() password_manager.add_password(None, host, user, passwd) auth_handler = urllib.HTTPBasicAuthHandler(password_manager) opener = urllib.build_opener(auth_handler) urllib.install_opener(opener) return http_request(url, user_agent)
def save_cookie(self, fileName, url): #不能在有别的实例运行时执行 #声明一个MozillaCookieJar实例保存cookie cookie = http.cookiejar.MozillaCookieJar(fileName) #构建opener opener = urllib.build_opener(urllib.HTTPCookieProcessor(cookie)) urllib.install_opener(opener) request = urllib.Request.open(url) request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36') response = urllib.urlopen(request) print (response.getcode()) cookie.save(ignore_discard=True, ignore_expires=True) print ('Successfully saved')
def get_url(self, url, proxy_dict): proxyIP = proxy_dict['ip'] proxyPort = proxy_dict['port'] proxyProtocol = proxy_dict['protocol'] proxy_handler = urllib.ProxyHandler({proxyProtocol: "{0}:{1}".format(proxyIP, proxyPort)}) opener_proxy = urllib.build_opener(proxy_handler) urllib.install_opener(opener_proxy) request = urllib.Request(url=url, headers=HEADERS) response = urllib.urlopen(request) html = response.read() return html
def get_with_username_password(username, password): # create a password manager password_mgr = _urllib.HTTPPasswordMgrWithDefaultRealm() password_mgr.add_password(None, 'http://' + ip_address, username, password) handler = _urllib.HTTPBasicAuthHandler(password_mgr) opener = _urllib.build_opener(handler) opener.open('http://' + ip_address) _urllib.install_opener(opener) request = _urllib.Request('http://' + ip_address) response = opener.open(request) data = response.read() return data
def useProxy(self, proxy): '''利用代理访问百度,并查找关键词''' protocol = proxy.split('//')[0].split(":", '') ip = proxy.split('//')[1] opener = urllib.build_opener(urllib.ProxyHandler({protocol: ip})) urllib.install_opener(opener) try: response = urllib.request.urlopen(self.url, timeout=self.timeout) except: print(u'连接错误,退出程序') exit() str = response.read() if re.search(self.flagword, str): print(u"已取得关键词,该代理可用") else: print("该代理不可用")
def SetProxiesIfNecessary(): global HTTP_PROXY global HTTPS_PROXY dProxies = {} if HTTP_PROXY != '': dProxies['http'] = HTTP_PROXY if HTTPS_PROXY != '': dProxies['https'] = HTTPS_PROXY if os.getenv('http_proxy') != None: dProxies['http'] = os.getenv('http_proxy') if os.getenv('https_proxy') != None: dProxies['https'] = os.getenv('https_proxy') if dProxies != {}: urllib.install_opener( urllib.build_opener(urllib.ProxyHandler(dProxies)))
def httpConnection(url, proxy): #TODO: habilitar autenticacion ntlm if (proxy.auth == "ntlm"): passman = urllib.HTTPPasswordMgrWithDefaultRealm() passman.add_password(None, proxy.url, proxy.user, proxy.password) auth = HTTPNtlmAuthHandler.HTTPNtlmAuthHandler(passman) else: passman = urllib.request.HTTPPasswordMgr() passman.add_password(None, proxy.url, proxy.user, proxy.password) auth = urllib.request.HTTPBasicAuthHandler(passman) if (proxy.url): proxy = urllib.ProxyHandler({'http': proxy.url}) opener = urllib.build_opener(proxy.url, auth, urllib2.HTTPHandler) urllib.install_opener(opener) return urllib.request.urlopen(url)
def __init__(self, username, passwd, debug=False): (self.username, self.passwd) = (username, passwd) self.cookie = cookielib.CookieJar() cookieHandler = urllib.HTTPCookieProcessor(self.cookie) self.is_debug = debug if self.is_debug: httpHandler = urllib.HTTPHandler(debuglevel=1) httpsHandler = urllib.HTTPSHandler(debuglevel=1) opener = urllib.build_opener(cookieHandler, httpHandler, httpsHandler) else: opener = urllib.build_opener(cookieHandler) urllib.install_opener(opener) self.last_url = ''
def spider_url_by_post(url, data, proxy=None, enable_proxy=False): # make a string with 'POST' method = 'POST' # create a handler proxy_handler = urllib2.ProxyHandler(proxy) null_proxy_handler = urllib2.ProxyHandler({}) # create an openerdirector instance according to enable_proxy if enable_proxy: cookies = urllib2.HTTPCookieProcessor() opener = urllib2.build_opener(cookies, proxy_handler, urllib2.HTTPHandler) print 'without using proxy to crawl pages' else : opener = urllib2.build_opener(null_proxy_handler) print 'using proxy to crawl pages' # install opener urllib.install_opener(opener) # buidl a request #data = urllib.urlencode(dictionary_of_POST_fiels_or_None) data = urllib.urlencode(data) request = urllib2.Request(url, data=data) # Ramdom choose the user_agent #user_agent = random.choice(USER_AGENT_LSIT) user_agent = USER_AGENT_LIST[1] request.add_header('User-Agent', user_agent) request.get_method = lambda:method try : connection = opener.open(request, timeout=5) if connection.code == 200: html = connection.read() return html #else if connection.code == 403: # return None except urllib2.HTTPError, ex: # print e.code, e.reason print 'spider_url_by_get() -------- ', str(ex) # connection = ex return None
def post(body): ''' This function actually communicates with the FreshBooks API ''' # setup HTTP basic authentication password_mgr = urllib.HTTPPasswordMgrWithDefaultRealm() url = "" if account_url.find('//') == -1: url = "https://" url += account_url + SERVICE_URL password_mgr.add_password(None, url, auth_token, '') handler = urllib.HTTPBasicAuthHandler(password_mgr) opener = urllib.build_opener(handler) urllib.install_opener(opener) # make the request and return the response body request = urllib.Request(url, body, request_headers) response = urllib.urlopen(request) response_content = response.read() return response_content
def run(self): url = self._url outputfile = self._target def reporthook(blocknum, blocksize, totalsize): readsofar = blocknum * blocksize percent = 0 if totalsize > 0: percent = readsofar * 1e2 / totalsize s = "\r%5.1f%% %*d / %d" % (percent, len( str(totalsize)), readsofar, totalsize) sys.stderr.write(s) if readsofar >= totalsize: sys.stderr.write("\n") else: sys.stderr.write("read %d\n" % (readsofar, )) self.signal.emit(int(percent)) proxy = urllib.ProxyHandler({'http': "myproxy"}) opener = urllib.build_opener(proxy) urllib.install_opener(opener) urllib.urlretrieve(url, outputfile, reporthook)
def CBDownload(env, target, url): try: import urllib # Python 3+ except ImportError: import urllib2 as urllib sys.stdout.write('Downloading ' + url + '.') sys.stdout.flush() ftp_proxy = os.getenv('ftp_proxy', None) http_proxy = os.getenv('http_proxy', None) if ftp_proxy or http_proxy: handlers = {} if ftp_proxy: handlers['ftp'] = ftp_proxy if http_proxy: handlers['http'] = http_proxy opener = urllib.build_opener(urllib.ProxyHandler(handlers)) urllib.install_opener(opener) f = None stream = None try: stream = urllib.urlopen(url) f = open(target, 'wb', 0) # Unbuffered while stream and f: data = stream.read(1024 * 1024) if not data: break f.write(data) sys.stdout.write('.') sys.stdout.flush() sys.stdout.write('ok\n') sys.stdout.flush() finally: if f is not None: f.close() if stream is not None: stream.close()
def download_from_url(url): proxy = env_server.get_proxy() if proxy['enabled']: server = proxy['server'].replace('http://', '') proxy_dict = { 'http': 'http://{login}:{pass}@{0}'.format(server, **proxy) } proxy_handler = urllib2.ProxyHandler(proxy_dict) auth = urllib2.HTTPBasicAuthHandler() opener = urllib2.build_opener(proxy_handler, auth, urllib2.HTTPHandler) urllib2.install_opener(opener) def url_open_agent(url=url, timeout=1): return urllib2.urlopen(url=url, timeout=timeout) query_worker = gf.get_thread_worker(url_open_agent, error_func=gf.error_handle) query_worker.try_start() thread_pool = query_worker.get_thread_pool() thread_pool.waitForDone() if query_worker.is_failed(): return False
def httpConnection(url, proxy, agent, ignore): if (proxy.auth == "ntlm"): #todo print("") else: auth_handler = urllib.request.HTTPBasicAuthHandler() auth_handler.add_password(realm='Proxy', uri=proxy.url, user=proxy.user, passwd=proxy.password) if (proxy.url): opener = urllib.request.build_opener(auth_handler) urllib.install_opener(opener) req = Request(url) if len(agent) > 2: req.add_header('user-agent', agent) if (ignore): return urlopen(req, context=ignore) else: return urlopen(req)
def run(self): QgsMessageLog.logMessage('Started task "{}"'.format( self.description()), MESSAGE_CATEGORY, Qgis.Info) if self.proxyHost!=None and self.ProxyPort!=None: QgsMessageLog.logMessage('Proxy? '+str(self.proxyHost), MESSAGE_CATEGORY, Qgis.Info) proxy = urllib.ProxyHandler({'http': proxyHost}) opener = urllib.build_opener(proxy) urllib.install_opener(opener) attlist={} attlist[self.item]=[] attlist[self.idfield]={} for f in self.layer.getFeatures(): if self.item in f: attlist[self.item].append(f[self.item]) attlist[self.idfield][f[self.idfield]]=True query="" if self.content=="Enrich URI": query+="SELECT ?item WHERE {\n" elif self.content=="Enrich Value" or self.strategy=="Enrich Both": query+="SELECT ?item ?val ?valLabel ?vals WHERE {\n" query+="VALUES ?vals { " print(attlist) for it in attlist[self.idfield]: if str(it).startswith("http"): query+="<"+str(it)+"> " elif self.idprop=="http://www.w3.org/2000/01/rdf-schema#label" and self.language!=None and self.language!="": query+="\""+str(it)+"\"@"+self.language+" " else: query+="\""+str(it)+"\" " query+=" } . \n" proppp=self.propertyy.data(1) if self.propertyy.data(1).startswith("//"): proppp="http:"+proppp if self.table.item(self.row, 7).text()!="" and "wikidata" in self.triplestoreurl: query+="?item wdt:P31 <"+self.table.item(self.row, 7).text()+"> .\n" else: query+="?item rdf:type <"+self.table.item(self.row, 7).text()+"> .\n" query+="?item <"+self.idprop+"> ?vals .\n" query+="?item <"+proppp+"> ?val . \n" if (self.content=="Enrich Value" or self.content=="Enrich Both") and not "wikidata" in self.triplestoreurl: query+="OPTIONAL{ ?val rdfs:label ?valLabel }" elif (self.content=="Enrich Value" or self.content=="Enrich Both") and "wikidata" in self.triplestoreurl: query+="SERVICE wikibase:label { bd:serviceParam wikibase:language \"[AUTO_LANGUAGE],"+self.language+"\". }\n" query+="} " QgsMessageLog.logMessage("proppp: "+str(proppp), MESSAGE_CATEGORY, Qgis.Info) QgsMessageLog.logMessage("idprop: "+self.idprop, MESSAGE_CATEGORY, Qgis.Info) QgsMessageLog.logMessage(query, MESSAGE_CATEGORY, Qgis.Info) QgsMessageLog.logMessage(self.triplestoreurl, MESSAGE_CATEGORY, Qgis.Info) print(self.triplestoreurl) try: sparql = SPARQLWrapper(self.triplestoreurl, agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11") sparql.setQuery(query) sparql.setMethod(POST) print("now sending query") sparql.setReturnFormat(JSON) results = sparql.query().convert() except Exception as e: QgsMessageLog.logMessage("Trying GET query", MESSAGE_CATEGORY, Qgis.Info) try: sparql = SPARQLWrapper(self.triplestoreurl, agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11") sparql.setQuery(query) sparql.setMethod(GET) sparql.setReturnFormat(JSON) results = sparql.query().convert() except Exception as e: #msgBox=QMessageBox() #msgBox.setText("The following exception occurred: "+str(e)) #msgBox.exec() return False print(str(results)) #resultcounter=0 for resultcounter in results["results"]["bindings"]: if self.content=="Enrich Value": self.resultmap[resultcounter["vals"]["value"]]=resultcounter["valLabel"]["value"] elif self.content=="Enrich URI": self.resultmap[resultcounter["vals"]["value"]]=resultcounter["val"]["value"] else: self.resultmap[resultcounter["vals"]["value"]]=resultcounter["valLabel"]["value"]+";"+resultcounter["val"]["value"] self.columntype=self.detectColumnType(self.resultmap,self.table) QgsMessageLog.logMessage(str(self.columntype), MESSAGE_CATEGORY, Qgis.Info) QgsMessageLog.logMessage(str(self.resultmap), MESSAGE_CATEGORY, Qgis.Info) return True
def main(): opener = urllib.build_opener(urllib.HTTPCookieProcessor(cookielib.CookieJar())) urllib.install_opener(opener) if getUUID() == False: print('获取uuid失败') return showQRImage() time.sleep(1) while waitForLogin() != '200': pass os.remove(QRImagePath) if login() == False: print('登录失败') return if webwxinit() == False: print('初始化失败') return MemberList = webwxgetcontact() MemberCount = len(MemberList) print('通讯录共%s位好友') % MemberCount ChatRoomName = '' result = [] for i in range(0, int(math.ceil(MemberCount / float(MAX_GROUP_NUM)))): UserNames = [] NickNames = [] DeletedList = '' for j in range(0, MAX_GROUP_NUM): if i * MAX_GROUP_NUM + j >= MemberCount: break Member = MemberList[i * MAX_GROUP_NUM + j] UserNames.append(Member['UserName']) NickNames.append(Member['NickName'].encode('utf-8')) print('第%s组...') % (i + 1) print(', '.join(NickNames)) print('回车键继续...') input('say soemthing:') # 新建群组/添加成员 if ChatRoomName == '': (ChatRoomName, DeletedList) = createChatroom(UserNames) else: DeletedList = addMember(ChatRoomName, UserNames) DeletedCount = len(DeletedList) if DeletedCount > 0: result += DeletedList print('找到%s个被删好友') % DeletedCount # raw_input() # 删除成员 deleteMember(ChatRoomName, UserNames) # todo 删除群组 resultNames = [] for Member in MemberList: if Member['UserName'] in result: NickName = Member['NickName'] if Member['RemarkName'] != '': NickName += '(%s)' % Member['RemarkName'] resultNames.append(NickName.encode('utf-8')) print('---------- 被删除的好友列表 ----------') print('\n'.join(resultNames)) print('-----------------------------------')
import urllib import time username = "******" password = "******" def send_to_twitter(msg): password_manager = urllib.HTTPPasswordMgr() password_manager.add_password("Twitter API", "http://twitter.com/statuses", username, password) http_handler = urllib.HTTPBasicAuthHandler(password_manager) page_opener = urllib.build_opener(http_handler) urllib.install_opener(page_opener) params = urllib.parse.urlencode( {'status': msg}) resp = urllib.urlopen("http://twitter.com/statuses/update.json", params) resp.read() def get_price(): price = urllib.urlopen("http://") text = page.read().decode("utf8") where = text.find('>$') start_of_price = text + 2 end_of_price = start_of_price +4 return(text[start_of_price:end_of_price]) get_price() price_now = input("Do you need a price right now? (Y/N)") if price_now == "Y": send_to_twitter(get_price()) else: price = 99.99
def OP_RETURN_bitcoin_cmd(command, testnet, *args): # more params are read from here if OP_RETURN_BITCOIN_USE_CMD: sub_args = [OP_RETURN_BITCOIN_PATH] if testnet: sub_args.append('-testnet') sub_args.append(command) for arg in args: sub_args.append( json.dumps(arg) if isinstance(arg, (dict, list, tuple)) else str(arg)) raw_result = subprocess.check_output(sub_args).decode("utf-8").rstrip( "\n") try: # decode JSON if possible result = json.loads(raw_result) except ValueError: result = raw_result else: request = { 'id': str(time.time()) + '-' + str(random.randint(100000, 999999)), 'method': command, 'params': args, } port = OP_RETURN_BITCOIN_PORT user = OP_RETURN_BITCOIN_USER password = OP_RETURN_BITCOIN_PASSWORD if not (len(port) and len(user) and len(password)): conf_lines = open(os.path.expanduser('~') + '/.zen/zen.conf').readlines() for conf_line in conf_lines: parts = conf_line.strip().split('=', 1) # up to 2 parts if (parts[0] == 'rpcport') and not len(port): port = int(parts[1]) if (parts[0] == 'rpcuser') and not len(user): user = parts[1] if (parts[0] == 'rpcpassword') and not len(password): password = parts[1] if not len(port): port = 18231 if testnet else 8231 if not (len(user) and len(password)): return None # no point trying in this case url = 'http://' + OP_RETURN_BITCOIN_IP + ':' + str(port) + '/' try: from urllib import HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, build_opener, install_opener, urlopen except ImportError: from urllib.request import HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, build_opener, install_opener, urlopen passman = HTTPPasswordMgrWithDefaultRealm() passman.add_password(None, url, user, password) auth_handler = HTTPBasicAuthHandler(passman) opener = build_opener(auth_handler) install_opener(opener) raw_result = urlopen(url, json.dumps(request).encode('utf-8'), OP_RETURN_NET_TIMEOUT).read() result_array = json.loads(raw_result.decode('utf-8')) result = result_array['result'] return result