def getRemoteFile(urlOrPath, destPath, proxy={}):
'''
Fetches URL to local path or just returns absolute path.
:param urlOrPath: resource locator, generally URL or path
:param destPath: path to store the resource, usually a path on file system
:return: tuple having (path, 'local'/'remote')
'''
urlp = urlparse(urlOrPath)
if urlp.scheme == '':
return (os.path.abspath(urlOrPath), 'local')
elif urlp.scheme not in ('http', 'https'):
return (urlOrPath, 'local')
else:
filename = toFilename(urlOrPath)
destPath = destPath + '/' + filename
log.info('Retrieving %s to %s.' % (urlOrPath, destPath))
try:
proxy = urllibreq.ProxyHandler(proxy)
opener = urllibreq.build_opener(proxy)
urllibreq.install_opener(opener)
urlretrieve(urlOrPath, destPath)
except IOError:
# monkey patch fix for SSL/Windows per Tika-Python #54
# https://github.com/chrismattmann/tika-python/issues/54
import ssl
if hasattr(ssl, '_create_unverified_context'):
ssl._create_default_https_context = ssl._create_unverified_context
# delete whatever we had there
if os.path.exists(destPath) and os.path.isfile(destPath):
os.remove(destPath)
urlretrieve(urlOrPath, destPath)
return (destPath, 'remote')
def crawler():
while not q.empty(): # 循环
path = q.get() #将line从队列 q 中取出来
url = "%s%s" % (domain_name, path) # 组合url地址，用于下一步提交
random_proxy = random.choice(proxy_list) # 随机使用一个代理服务器
proxy_support = urllib.ProxyHandler(random_proxy)
opener = urllib.build_opener(proxy_support)
urllib.install_opener(opener)
headers = {}
headers['User-Agent'] = Baidu_spider # 蜘蛛的头部信息
# 玩蛇网 www.iplaypython.com
request = urllib.Request(url, headers=headers)
try:
response = urllib.urlopen(request)
content = response.read()
if len(content): # 内容不为空的情况下返回状态码、路径
print ("Status [%s] - path: %s" % (response.code, path))
response.close()
time.sleep(1) # 休息一会儿，防止速度过快连接数过大被封掉IP
except urllib.HTTPError as e:
# print e.code, path
pass # 异常处理，先暂时pass掉
def run(self):
if self.proxyHost!=None and self.ProxyPort!=None:
QgsMessageLog.logMessage('Proxy? '+str(self.proxyHost), MESSAGE_CATEGORY, Qgis.Info)
proxy = urllib.ProxyHandler({'http': proxyHost})
opener = urllib.build_opener(proxy)
urllib.install_opener(opener)
QgsMessageLog.logMessage('Started task "{}"'.format(self.description()),MESSAGE_CATEGORY, Qgis.Info)
self.graph = Graph()
try:
if self.filename.startswith("http"):
self.graph.load(self.filename)
else:
filepath=self.filename.split(".")
result = self.graph.parse(self.filename, format=filepath[len(filepath)-1])
except Exception as e:
QgsMessageLog.logMessage('Failed "{}"'.format(self.description()),MESSAGE_CATEGORY, Qgis.Info)
self.exception=str(e)
return False
self.geoconcepts=[]
if self.graph!=None:
print("WE HAVE A GRAPH")
results = self.graph.query(self.query)
for row in results:
self.geoconcepts.append(str(row[0]))
return True
def __init__(self, proxy):
self.register_in_service_locator()
self.db_config = self.get_component("db_config")
self.target = self.get_component("target")
self.transaction = self.get_component("transaction")
self.url_manager = self.get_component("url_manager")
self.error_handler = self.get_component("error_handler")
self.plugin_handler = self.get_component("plugin_handler")
self.timer = self.get_component("timer")
self.http_transaction = None
self.headers = {'User-Agent': self.db_config.get('USER_AGENT')}
self.req_count_refused = 0
self.req_count_total = 0
self.log_transactions = False
self.proxy = proxy
if proxy is None:
logging.debug(
"WARNING: No outbound proxy selected. It is recommended to "
"use an outbound proxy for tactical fuzzing later")
self.opener = build_opener(MyHTTPHandler, MyHTTPSHandler,
SmartRedirectHandler)
else: # All requests must use the outbound proxy.
logging.debug("Setting up proxy(inbound) for OWTF requests..")
ip, port = proxy
proxy_conf = {
'http': 'http://%s:%s' % (ip, port),
'https': 'http://%s:%s' % (ip, port)
}
proxy_handler = ProxyHandler(proxy_conf)
# FIXME: Works except no raw request on https.
self.opener = build_opener(proxy_handler, MyHTTPHandler,
MyHTTPSHandler, SmartRedirectHandler)
install_opener(self.opener)
def testTripleStoreConnection(
self, query="SELECT ?a ?b ?c WHERE { ?a ?b ?c .} LIMIT 1"):
if self.proxyHost != None and self.ProxyPort != None:
proxy = urllib.ProxyHandler({'http': proxyHost})
opener = urllib.build_opener(proxy)
urllib.install_opener(opener)
sparql = SPARQLWrapper(
self.triplestoreurl,
agent=
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11"
)
sparql.setQuery(query)
sparql.setReturnFormat(JSON)
print("now sending query")
try:
results = sparql.query().convert()
if self.testURL and not self.testConfiguration:
self.message = "URL depicts a valid SPARQL Endpoint!"
if "ASK" in query:
return results["boolean"]
self.feasibleConfiguration = True
return True
except:
self.message = "URL does not depict a valid SPARQL Endpoint!"
self.feasibleConfiguration = False
return False
def post_data(self, data, username, password):
utc_datetime = datetime.datetime.utcnow()
url_parameters = {
'cluster': self.elasticMonitoringCluster,
'index': self.elasticIndex,
'index_period': utc_datetime.strftime("%Y.%m.%d"),
}
url = "%(cluster)s/%(index)s-%(index_period)s/hrbdata" % url_parameters
headers = {'content-type': 'application/json'}
try:
req = Request(url)
req.add_header('Content-Type', 'application/json; charset=utf-8')
jsondata = json.dumps(data)
jsondataasbytes = jsondata.encode('utf-8')
req.add_header('Content-Length', len(jsondataasbytes))
data = urllib.parse.urlencode(data).encode("utf-8")
# req = urllib.request.urlopen(url, headers=headers, data=json.dumps(data))
if self.write_es_security_enable:
password_mgr = urllib.HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, url, username, password)
handler = urllib.HTTPBasicAuthHandler(password_mgr)
opener = urllib.build_opener(handler)
urllib.install_opener(opener)
response = urllib.request.urlopen(req, jsondataasbytes)
else:
response = urllib.request.urlopen(req, jsondataasbytes)
except Exception as e:
print("Error: {}".format(str(e)))
def auth_as_app(self) -> str:
logger.info("Getting access token for app")
self.request_time = datetime.datetime.now()
data = {'grant_type': 'client_credentials'}
pw_manager = urllib.HTTPPasswordMgrWithDefaultRealm()
pw_manager.add_password(None, constants.OAUTH_URL, self.app_id,
self.app_secret)
urllib.install_opener(
urllib.build_opener(urllib.HTTPBasicAuthHandler(pw_manager)))
request = urllib.Request(constants.OAUTH_URL,
data=data,
headers=constants.OAUTH_REQUEST_HEADERS)
try:
with urllib.request.urlopen(request) as response:
self.full_response = json.loads(response.read())
self.jwt = self.full_response["access_token"]
self.expires_in_secs = datetime.timedelta(
seconds=self.full_response["expires_in"])
self.expiry_time = self.request_time + self.expires_in_secs - constants.JWT_TOKEN_REFRESH_BUFFER
logger.log(
5,
"refreshed auth token at %s - token will expire in %s, with a buffer next refresh will be at %s",
self.request_time, self.expires_in_secs, self.expiry_time)
return self.full_response
except urllib.error.HTTPError as err:
logger.critical(
"Tried to authenticate an app, got error code %s with reason %s",
err.code, err.reason)
raise UnauthorizedRequestError()
def run(self):
QgsMessageLog.logMessage(
'Started task "{}"'.format(self.description()), MESSAGE_CATEGORY,
Qgis.Info)
if self.proxyHost != None and self.ProxyPort != None:
QgsMessageLog.logMessage('Proxy? ' + str(self.proxyHost),
MESSAGE_CATEGORY, Qgis.Info)
proxy = urllib.ProxyHandler({'http': proxyHost})
opener = urllib.build_opener(proxy)
urllib.install_opener(opener)
sparql = SPARQLWrapper(
self.triplestoreurl,
agent=
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11"
)
sparql.setQuery(self.query)
print("now sending query")
sparql.setReturnFormat(JSON)
results = sparql.query().convert()
for result in results["results"]["bindings"]:
self.viewlist.append(str(result[self.queryvar]["value"]))
print(self.viewlist)
#self.layercount.setText("["+str(len(viewlist))+"]")
if self.getlabels and "classlabelquery" in self.triplestoreconf and self.triplestoreconf[
"classlabelquery"] != "":
labels = self.getLabelsForClasses(
self.viewlist, self.triplestoreconf["classlabelquery"])
print(labels)
self.amountoflabels = len(labels)
i = 0
sorted_labels = sorted(labels.items(), key=lambda x: x[1])
for lab in sorted_labels:
self.resultlist.append(labels[lab[0]] + " (" + lab[0] + ")")
i = i + 1
return True
def crawler():
while not q.empty(): # 循环
path = q.get() #将line从队列 q 中取出来
url = "%s%s" % (domain_name, path) # 组合url地址，用于下一步提交
random_proxy = random.choice(proxy_list) # 随机使用一个代理服务器
proxy_support = urllib.ProxyHandler(random_proxy)
opener = urllib.build_opener(proxy_support)
urllib.install_opener(opener)
headers = {}
headers['User-Agent'] = Baidu_spider # 蜘蛛的头部信息
# 玩蛇网 www.iplaypython.com
request = urllib.Request(url, headers=headers)
try:
response = urllib.urlopen(request)
content = response.read()
if len(content): # 内容不为空的情况下返回状态码、路径
print("Status [%s] - path: %s" % (response.code, path))
response.close()
time.sleep(1) # 休息一会儿，防止速度过快连接数过大被封掉IP
except urllib.HTTPError as e:
# print e.code, path
pass # 异常处理，先暂时pass掉
def run(self):
QgsMessageLog.logMessage('Started task "{}"'.format(
self.description()),
MESSAGE_CATEGORY, Qgis.Info)
if self.proxyHost!=None and self.ProxyPort!=None:
QgsMessageLog.logMessage('Proxy? '+str(self.proxyHost), MESSAGE_CATEGORY, Qgis.Info)
proxy = urllib.ProxyHandler({'http': proxyHost})
opener = urllib.build_opener(proxy)
urllib.install_opener(opener)
sparql = SPARQLWrapper(self.triplestoreurl, agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11")
sparql.setQuery(self.query)
sparql.setMethod(POST)
sparql.setReturnFormat(JSON)
try:
results = sparql.query().convert()
except Exception as e:
try:
sparql = SPARQLWrapper(self.triplestoreurl, agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11")
sparql.setQuery(self.query)
sparql.setMethod(GET)
sparql.setReturnFormat(JSON)
results = sparql.query().convert()
except Exception as e:
self.exception=e
return
#print(results)
# geojson stuff
self.geojson=self.processResults(results,(self.triplestoreconf["crs"] if "crs" in self.triplestoreconf else ""),self.triplestoreconf["mandatoryvariables"][1:],self.allownongeo)
return True
def detectNamespaces(self, subpredobj):
if subpredobj < 0 or subpredobj == None:
query = "select distinct ?ns where { ?s ?p ?o . bind( replace( str(?s), \"(#|/)[^#/]*$\", \"$1\" ) as ?ns )} limit 10"
elif subpredobj == 0:
query = "select distinct ?ns where { ?s ?p ?o . bind( replace( str(?p), \"(#|/)[^#/]*$\", \"$1\" ) as ?ns )} limit 10"
else:
query = "select distinct ?ns where { ?s ?p ?o . bind( replace( str(?o), \"(#|/)[^#/]*$\", \"$1\" ) as ?ns )} limit 10"
if self.proxyHost != None and self.ProxyPort != None:
proxy = urllib.ProxyHandler({'http': proxyHost})
opener = urllib.build_opener(proxy)
urllib.install_opener(opener)
sparql = SPARQLWrapper(
self.triplestoreurl,
agent=
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11"
)
sparql.setQuery(query)
sparql.setReturnFormat(JSON)
print("now sending query")
try:
results = sparql.query().convert()
reslist = []
for nss in results["results"]["bindings"]:
if "ns" in nss:
reslist.append(nss["ns"]["value"])
return reslist
except:
return []
def fetion(msg):
cj = cookielib.LWPCookieJar()
opener = urllib.build_opener(urllib.HTTPCookieProcessor(cj))
urllib.install_opener(opener)
args = {'pass': password, 'm': user, 'loginstatus': loginstatus}
print('Logining...')
req = urllib.Request(url_login, urllib.urlencode(args))
jump = opener.open(req)
page = jump.read()
url = re.compile(r'<card id="start".*?ontimer="(.*?);').findall(page)[
0] #��ȡ��ת����
arg_t = re.compile(r't=(\d*)').findall(page)[0]
if url == '/im/login/login.action': #��¼ʧ��
print('Login Failed!')
input('Press any key to exit.')
return
else:
print('Login Successfully!')
sendmsg = urllib.Request(
url_msg, urllib.urlencode({'msg': msg.decode('gbk').encode('utf-8')}))
finish = urllib.urlopen(sendmsg)
if finish.geturl == 'http://f.10086.cn/im/user/sendMsgToMyself.action':
print('Send Failed!')
else:
print('Send Successfully')
logout = urllib.Request(url_logout + arg_t)
response = urllib.urlopen(logout) #ע��
print('Logout Successfully!')
def __init__(self,
column,
row,
triplestoreconf,
prefixes,
interlinkOrEnrich,
table,
propOrClass=False,
bothOptions=False,
currentprefixes=None,
addVocab=None):
super(QDialog, self).__init__()
self.setupUi(self)
self.currentcol = column
self.currentrow = row
self.table = table
self.prefixes = prefixes
self.currentprefixes = currentprefixes
self.bothOptions = bothOptions
self.triplestoreconf = triplestoreconf
self.interlinkOrEnrich = interlinkOrEnrich
self.addVocab = addVocab
if column != 4:
self.findConcept.setChecked(True)
if column == 4 or (not interlinkOrEnrich
and column != 4) or (not interlinkOrEnrich
and propOrClass):
self.findProperty.setChecked(True)
if not bothOptions:
self.findProperty.setEnabled(False)
self.findConcept.setEnabled(False)
self.tripleStoreEdit.setEnabled(False)
for triplestore in self.triplestoreconf:
if not "File" == triplestore["name"]:
self.tripleStoreEdit.addItem(triplestore["name"])
if addVocab != None:
for cov in addVocab:
self.tripleStoreEdit.addItem(addVocab[cov]["label"])
self.searchButton.clicked.connect(self.getClassesFromLabel)
urlregex = QRegExp(
"http[s]?://(?:[a-zA-Z#]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+"
)
urlvalidator = QRegExpValidator(urlregex, self)
self.costumproperty.setValidator(urlvalidator)
self.costumproperty.textChanged.connect(self.check_state3)
self.costumproperty.textChanged.emit(self.costumproperty.text())
self.costumpropertyButton.clicked.connect(self.applyConceptToColumn2)
self.applyButton.clicked.connect(self.applyConceptToColumn)
s = QSettings() #getting proxy from qgis options settings
self.proxyEnabled = s.value("proxy/proxyEnabled")
self.proxyType = s.value("proxy/proxyType")
self.proxyHost = s.value("proxy/proxyHost")
self.proxyPort = s.value("proxy/proxyPort")
self.proxyUser = s.value("proxy/proxyUser")
self.proxyPassword = s.value("proxy/proxyPassword")
if self.proxyHost != None and self.ProxyPort != None:
proxy = urllib.ProxyHandler({'http': proxyHost})
opener = urllib.build_opener(proxy)
urllib.install_opener(opener)
def getHtml(url):
USER_AGENTS = [
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10",
"Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/533.17.8 (KHTML, like Gecko) Version/5.0.1 Safari/533.17.8",
"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5",
"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.1.17) Gecko/20110123 (like Firefox/3.x) SeaMonkey/2.0.12",
"Mozilla/5.0 (Windows NT 5.2; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 SeaMonkey/2.7.1",
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/532.8 (KHTML, like Gecko) Chrome/4.0.302.2 Safari/532.8",
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.464.0 Safari/534.3",
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.15 Safari/534.13",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.54 Safari/535.2",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7",
"Mozilla/5.0 (Macintosh; U; Mac OS X Mach-O; en-US; rv:2.0a) Gecko/20040614 Firefox/3.0.0 ",
"Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10.5; en-US; rv:1.9.0.3) Gecko/2008092414 Firefox/3.0.3",
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-US; rv:1.9.1) Gecko/20090624 Firefox/3.5",
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2.14) Gecko/20110218 AlexaToolbar/alxf-2.0 Firefox/3.6.14",
"Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10.5; en-US; rv:1.9.2.15) Gecko/20110303 Firefox/3.6.15",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1"
]
# 代理的IP设置
proxies = ['114.215.95.188:3128', '218.14.115.211:3128']
req = urllib.Request(url)
# 设置消息头
req.add_header('User-Agent', random.choice(USER_AGENTS))
# 设置代理ip地址
proxy_support = urllib.ProxyHandler({"http": random.choice(proxies)})
opener = urllib.build_opener(proxy_support)
urllib.install_opener(opener)
# 访问并获取服务端返回的对象
times = 0
try:
res = urllib.urlopen(req)
html = res.read()
return html
except:
times += 1
getHtml(url)
def get_unread_msgs(user, passwd):
auth_handler = urllib.HTTPBasicAuthHandler()
auth_handler.add_password(realm='New mail feed',
uri='https://mail.google.com',
user='******'.format(user=user),
passwd=passwd)
opener = urllib.build_opener(auth_handler)
urllib.install_opener(opener)
feed = urllib.request.urlopen(FEED_URL)
return feed.read()
def handle_urlopen(self, urlData, username, password):
if self.read_es_security_enable:
password_mgr = urllib.HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, urlData, username, password)
handler = urllib.HTTPBasicAuthHandler(password_mgr)
opener = urllib.build_opener(handler)
urllib.install_opener(opener)
response = urllib.request.urlopen(urlData)
else:
response = urllib.request.urlopen(urlData)
return response.read().decode('utf-8')
def setupProxy():
proxies = {}
if _proxyhttp != None:
proxies['http'] = 'http://' + _proxyhttp
os.environ['http'] = _proxyhttp
if _proxyhttps != None:
proxies['https'] = _proxyhttps
os.environ['https'] = 'http://' + _proxyhttps
if proxies != {}:
proxy = urllib.ProxyHandler(proxies)
opener = urllib.build_opener(proxy)
urllib.install_opener(opener)
def __init__(self, ip_address, port, username="******", password="******"):
PowerDevice.__init__(self, ip_address, username, password)
self._ip_address = ip_address
self.port = port
# create a password manager
password_mgr = _urllib.HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, 'http://' + ip_address, username, password)
handler = _urllib.HTTPBasicAuthHandler(password_mgr)
opener = _urllib.build_opener(handler)
# Now all calls to _urllib.urlopen use our opener.
_urllib.install_opener(opener)
def setupProxy():
proxies = {}
if _proxyhttp != None:
proxies['http'] = 'http://' + _proxyhttp
os.environ['http'] = _proxyhttp
if _proxyhttps != None:
proxies['https'] = _proxyhttps
os.environ['https'] = 'http://' + _proxyhttps
if proxies != {}:
proxy = urllib.ProxyHandler(proxies)
opener = urllib.build_opener(proxy)
urllib.install_opener(opener)
def http_auth_request(url, host, user, passwd, user_agent=USER_AGENT):
"""Call an HTTP server with authorization credentials using urllib.
"""
if DEBUG: http.client.HTTPConnection.debuglevel = 1
# Hook up handler/opener to urllib
password_manager = urllib.HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(None, host, user, passwd)
auth_handler = urllib.HTTPBasicAuthHandler(password_manager)
opener = urllib.build_opener(auth_handler)
urllib.install_opener(opener)
return http_request(url, user_agent)
def save_cookie(self, fileName, url): #不能在有别的实例运行时执行
#声明一个MozillaCookieJar实例保存cookie
cookie = http.cookiejar.MozillaCookieJar(fileName)
#构建opener
opener = urllib.build_opener(urllib.HTTPCookieProcessor(cookie))
urllib.install_opener(opener)
request = urllib.Request.open(url)
request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36')
response = urllib.urlopen(request)
print (response.getcode())
cookie.save(ignore_discard=True, ignore_expires=True)
print ('Successfully saved')
def get_url(self, url, proxy_dict):
proxyIP = proxy_dict['ip']
proxyPort = proxy_dict['port']
proxyProtocol = proxy_dict['protocol']
proxy_handler = urllib.ProxyHandler({proxyProtocol: "{0}:{1}".format(proxyIP, proxyPort)})
opener_proxy = urllib.build_opener(proxy_handler)
urllib.install_opener(opener_proxy)
request = urllib.Request(url=url, headers=HEADERS)
response = urllib.urlopen(request)
html = response.read()
return html
def get_with_username_password(username, password):
# create a password manager
password_mgr = _urllib.HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, 'http://' + ip_address, username, password)
handler = _urllib.HTTPBasicAuthHandler(password_mgr)
opener = _urllib.build_opener(handler)
opener.open('http://' + ip_address)
_urllib.install_opener(opener)
request = _urllib.Request('http://' + ip_address)
response = opener.open(request)
data = response.read()
return data
def useProxy(self, proxy):
'''利用代理访问百度，并查找关键词'''
protocol = proxy.split('//')[0].split(":", '')
ip = proxy.split('//')[1]
opener = urllib.build_opener(urllib.ProxyHandler({protocol: ip}))
urllib.install_opener(opener)
try:
response = urllib.request.urlopen(self.url, timeout=self.timeout)
except:
print(u'连接错误，退出程序')
exit()
str = response.read()
if re.search(self.flagword, str):
print(u"已取得关键词，该代理可用")
else:
print("该代理不可用")
def SetProxiesIfNecessary():
global HTTP_PROXY
global HTTPS_PROXY
dProxies = {}
if HTTP_PROXY != '':
dProxies['http'] = HTTP_PROXY
if HTTPS_PROXY != '':
dProxies['https'] = HTTPS_PROXY
if os.getenv('http_proxy') != None:
dProxies['http'] = os.getenv('http_proxy')
if os.getenv('https_proxy') != None:
dProxies['https'] = os.getenv('https_proxy')
if dProxies != {}:
urllib.install_opener(
urllib.build_opener(urllib.ProxyHandler(dProxies)))
def httpConnection(url, proxy):
#TODO: habilitar autenticacion ntlm
if (proxy.auth == "ntlm"):
passman = urllib.HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, proxy.url, proxy.user, proxy.password)
auth = HTTPNtlmAuthHandler.HTTPNtlmAuthHandler(passman)
else:
passman = urllib.request.HTTPPasswordMgr()
passman.add_password(None, proxy.url, proxy.user, proxy.password)
auth = urllib.request.HTTPBasicAuthHandler(passman)
if (proxy.url):
proxy = urllib.ProxyHandler({'http': proxy.url})
opener = urllib.build_opener(proxy.url, auth, urllib2.HTTPHandler)
urllib.install_opener(opener)
return urllib.request.urlopen(url)
def __init__(self, username, passwd, debug=False):
(self.username, self.passwd) = (username, passwd)
self.cookie = cookielib.CookieJar()
cookieHandler = urllib.HTTPCookieProcessor(self.cookie)
self.is_debug = debug
if self.is_debug:
httpHandler = urllib.HTTPHandler(debuglevel=1)
httpsHandler = urllib.HTTPSHandler(debuglevel=1)
opener = urllib.build_opener(cookieHandler, httpHandler, httpsHandler)
else:
opener = urllib.build_opener(cookieHandler)
urllib.install_opener(opener)
self.last_url = ''
def spider_url_by_post(url, data, proxy=None, enable_proxy=False):
# make a string with 'POST'
method = 'POST'
# create a handler
proxy_handler = urllib2.ProxyHandler(proxy)
null_proxy_handler = urllib2.ProxyHandler({})
# create an openerdirector instance according to enable_proxy
if enable_proxy:
cookies = urllib2.HTTPCookieProcessor()
opener = urllib2.build_opener(cookies, proxy_handler, urllib2.HTTPHandler)
print 'without using proxy to crawl pages'
else :
opener = urllib2.build_opener(null_proxy_handler)
print 'using proxy to crawl pages'
# install opener
urllib.install_opener(opener)
# buidl a request
#data = urllib.urlencode(dictionary_of_POST_fiels_or_None)
data = urllib.urlencode(data)
request = urllib2.Request(url, data=data)
# Ramdom choose the user_agent
#user_agent = random.choice(USER_AGENT_LSIT)
user_agent = USER_AGENT_LIST[1]
request.add_header('User-Agent', user_agent)
request.get_method = lambda:method
try :
connection = opener.open(request, timeout=5)
if connection.code == 200:
html = connection.read()
return html
#else if connection.code == 403:
# return None
except urllib2.HTTPError, ex:
# print e.code, e.reason
print 'spider_url_by_get（） -------- ', str(ex)
# connection = ex
return None
def post(body):
'''
This function actually communicates with the FreshBooks API
'''
# setup HTTP basic authentication
password_mgr = urllib.HTTPPasswordMgrWithDefaultRealm()
url = ""
if account_url.find('//') == -1:
url = "https://"
url += account_url + SERVICE_URL
password_mgr.add_password(None, url, auth_token, '')
handler = urllib.HTTPBasicAuthHandler(password_mgr)
opener = urllib.build_opener(handler)
urllib.install_opener(opener)
# make the request and return the response body
request = urllib.Request(url, body, request_headers)
response = urllib.urlopen(request)
response_content = response.read()
return response_content
def run(self):
url = self._url
outputfile = self._target
def reporthook(blocknum, blocksize, totalsize):
readsofar = blocknum * blocksize
percent = 0
if totalsize > 0:
percent = readsofar * 1e2 / totalsize
s = "\r%5.1f%% %*d / %d" % (percent, len(
str(totalsize)), readsofar, totalsize)
sys.stderr.write(s)
if readsofar >= totalsize:
sys.stderr.write("\n")
else:
sys.stderr.write("read %d\n" % (readsofar, ))
self.signal.emit(int(percent))
proxy = urllib.ProxyHandler({'http': "myproxy"})
opener = urllib.build_opener(proxy)
urllib.install_opener(opener)
urllib.urlretrieve(url, outputfile, reporthook)
def CBDownload(env, target, url):
try:
import urllib # Python 3+
except ImportError:
import urllib2 as urllib
sys.stdout.write('Downloading ' + url + '.')
sys.stdout.flush()
ftp_proxy = os.getenv('ftp_proxy', None)
http_proxy = os.getenv('http_proxy', None)
if ftp_proxy or http_proxy:
handlers = {}
if ftp_proxy: handlers['ftp'] = ftp_proxy
if http_proxy: handlers['http'] = http_proxy
opener = urllib.build_opener(urllib.ProxyHandler(handlers))
urllib.install_opener(opener)
f = None
stream = None
try:
stream = urllib.urlopen(url)
f = open(target, 'wb', 0) # Unbuffered
while stream and f:
data = stream.read(1024 * 1024)
if not data: break
f.write(data)
sys.stdout.write('.')
sys.stdout.flush()
sys.stdout.write('ok\n')
sys.stdout.flush()
finally:
if f is not None: f.close()
if stream is not None: stream.close()
def CBDownload(env, target, url):
try:
import urllib # Python 3+
except ImportError:
import urllib2 as urllib
sys.stdout.write('Downloading ' + url + '.')
sys.stdout.flush()
ftp_proxy = os.getenv('ftp_proxy', None)
http_proxy = os.getenv('http_proxy', None)
if ftp_proxy or http_proxy:
handlers = {}
if ftp_proxy: handlers['ftp'] = ftp_proxy
if http_proxy: handlers['http'] = http_proxy
opener = urllib.build_opener(urllib.ProxyHandler(handlers))
urllib.install_opener(opener)
f = None
stream = None
try:
stream = urllib.urlopen(url)
f = open(target, 'wb', 0) # Unbuffered
while stream and f:
data = stream.read(1024 * 1024)
if not data: break
f.write(data)
sys.stdout.write('.')
sys.stdout.flush()
sys.stdout.write('ok\n')
sys.stdout.flush()
finally:
if f is not None: f.close()
if stream is not None: stream.close()
def download_from_url(url):
proxy = env_server.get_proxy()
if proxy['enabled']:
server = proxy['server'].replace('http://', '')
proxy_dict = {
'http': 'http://{login}:{pass}@{0}'.format(server, **proxy)
}
proxy_handler = urllib2.ProxyHandler(proxy_dict)
auth = urllib2.HTTPBasicAuthHandler()
opener = urllib2.build_opener(proxy_handler, auth, urllib2.HTTPHandler)
urllib2.install_opener(opener)
def url_open_agent(url=url, timeout=1):
return urllib2.urlopen(url=url, timeout=timeout)
query_worker = gf.get_thread_worker(url_open_agent,
error_func=gf.error_handle)
query_worker.try_start()
thread_pool = query_worker.get_thread_pool()
thread_pool.waitForDone()
if query_worker.is_failed():
return False
def httpConnection(url, proxy, agent, ignore):
if (proxy.auth == "ntlm"):
#todo
print("")
else:
auth_handler = urllib.request.HTTPBasicAuthHandler()
auth_handler.add_password(realm='Proxy',
uri=proxy.url,
user=proxy.user,
passwd=proxy.password)
if (proxy.url):
opener = urllib.request.build_opener(auth_handler)
urllib.install_opener(opener)
req = Request(url)
if len(agent) > 2:
req.add_header('user-agent', agent)
if (ignore):
return urlopen(req, context=ignore)
else:
return urlopen(req)
def run(self):
QgsMessageLog.logMessage('Started task "{}"'.format(
self.description()),
MESSAGE_CATEGORY, Qgis.Info)
if self.proxyHost!=None and self.ProxyPort!=None:
QgsMessageLog.logMessage('Proxy? '+str(self.proxyHost), MESSAGE_CATEGORY, Qgis.Info)
proxy = urllib.ProxyHandler({'http': proxyHost})
opener = urllib.build_opener(proxy)
urllib.install_opener(opener)
attlist={}
attlist[self.item]=[]
attlist[self.idfield]={}
for f in self.layer.getFeatures():
if self.item in f:
attlist[self.item].append(f[self.item])
attlist[self.idfield][f[self.idfield]]=True
query=""
if self.content=="Enrich URI":
query+="SELECT ?item WHERE {\n"
elif self.content=="Enrich Value" or self.strategy=="Enrich Both":
query+="SELECT ?item ?val ?valLabel ?vals WHERE {\n"
query+="VALUES ?vals { "
print(attlist)
for it in attlist[self.idfield]:
if str(it).startswith("http"):
query+="<"+str(it)+"> "
elif self.idprop=="http://www.w3.org/2000/01/rdf-schema#label" and self.language!=None and self.language!="":
query+="\""+str(it)+"\"@"+self.language+" "
else:
query+="\""+str(it)+"\" "
query+=" } . \n"
proppp=self.propertyy.data(1)
if self.propertyy.data(1).startswith("//"):
proppp="http:"+proppp
if self.table.item(self.row, 7).text()!="" and "wikidata" in self.triplestoreurl:
query+="?item wdt:P31 <"+self.table.item(self.row, 7).text()+"> .\n"
else:
query+="?item rdf:type <"+self.table.item(self.row, 7).text()+"> .\n"
query+="?item <"+self.idprop+"> ?vals .\n"
query+="?item <"+proppp+"> ?val . \n"
if (self.content=="Enrich Value" or self.content=="Enrich Both") and not "wikidata" in self.triplestoreurl:
query+="OPTIONAL{ ?val rdfs:label ?valLabel }"
elif (self.content=="Enrich Value" or self.content=="Enrich Both") and "wikidata" in self.triplestoreurl:
query+="SERVICE wikibase:label { bd:serviceParam wikibase:language \"[AUTO_LANGUAGE],"+self.language+"\". }\n"
query+="} "
QgsMessageLog.logMessage("proppp: "+str(proppp),
MESSAGE_CATEGORY, Qgis.Info)
QgsMessageLog.logMessage("idprop: "+self.idprop,
MESSAGE_CATEGORY, Qgis.Info)
QgsMessageLog.logMessage(query,
MESSAGE_CATEGORY, Qgis.Info)
QgsMessageLog.logMessage(self.triplestoreurl,
MESSAGE_CATEGORY, Qgis.Info)
print(self.triplestoreurl)
try:
sparql = SPARQLWrapper(self.triplestoreurl, agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11")
sparql.setQuery(query)
sparql.setMethod(POST)
print("now sending query")
sparql.setReturnFormat(JSON)
results = sparql.query().convert()
except Exception as e:
QgsMessageLog.logMessage("Trying GET query",
MESSAGE_CATEGORY, Qgis.Info)
try:
sparql = SPARQLWrapper(self.triplestoreurl, agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11")
sparql.setQuery(query)
sparql.setMethod(GET)
sparql.setReturnFormat(JSON)
results = sparql.query().convert()
except Exception as e:
#msgBox=QMessageBox()
#msgBox.setText("The following exception occurred: "+str(e))
#msgBox.exec()
return False
print(str(results))
#resultcounter=0
for resultcounter in results["results"]["bindings"]:
if self.content=="Enrich Value":
self.resultmap[resultcounter["vals"]["value"]]=resultcounter["valLabel"]["value"]
elif self.content=="Enrich URI":
self.resultmap[resultcounter["vals"]["value"]]=resultcounter["val"]["value"]
else:
self.resultmap[resultcounter["vals"]["value"]]=resultcounter["valLabel"]["value"]+";"+resultcounter["val"]["value"]
self.columntype=self.detectColumnType(self.resultmap,self.table)
QgsMessageLog.logMessage(str(self.columntype),
MESSAGE_CATEGORY, Qgis.Info)
QgsMessageLog.logMessage(str(self.resultmap),
MESSAGE_CATEGORY, Qgis.Info)
return True
def main():
opener = urllib.build_opener(urllib.HTTPCookieProcessor(cookielib.CookieJar()))
urllib.install_opener(opener)
if getUUID() == False:
print('获取uuid失败')
return
showQRImage()
time.sleep(1)
while waitForLogin() != '200':
pass
os.remove(QRImagePath)
if login() == False:
print('登录失败')
return
if webwxinit() == False:
print('初始化失败')
return
MemberList = webwxgetcontact()
MemberCount = len(MemberList)
print('通讯录共%s位好友') % MemberCount
ChatRoomName = ''
result = []
for i in range(0, int(math.ceil(MemberCount / float(MAX_GROUP_NUM)))):
UserNames = []
NickNames = []
DeletedList = ''
for j in range(0, MAX_GROUP_NUM):
if i * MAX_GROUP_NUM + j >= MemberCount:
break
Member = MemberList[i * MAX_GROUP_NUM + j]
UserNames.append(Member['UserName'])
NickNames.append(Member['NickName'].encode('utf-8'))
print('第%s组...') % (i + 1)
print(', '.join(NickNames))
print('回车键继续...')
input('say soemthing:')
# 新建群组/添加成员
if ChatRoomName == '':
(ChatRoomName, DeletedList) = createChatroom(UserNames)
else:
DeletedList = addMember(ChatRoomName, UserNames)
DeletedCount = len(DeletedList)
if DeletedCount > 0:
result += DeletedList
print('找到%s个被删好友') % DeletedCount
# raw_input()
# 删除成员
deleteMember(ChatRoomName, UserNames)
# todo 删除群组
resultNames = []
for Member in MemberList:
if Member['UserName'] in result:
NickName = Member['NickName']
if Member['RemarkName'] != '':
NickName += '(%s)' % Member['RemarkName']
resultNames.append(NickName.encode('utf-8'))
print('---------- 被删除的好友列表 ----------')
print('\n'.join(resultNames))
print('-----------------------------------')
import urllib
import time
username = "******"
password = "******"
def send_to_twitter(msg):
password_manager = urllib.HTTPPasswordMgr()
password_manager.add_password("Twitter API", "http://twitter.com/statuses", username, password)
http_handler = urllib.HTTPBasicAuthHandler(password_manager)
page_opener = urllib.build_opener(http_handler)
urllib.install_opener(page_opener)
params = urllib.parse.urlencode( {'status': msg})
resp = urllib.urlopen("http://twitter.com/statuses/update.json", params)
resp.read()
def get_price():
price = urllib.urlopen("http://")
text = page.read().decode("utf8")
where = text.find('>$')
start_of_price = text + 2
end_of_price = start_of_price +4
return(text[start_of_price:end_of_price])
get_price()
price_now = input("Do you need a price right now? (Y/N)")
if price_now == "Y":
send_to_twitter(get_price())
else:
price = 99.99
def OP_RETURN_bitcoin_cmd(command, testnet,
*args): # more params are read from here
if OP_RETURN_BITCOIN_USE_CMD:
sub_args = [OP_RETURN_BITCOIN_PATH]
if testnet:
sub_args.append('-testnet')
sub_args.append(command)
for arg in args:
sub_args.append(
json.dumps(arg) if isinstance(arg, (dict, list,
tuple)) else str(arg))
raw_result = subprocess.check_output(sub_args).decode("utf-8").rstrip(
"\n")
try: # decode JSON if possible
result = json.loads(raw_result)
except ValueError:
result = raw_result
else:
request = {
'id': str(time.time()) + '-' + str(random.randint(100000, 999999)),
'method': command,
'params': args,
}
port = OP_RETURN_BITCOIN_PORT
user = OP_RETURN_BITCOIN_USER
password = OP_RETURN_BITCOIN_PASSWORD
if not (len(port) and len(user) and len(password)):
conf_lines = open(os.path.expanduser('~') +
'/.zen/zen.conf').readlines()
for conf_line in conf_lines:
parts = conf_line.strip().split('=', 1) # up to 2 parts
if (parts[0] == 'rpcport') and not len(port):
port = int(parts[1])
if (parts[0] == 'rpcuser') and not len(user):
user = parts[1]
if (parts[0] == 'rpcpassword') and not len(password):
password = parts[1]
if not len(port):
port = 18231 if testnet else 8231
if not (len(user) and len(password)):
return None # no point trying in this case
url = 'http://' + OP_RETURN_BITCOIN_IP + ':' + str(port) + '/'
try:
from urllib import HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, build_opener, install_opener, urlopen
except ImportError:
from urllib.request import HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, build_opener, install_opener, urlopen
passman = HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, url, user, password)
auth_handler = HTTPBasicAuthHandler(passman)
opener = build_opener(auth_handler)
install_opener(opener)
raw_result = urlopen(url,
json.dumps(request).encode('utf-8'),
OP_RETURN_NET_TIMEOUT).read()
result_array = json.loads(raw_result.decode('utf-8'))
result = result_array['result']
return result
