def execute(self, method, *args, **kwargs):
header = {
'Content-Type' : 'application/json',
'User-Agent' : 'python-xbmc'
}
# Params are given as a dictionnary
if len(args) == 1:
args=args[0]
params = kwargs
# Use kwargs for param=value style
else:
args = kwargs
params={}
params['jsonrpc']='2.0'
params['id']=self.id
self.id +=1
params['method']=method
params['params']=args
values=json.dumps(params)
# HTTP Authentication
password_mgr = HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, self.url, self.username, self.password)
auth_handler = HTTPBasicAuthHandler(password_mgr)
opener = build_opener(auth_handler)
install_opener(opener)
data = values
req = Request(self.url, data.encode('utf-8'), header)
response = urlopen(req)
the_page = response.read()
if len(the_page) > 0 :
return json.load(StringIO(the_page.decode('utf-8')))
else:
return None # for readability
def download(self, source, dest):
"""
Download an archive file.
:param str source: URL pointing to an archive file.
:param str dest: Local path location to download archive file to.
"""
# propogate all exceptions
# URLError, OSError, etc
proto, netloc, path, params, query, fragment = urlparse(source)
if proto in ('http', 'https'):
auth, barehost = splituser(netloc)
if auth is not None:
source = urlunparse((proto, barehost, path, params, query, fragment))
username, password = splitpasswd(auth)
passman = HTTPPasswordMgrWithDefaultRealm()
# Realm is set to None in add_password to force the username and password
# to be used whatever the realm
passman.add_password(None, source, username, password)
authhandler = HTTPBasicAuthHandler(passman)
opener = build_opener(authhandler)
install_opener(opener)
response = urlopen(source)
try:
with open(dest, 'w') as dest_file:
dest_file.write(response.read())
except Exception as e:
if os.path.isfile(dest):
os.unlink(dest)
raise e
def __init__(
self,
cachedir="/tmp",
api_host_options={},
urllist=[],
http_debug=False,
cookiejar=None,
offline=False,
enable_cpio=True,
):
# set up progress bar callback
if sys.stdout.isatty() and TextMeter:
self.progress_obj = TextMeter(fo=sys.stdout)
else:
self.progress_obj = None
self.cachedir = cachedir
self.urllist = urllist
self.http_debug = http_debug
self.offline = offline
self.cpio = {}
self.enable_cpio = enable_cpio
passmgr = HTTPPasswordMgrWithDefaultRealm()
for host in api_host_options:
passmgr.add_password(None, host, api_host_options[host]["user"], api_host_options[host]["pass"])
openers = (HTTPBasicAuthHandler(passmgr),)
if cookiejar:
openers += (HTTPCookieProcessor(cookiejar),)
self.gr = OscFileGrabber(progress_obj=self.progress_obj)
def urlopen(url, headers=None, data=None, timeout=None):
"""
An URL opener with the User-agent set to gPodder (with version)
"""
username, password = username_password_from_url(url)
if username is not None or password is not None:
url = url_strip_authentication(url)
password_mgr = HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, url, username, password)
handler = HTTPBasicAuthHandler(password_mgr)
opener = build_opener(handler)
else:
opener = build_opener()
if headers is None:
headers = {}
else:
headers = dict(headers)
headers.update({'User-agent': USER_AGENT})
request = Request(url, data=data, headers=headers)
if timeout is None:
return opener.open(request)
else:
return opener.open(request, timeout=timeout)
def __init__(self, host, port, username, password):
self.url = "http://%s:%s" % (host, port)
pwdmgr = HTTPPasswordMgrWithDefaultRealm()
pwdmgr.add_password(None, self.url, username, password)
pwdhandler = HTTPBasicAuthHandler(pwdmgr)
self.opener = build_opener(pwdhandler)
def openURL(url_base, data, method='Get', cookies=None, username=None, password=None):
''' function to open urls - wrapper around urllib2.urlopen but with additional checks for OGC service exceptions and url formatting, also handles cookies and simple user password authentication'''
url_base.strip()
lastchar = url_base[-1]
if lastchar not in ['?', '&']:
if url_base.find('?') == -1:
url_base = url_base + '?'
else:
url_base = url_base + '&'
if username and password:
# Provide login information in order to use the WMS server
# Create an OpenerDirector with support for Basic HTTP
# Authentication...
passman = HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, url_base, username, password)
auth_handler = HTTPBasicAuthHandler(passman)
opener = urllib.request.build_opener(auth_handler)
openit = opener.open
else:
# NOTE: optionally set debuglevel>0 to debug HTTP connection
#opener = urllib2.build_opener(urllib2.HTTPHandler(debuglevel=0))
#openit = opener.open
openit = urlopen
try:
if method == 'Post':
req = Request(url_base, data)
# set appropriate header if posting XML
try:
xml = etree.fromstring(data)
req.add_header('Content-Type', "text/xml")
except:
pass
else:
req=Request(url_base + data)
if cookies is not None:
req.add_header('Cookie', cookies)
u = openit(req)
except HTTPError as e: #Some servers may set the http header to 400 if returning an OGC service exception or 401 if unauthorised.
if e.code in [400, 401]:
raise ServiceException(e.read())
else:
raise e
# check for service exceptions without the http header set
if u.info()['Content-Type'] in ['text/xml', 'application/xml']:
#just in case 400 headers were not set, going to have to read the xml to see if it's an exception report.
#wrap the url stram in a extended StringIO object so it's re-readable
u=RereadableURL(u)
se_xml= u.read()
se_tree = etree.fromstring(se_xml)
serviceException=se_tree.find('{http://www.opengis.net/ows}Exception')
if serviceException is None:
serviceException=se_tree.find('ServiceException')
if serviceException is not None:
raise ServiceException(str(serviceException.text).strip())
u.seek(0) #return cursor to start of u
return u
def create_opener(aur_server_tag: str) -> OpenerDirector:
server = _aur_server(aur_server_tag)
password_manager = HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(realm=None,
uri=server.address,
user=server.user,
passwd=server.password)
handler = HTTPBasicAuthHandler(password_manager)
return build_opener(handler)
def init_opener(self):
""" chinachuのAPIを実行するためのopenerを初期化する """
pm = HTTPPasswordMgrWithDefaultRealm()
url = self._config["chinachu"]["apiEndpoint"]
user = self._config["chinachu"]["username"]
password = self._config["chinachu"]["password"]
pm.add_password(None, url, user, password)
handler = HTTPBasicAuthHandler(pm)
self._opener = build_opener(handler)
def auth(cls, username, password, uri, realm=None, timeout=None):
'''Create an httplib1 instance witn a basic authentication handler.
The authentication'''
if realm is None:
password_mgr = HTTPPasswordMgrWithDefaultRealm()
else:
password_mgr = HTTPPasswordMgr()
password_mgr.add_password(realm, uri, user, passwd)
opener = HTTPBasicAuthHandler(password_mgr)
return cls(opener,timeout)
def authorize():
mal_config = read_mal_config()
pass_manager = HTTPPasswordMgrWithDefaultRealm()
pass_manager.add_password(None, 'http://myanimelist.net/api',
mal_config['UserName'], mal_config['Password'])
auth_handler = HTTPBasicAuthHandler(pass_manager)
opener = build_opener(auth_handler)
install_opener(opener)
def index(username, password):
global url
p = HTTPPasswordMgrWithDefaultRealm()
p.add_password(None, url, username, password)
auth_handler = HTTPBasicAuthHandler(p)
opener = build_opener(auth_handler)
try:
result = opener.open(url)
html = result.read().decode('utf-8')
print(html)
except URLError as e:
print(e.reason)
def get_web_page(url, username=None, password=None, login_url=None):
"""Get url page possible with username and password.
"""
if login_url:
# Login via a form
cookies = HTTPCookieProcessor()
opener = build_opener(cookies)
install_opener(opener)
opener.open(login_url)
try:
token = [
x.value for x in cookies.cookiejar if x.name == 'csrftoken'][0]
except IndexError:
return False, "no csrftoken"
params = dict(username=username, password=password,
this_is_the_login_form=True,
csrfmiddlewaretoken=token,
)
encoded_params = urlencode(params)
with contextlib.closing(opener.open(login_url, encoded_params)) as f:
f.read()
elif username is not None:
# Login using basic auth
# Create password manager
passman = HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, url, username, password)
# create the handler
authhandler = HTTPBasicAuthHandler(passman)
opener = build_opener(authhandler)
install_opener(opener)
try:
pagehandle = urlopen(url)
except HTTPError as e:
msg = ('The server couldn\'t fulfill the request. '
'Error code: %s' % e.status_code)
e.args = (msg,)
raise
except URLError as e:
msg = 'Could not open URL "%s": %s' % (url, e)
e.args = (msg,)
raise
else:
page = pagehandle.read()
return page
def url_get(url, user, password):
top_level_url = url.rsplit('/', 1)
password_mgr = HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, top_level_url, user, password)
handler = HTTPBasicAuthHandler(password_mgr)
opener = build_opener(handler)
opener.open(url)
install_opener(opener)
return urlopen(url)
class HttpAuthenticated(HttpTransport):
"""
Provides basic http authentication that follows the RFC-2617 specification.
As defined by specifications, credentials are provided to the server
upon request (HTTP/1.0 401 Authorization Required) by the server only.
@ivar pm: The password manager.
@ivar handler: The authentication handler.
"""
def __init__(self, **kwargs):
"""
@param kwargs: Keyword arguments.
- B{proxy} - An http proxy to be specified on requests.
The proxy is defined as {protocol:proxy,}
- type: I{dict}
- default: {}
- B{timeout} - Set the url open timeout (seconds).
- type: I{float}
- default: 90
- B{username} - The username used for http authentication.
- type: I{str}
- default: None
- B{password} - The password used for http authentication.
- type: I{str}
- default: None
"""
HttpTransport.__init__(self, **kwargs)
self.pm = HTTPPasswordMgrWithDefaultRealm()
def open(self, request):
self.addcredentials(request)
return HttpTransport.open(self, request)
def send(self, request):
self.addcredentials(request)
return HttpTransport.send(self, request)
def addcredentials(self, request):
credentials = self.credentials()
if not (None in credentials):
u = credentials[0]
p = credentials[1]
self.pm.add_password(None, request.url, u, p)
def credentials(self):
return (self.options.username, self.options.password)
def u2handlers(self):
handlers = HttpTransport.u2handlers(self)
handlers.append(HTTPBasicAuthHandler(self.pm))
return handlers
class HttpAuthenticated(HttpTransport):
"""
Provides basic http authentication that follows the RFC-2617 specification.
As defined by specifications, credentials are provided to the server
upon request (HTTP/1.0 401 Authorization Required) by the server only.
@ivar pm: The password manager.
@ivar handler: The authentication handler.
"""
def __init__(self, **kwargs):
"""
@param kwargs: Keyword arguments.
- B{proxy} - An http proxy to be specified on requests.
The proxy is defined as {protocol:proxy,}
- type: I{dict}
- default: {}
- B{timeout} - Set the url open timeout (seconds).
- type: I{float}
- default: 90
- B{username} - The username used for http authentication.
- type: I{str}
- default: None
- B{password} - The password used for http authentication.
- type: I{str}
- default: None
"""
HttpTransport.__init__(self, **kwargs)
self.pm = HTTPPasswordMgrWithDefaultRealm()
def open(self, request):
self.addcredentials(request)
return HttpTransport.open(self, request)
def send(self, request):
self.addcredentials(request)
return HttpTransport.send(self, request)
def addcredentials(self, request):
credentials = self.credentials()
if not (None in credentials):
u = credentials[0]
p = credentials[1]
self.pm.add_password(None, request.url, u, p)
def credentials(self):
return (self.options.username, self.options.password)
def u2handlers(self):
handlers = HttpTransport.u2handlers(self)
handlers.append(HTTPBasicAuthHandler(self.pm))
return handlers
def authenticated_urlopen(location):
""" A wrapper around urlopen adding authentication information if provided by the user. """
passman = HTTPPasswordMgrWithDefaultRealm()
server_name = urlparse.urlsplit(location).netloc
access = get_server_access(server_name)
if access is not None:
user = access.username
password = access.password
if user is not None and password is not None:
passman.add_password(None, location, user, password)
authhandler = HTTPBasicAuthHandler(passman)
opener = build_opener(authhandler)
install_opener(opener)
return urlopen(location)
def authenticated_urlopen(location):
""" A wrapper around urlopen adding authentication information if provided by the user. """
passman = HTTPPasswordMgrWithDefaultRealm()
server_name = urlparse.urlsplit(location).netloc
access = get_server_access(server_name)
if access is not None:
user = access.username
password = access.password
if user is not None and password is not None:
passman.add_password(None, location, user, password)
authhandler = HTTPBasicAuthHandler(passman)
opener = build_opener(authhandler)
install_opener(opener)
return urlopen(location)
def get_port_info_from_url(port):
url = 'http://192.168.38.128:8181/restconf/operational/opendaylight-inventory:nodes/node/' + str(
port.node) + '/node-connector/' + str(port) + '/'
username = '******'
password = '******'
p = HTTPPasswordMgrWithDefaultRealm()
p.add_password(None, url, username, password)
handler = HTTPBasicAuthHandler(p)
opener = build_opener(handler)
install_opener(opener)
content = urlopen(url).read()
#print(content)
data = content.decode("utf-8")
print(data)
filename = 'port_info.json'
with open(filename, 'w') as outfile:
json.dump(data, outfile)
with fileinput.FileInput(filename, inplace=True) as file:
for line in file:
print(line.replace("\\", "").replace("\"{",
"{").replace("}\"", "}"),
end='')
client = MongoClient('localhost', 27017)
db = client.nets
db.port_info.remove({})
mongoimport('localhost', 27017, 'nets', 'port_info', 'port_info.json')
datos = {}
link_down = query_get_link_down_port_info()
datos["link-down"] = link_down
blocked = query_get_blocked_port_info()
datos["blocked"] = blocked
tbytes = query_get_bytes_transmitted_port_info()
datos["tbytes"] = tbytes
rbytes = query_get_bytes_received_port_info()
datos["rbytes"] = rbytes
drops = query_get_receive_drops_port_info()
datos["drops"] = drops
print(datos)
return datos
def init_auth(user, password, url, use_ssl=False):
""" Init authentication """
pass_mgr = HTTPPasswordMgrWithDefaultRealm()
pass_mgr.add_password(None, url, user, password)
auth_handler = HTTPBasicAuthHandler(pass_mgr)
if use_ssl:
import ssl
from urllib.request import HTTPSHandler
opener = build_opener(auth_handler, HTTPSHandler(context=ssl._create_unverified_context()))
else:
opener = build_opener(auth_handler)
install_opener(opener)
def __init__(self, device_data):
"""Set up how we are going to connect to the ICS Calendar"""
self.name = device_data[CONF_NAME]
self.url = device_data[CONF_URL]
self.include_all_day = device_data[CONF_INCLUDE_ALL_DAY]
self.event = None
if device_data[CONF_USERNAME] != '' \
and device_data[CONF_PASSWORD] != '':
passman = HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, self.url, device_data[CONF_USERNAME], device_data[CONF_PASSWORD])
basicAuthHandler = HTTPBasicAuthHandler(passman)
digestAuthHandler = HTTPDigestAuthHandler(passman)
opener = build_opener(digestAuthHandler, basicAuthHandler)
install_opener(opener)
def get_opener(self):
# The opener is yet build ?
if self.opener is not None:
return self.opener
# Build a new opener
opener = build_opener()
headers = [ ('User-agent', 'restedit/%s' % __version__) ]
# Add the "includes"
for include in self.includes:
headers.append(include)
# An authentication ?
auth_header = self.metadata.get('auth')
if auth_header is not None:
if auth_header.lower().startswith('basic'):
cls_handler = HTTPBasicAuthHandler
chal = auth_header[6:].strip()
# Automatically find the username and the password
username, password = decode_base64(chal).split(':', 1)
elif auth_header.lower().startswith('digest'):
cls_handler = HTTPDigestAuthHandler
# Automatically find the username, but we must ask the password
# XXX undocumented functions
chal = parse_keqv_list(parse_http_list(auth_header[7:]))
username = chal['username']
password = askPassword(chal['realm'], username)
else:
raise NotImplemented
password_mgr = HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(realm=None,
uri=self.url,
user=username,
passwd=password)
auth_handler = cls_handler(password_mgr)
opener.add_handler(auth_handler)
# A cookie ?
if self.metadata.get('cookie'):
headers.append( ('Cookie', self.metadata['cookie']) )
# All OK
opener.addheaders = headers
self.opener = opener
return opener
def testPassWord():
username = '******'
password = '******'
url = 'http://localhost:5000/'
p = HTTPPasswordMgrWithDefaultRealm()
p.add_password(None, url, username, password)
auth_handler = HTTPBasicAuthHandler(p)
opener = build_opener(auth_handler)
try:
result = opener.open(url)
html = result.read().decode('utf-8')
print(html)
except URLError as e:
print(e.reason)
def __init__(self, base_url, user=None, password=None, log_function=None):
"""
Args:
base_url (str): the URL displayed in the Sweech app (ex: 'http://192.168.0.65:4444')
user (str, optional): the username if login is enabled in the app
password (str, optional): the password if login is enabled in the app
log_function (function): a function to be called to log runtime information
"""
self.base_url = base_url
self._log_function = log_function
passwordmgr = HTTPPasswordMgrWithDefaultRealm()
passwordmgr.add_password('Sweech', base_url, user, password)
auth_handler = HTTPDigestAuthHandler(passwordmgr)
https_auth_handler = HTTPSDigestAuthHandler(
passwordmgr, ssl.SSLContext(ssl.PROTOCOL_SSLv23))
self._opener = build_opener(auth_handler, https_auth_handler)
def request_http_basic_handler():
username = '******'
password = '******'
url = 'https://www.douban.com/'
p = HTTPPasswordMgrWithDefaultRealm()
p.add_password(None, url, username, password)
auth_handler = HTTPBasicAuthHandler(p)
opener = build_opener(auth_handler)
try:
result = opener.open(url)
html = result.read().decode('utf-8')
print(html)
except URLError as e:
print(e.reason)
def run_query(search_terms):
root_url = "https://api.datamarket.azure.com/Bing/Search/"
source = "Web"
results_per_page = 10
offset = 0
# Wrap quotes around our query terms as required by the Bing API.
# The query we will then use is stored within variable query.
query = "'{0}'".format(search_terms)
query = quote(query)
# Construct the latter part of our request's URL.
# Sets the format of the response to JSON and sets other properties.
search_url = "{0}{1}?$format=json&$top={2}&$skip={3}&Query={4}".format(
root_url, source, results_per_page, offset, query
)
# Setup authentication with the Bing servers.
# The username MUST be a blank string, and put in your API key!
username = ""
# Create a 'password manager' which handles authentication for us.
password_mgr = HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, search_url, username, BING_API_KEY)
# Create our results list which we'll populate.
results = []
print(search_url)
try:
# Prepare for connecting to Bing's servers.
handler = HTTPBasicAuthHandler(password_mgr)
opener = build_opener(handler)
install_opener(opener)
# Connect to the server and read the response generated.
response = urlopen(search_url).read().decode(encoding="utf-8").replace("<", "")
# print(response)
json_response = json.loads(response, parse_int=True)
# Loop through each page returned, populating out results list.
for result in json_response["d"]["results"]:
results.append({"title": result["Title"], "link": result["Url"], "summary": result["Description"]})
except URLError as e:
print("Error when querying the Bing API: ", e)
return results
def get_opener(url, user, password):
'''Returns an opener to be used for downloading data with a given user and password.
All arguments should be strings.
:param url: the domain name of the given url
:param: string, the user name
:param password: the password
:return: an urllib opener
'''
parsed_url = urlparse(url)
base_url = "%s://%s" % (parsed_url.scheme, parsed_url.netloc)
handlers = []
password_mgr = HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, base_url, user, password)
handlers.append(HTTPDigestAuthHandler(password_mgr))
return build_opener(*handlers)
def push_to_web_interface(self, json_body_dict):
destination_url = urljoin(self.notification_server_host, URL_PUSH_PATH)
data = json.dumps(json_body_dict).encode("utf-8")
passman = HTTPPasswordMgrWithDefaultRealm()
un = self.notification_server_username
pw = self.notification_server_password
top_level_url = self.notification_server_host
passman.add_password(None, top_level_url, un, pw)
auth_handler = HTTPBasicAuthHandler(passman)
opener = build_opener(auth_handler)
opener.open(top_level_url)
install_opener(opener)
request = Request(destination_url, data=data, headers={'Content-Type': 'application/json',
'User-Agent': self.user_agent})
opener.open(request)
def curl(url, params=None, auth=None, req_type="GET", data=None, headers=None):
post_req = ["POST", "PUT"]
get_req = ["GET", "DELETE"]
if params is not None:
url += "?" + urlencode(params)
if req_type not in post_req + get_req:
raise IOError("Wrong request type \"%s\" passed" % req_type)
_headers = {}
handler_chain = []
if auth is not None:
manager = HTTPPasswordMgrWithDefaultRealm()
manager.add_password(None, url, auth["user"], auth["pass"])
handler_chain.append(HTTPBasicAuthHandler(manager))
if req_type in post_req and data is not None:
_headers["Content-Length"] = len(data)
if headers is not None:
_headers.update(headers)
director = build_opener(*handler_chain)
if req_type in post_req:
if sys.version_info.major == 3:
_data = bytes(data, encoding='utf8')
else:
_data = bytes(data)
req = Request(url, headers=_headers, data=_data)
else:
req = Request(url, headers=_headers)
req.get_method = lambda: req_type
result = director.open(req)
return {
"httpcode": result.code,
"headers": result.info(),
"content": result.read()
}
def set_user_name_password(self, user_name: str, password: str):
"""Set a user name and password to use.
The user name and password will be set into an HTTPBasicAuthHandler an
an HTTPDigestAuthHandler. Both are attached to a new urlopener, so
that HTTP Basic Auth and HTTP Digest Auth will be supported when
opening the URL.
:param user_name: The user name
:type user_name: str
:param password: The password
:type password: str
"""
passman = HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, self.url, user_name, password)
basic_auth_handler = HTTPBasicAuthHandler(passman)
digest_auth_handler = HTTPDigestAuthHandler(passman)
opener = build_opener(digest_auth_handler, basic_auth_handler)
install_opener(opener)
def get_host_info_from_url(host):
url = 'http://192.168.38.128:8181/restconf/operational/network-topology:network-topology/topology/flow:1/node/' + str(
host) + '/'
username = '******'
password = '******'
p = HTTPPasswordMgrWithDefaultRealm()
p.add_password(None, url, username, password)
handler = HTTPBasicAuthHandler(p)
opener = build_opener(handler)
install_opener(opener)
content = urlopen(url).read()
#print(content)
data = content.decode("utf-8")
print(data)
filename = 'host_info.json'
with open(filename, 'w') as outfile:
json.dump(data, outfile)
with fileinput.FileInput(filename, inplace=True) as file:
for line in file:
print(line.replace("\\", "").replace("\"{",
"{").replace("}\"", "}"),
end='')
client = MongoClient('localhost', 27017)
db = client.nets
db.port_info.remove({})
mongoimport('localhost', 27017, 'nets', 'host_info', 'host_info.json')
datos = {}
mac = query_get_mac_host_info()
datos["mac"] = mac
ip = query_get_ip_host_info()
datos["ip"] = ip
print(datos)
return datos
def auth():
"""
用于处理需要验证账号密码的情况
"""
username = '******'
password = '******'
url = 'http://localhost:5000/'
p = HTTPPasswordMgrWithDefaultRealm()
# 添加需要认证的账号密码
p.add_password(None, url, username, password)
# 构建BasicAuthHandler然后构建opener
auth_handler = HTTPBasicAuthHandler(p)
opener = build_opener(auth_handler)
try:
response = opener.open(url)
print(response.read().decode('utf-8'))
except urllib.request.URLError as e:
print(e.reason)
def get_user_avatar(user_id):
"""
Downloads from Artella the avatar of the given user id
Only works if the user is loaded before to Artella
:param user_id: str
:return:
"""
project_type = artellapipe.project.get_project_type()
artella_cms_url = artella_lib.config.get('server',
project_type).get('cms_url')
manager = HTTPPasswordMgrWithDefaultRealm()
manager.add_password(None, artella_cms_url, 'default', 'default')
auth = HTTPBasicAuthHandler(manager)
opener = build_opener(auth)
install_opener(opener)
response = urlopen('{0}/profile/{1}/avatarfull.img'.format(
artella_cms_url, user_id))
return response
def request_demo4():
# 1、请求验证
username = '******'
password = '******'
url = 'http://localhost:5000/'
p = HTTPPasswordMgrWithDefaultRealm()
p.add_password(None, url, username, password)
auth_handler = HTTPBasicAuthHandler(p)
opener = build_opener(auth_handler)
try:
result = opener.open(url)
html = result.read().decode('utf-8')
print(html)
except URLError as e:
print(e.reason)
# 2、代理：
from urllib.request import ProxyHandler
proxy_handler = ProxyHandler({
'http': 'http://127.0.0.1:9743',
'https': 'https://127.0.0.1:9743'
})
opener = build_opener(proxy_handler)
try:
response = opener.open('https://www.baidu.com')
print(response.read().decode('utf-8'))
except URLError as e:
print(e.reason)
# 3、cookie
import http.cookiejar
cookie = http.cookiejar.CookieJar()
handler = request.HTTPCookieProcessor(cookie)
opener = build_opener(handler)
response = opener.open('http://www.baidu.com')
for item in cookie:
print(item.name + "=" + item.value)
def urlopener_with_auth(url):
"""
Given a URL, return the URL with auth stripped, and a urlopener that can
open it.
Uses urllib2, so SSL certs aren't verified.
"""
opener = build_opener()
parsed = urlparse(url)
if parsed.username and parsed.password:
host = parsed.hostname
if parsed.port:
host += ':%i' % parsed.port
stripped_auth = (parsed[0], host) + parsed[2:]
url = urlunparse(stripped_auth)
passwd_mgr = HTTPPasswordMgrWithDefaultRealm()
base_url = urlunparse((parsed[0], host, '', '', '', ''))
passwd_mgr.add_password(realm=None, uri=base_url,
user=parsed.username, passwd=parsed.password)
auth_handler = HTTPBasicAuthHandler(passwd_mgr)
opener = build_opener(auth_handler)
return url, opener
def query(self, query, ts_start, ts_end):
# target = 'summarize({},"{}","avg")'.format(
# query, '99year') @TODO remove if not needed
# build graphite url
args = {
'__auth_token': self.token,
'target': query,
'format': 'json',
'from': ts_start,
'until': ts_end,
}
url = '{}/render?'.format(self.url)
for k, v in args.iteritems():
url += '{}={}&'.format(quote(k), quote(v))
logger.debug('Query URL is {}'.format(url))
# Basic auth header
password_mgr = HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(
None,
self.graphite_url,
self.username,
self.password,
)
auth_handler = HTTPBasicAuthHandler(password_mgr)
# Ignore ssl cert check
ctx = create_default_context()
ctx.check_hostname = False
ctx.verify_mode = CERT_NONE
ssl_handler = HTTPSHandler(context=ctx)
opener = build_opener(ssl_handler, auth_handler)
install_opener(opener)
result = json.loads(urlopen(url).read())
return result
def __init__(self, cachedir = '/tmp', api_host_options = {}, urllist = [], http_debug = False,
cookiejar = None, offline = False, enable_cpio = True):
# set up progress bar callback
if sys.stdout.isatty() and TextMeter:
self.progress_obj = TextMeter(fo=sys.stdout)
else:
self.progress_obj = None
self.cachedir = cachedir
self.urllist = urllist
self.http_debug = http_debug
self.offline = offline
self.cpio = {}
self.enable_cpio = enable_cpio
passmgr = HTTPPasswordMgrWithDefaultRealm()
for host in api_host_options.keys():
passmgr.add_password(None, host, api_host_options[host]['user'], api_host_options[host]['pass'])
openers = (HTTPBasicAuthHandler(passmgr), )
if cookiejar:
openers += (HTTPCookieProcessor(cookiejar), )
self.gr = OscFileGrabber(progress_obj=self.progress_obj)
def _netrc_open(uri, filename=None):
'''
open uri using netrc credentials.
:param uri: uri to open
:param filename: optional, path to non-default netrc config file
:returns: file-like object from opening a socket to uri, or None
:raises IOError: if opening .netrc file fails (unless file not found)
'''
if not uri:
return None
parsed_uri = urlparse(uri)
machine = parsed_uri.netloc
if not machine:
return None
opener = None
try:
info = netrc.netrc(filename).authenticators(machine)
if info is not None:
(username, _ , password) = info
if username and password:
pass_man = HTTPPasswordMgrWithDefaultRealm()
pass_man.add_password(None, machine, username, password)
authhandler = HTTPBasicAuthHandler(pass_man)
opener = build_opener(authhandler)
return opener.open(uri)
else:
# caught below, like other netrc parse errors
raise netrc.NetrcParseError('No authenticators for "%s"' % machine)
except IOError as ioe:
if ioe.errno != 2:
# if = 2, User probably has no .netrc, this is not an error
raise
except netrc.NetrcParseError as neterr:
logger = logging.getLogger('vcstools')
logger.warn('WARNING: parsing .netrc: %s' % str(neterr))
# we could install_opener() here, but prefer to keep
# default opening clean. Client can do that, though.
return None
def _netrc_open(uri, filename=None):
'''
open uri using netrc credentials.
:param uri: uri to open
:param filename: optional, path to non-default netrc config file
:returns: file-like object from opening a socket to uri, or None
:raises IOError: if opening .netrc file fails (unless file not found)
'''
if not uri:
return None
parsed_uri = urlparse(uri)
machine = parsed_uri.netloc
if not machine:
return None
opener = None
try:
info = netrc.netrc(filename).authenticators(machine)
if info is not None:
(username, _, password) = info
if username and password:
pass_man = HTTPPasswordMgrWithDefaultRealm()
pass_man.add_password(None, machine, username, password)
authhandler = HTTPBasicAuthHandler(pass_man)
opener = build_opener(authhandler)
return opener.open(uri)
else:
# caught below, like other netrc parse errors
raise netrc.NetrcParseError('No authenticators for "%s"' % machine)
except IOError as ioe:
if ioe.errno != 2:
# if = 2, User probably has no .netrc, this is not an error
raise
except netrc.NetrcParseError as neterr:
logger = logging.getLogger('vcstools')
logger.warn('WARNING: parsing .netrc: %s' % str(neterr))
# we could install_opener() here, but prefer to keep
# default opening clean. Client can do that, though.
return None
def __init__(self, cachedir='/tmp', api_host_options={}, urllist=[],
http_debug=False, cookiejar=None, offline=False, enable_cpio=True):
# set up progress bar callback
self.progress_obj = None
if sys.stdout.isatty():
self.progress_obj = create_text_meter(use_pb_fallback=False)
self.cachedir = cachedir
self.urllist = urllist
self.http_debug = http_debug
self.offline = offline
self.cpio = {}
self.enable_cpio = enable_cpio
passmgr = HTTPPasswordMgrWithDefaultRealm()
for host in api_host_options:
passmgr.add_password(None, host, api_host_options[host]['user'],
api_host_options[host]['pass'])
openers = (HTTPBasicAuthHandler(passmgr), )
if cookiejar:
openers += (HTTPCookieProcessor(cookiejar), )
self.gr = OscFileGrabber(progress_obj=self.progress_obj)
def set_proxy(proxy, user=None, password=""):
"""
Set the HTTP proxy for Python to download through.
If ``proxy`` is None then tries to set proxy from environment or system
settings.
:param proxy: The HTTP proxy server to use. For example:
'http://proxy.example.com:3128/'
:param user: The username to authenticate with. Use None to disable
authentication.
:param password: The password to authenticate with.
"""
from nltk import compat
if proxy is None:
# Try and find the system proxy settings
try:
proxy = getproxies()["http"]
except KeyError:
raise ValueError("Could not detect default proxy settings")
# Set up the proxy handler
proxy_handler = ProxyHandler({"https": proxy, "http": proxy})
opener = build_opener(proxy_handler)
if user is not None:
# Set up basic proxy authentication if provided
password_manager = HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(realm=None,
uri=proxy,
user=user,
passwd=password)
opener.add_handler(ProxyBasicAuthHandler(password_manager))
opener.add_handler(ProxyDigestAuthHandler(password_manager))
# Overide the existing url opener
install_opener(opener)
def __init__(self, url):
self.handlers = []
self.parsed_url = url = urlparse(url)
self.scheme = url.scheme
self.insecure = False
if url.scheme == 'https+insecure':
self.insecure = True
url = url._replace(scheme='https')
if url.port:
self.netloc = '{0.hostname}:{0.port}'.format(url)
else:
self.netloc = url.hostname
cleaned_url = urlunparse(url._replace(netloc=self.netloc))
if url.username or url.password:
password_mgr = HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, cleaned_url, url.username, url.password)
self.handlers.append(HTTPDigestAuthHandler(passwd=password_mgr))
self.handlers.append(HTTPBasicAuthHandler(password_mgr=password_mgr))
self.url = cleaned_url
def srtm_login_or_skip(monkeypatch):
import os
try:
srtm_username = os.environ['SRTM_USERNAME']
except KeyError:
pytest.skip('SRTM_USERNAME environment variable is unset.')
try:
srtm_password = os.environ['SRTM_PASSWORD']
except KeyError:
pytest.skip('SRTM_PASSWORD environment variable is unset.')
from urllib.request import (HTTPBasicAuthHandler, HTTPCookieProcessor,
HTTPPasswordMgrWithDefaultRealm, build_opener)
from http.cookiejar import CookieJar
password_manager = HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(None, "https://urs.earthdata.nasa.gov",
srtm_username, srtm_password)
cookie_jar = CookieJar()
opener = build_opener(HTTPBasicAuthHandler(password_manager),
HTTPCookieProcessor(cookie_jar))
monkeypatch.setattr(cartopy.io, 'urlopen', opener.open)
def asf_dl(d, opt_dict):
user_name = password_config.asfuser
user_password = password_config.asfpass
cookie_handler = HTTPCookieProcessor()
pm = HTTPPasswordMgrWithDefaultRealm()
pm.add_password(None, 'urs.earthdata.nasa.gov', user_name, user_password)
auth_handler = HTTPBasicAuthHandler(pm)
opener = build_opener(cookie_handler, auth_handler)
install_opener(opener)
url = d['downloadUrl']
filename = os.path.basename(url)
print("ASF Download:", filename)
start = time.time()
try:
f = urlopen(url)
except HTTPError as e:
print(e)
return
dl_file_size = int(f.info()['Content-Length'])
if os.path.exists(filename):
file_size = os.path.getsize(filename)
if dl_file_size == file_size:
print("%s already downloaded" % filename)
f.close()
return
start = time.time()
CHUNK = 256 * 10240
with open(filename, 'wb') as fp:
while True:
chunk = f.read(CHUNK)
if not chunk: break
fp.write(chunk)
total_time = time.time() - start
mb_sec = (os.path.getsize(filename) / (1024 * 1024.0)) / total_time
print("%s download time: %.2f secs (%.2f MB/sec)" %
(filename, total_time, mb_sec))
f.close()
def get_json_from_url():
url = 'http://192.168.38.128:8181/restconf/operational/network-topology:network-topology/'
username = '******'
password = '******'
p = HTTPPasswordMgrWithDefaultRealm()
p.add_password(None, url, username, password)
handler = HTTPBasicAuthHandler(p)
opener = build_opener(handler)
install_opener(opener)
content = urlopen(url).read()
#print(content)
data = content.decode("utf-8")
print(data)
filename = 'data.json'
with open(filename, 'w') as outfile:
json.dump(data, outfile)
with fileinput.FileInput(filename, inplace=True) as file:
for line in file:
print(line.replace("\\", "").replace("\"{",
"{").replace("}\"", "}"),
end='')
client = MongoClient('localhost', 27017)
db = client.nets
db.nodes.remove({})
mongoimport('localhost', 27017, 'nets', 'nodes', 'data.json')
query_get_all_nodes_info()
query_get_all_ports_info()
query_get_all_links_info()
def get_page(ic_url, user, pw, hostname, verify_ssl,
cafile): # TODO: ignore hostname for now
'''reads icinga service status page from API and returns json'''
url = ic_url + 'v1/objects/services'
logger.debug('url: ' + url)
# authenticate
passman = HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, ic_url, user, pw)
opener = build_opener(HTTPBasicAuthHandler(passman))
install_opener(opener)
opener.addheaders = [('User-agent', 'qicinga2'),
('Accept', 'application/json'),
('X-HTTP-Method-Override', 'GET')]
postdata = '{ "attrs": [ "__name", "last_check_result" ] }'
# setup TLS trust
if cafile != '':
cafile = os.path.expanduser(cafile)
req = urlopen(url, postdata.encode("utf-8"), cafile=cafile)
else:
if not verify_ssl:
ssl._create_default_https_context = ssl._create_unverified_context
req = urlopen(url, postdata.encode("utf-8"))
data = req.read()
return data
def get_from_graphite(self, params):
"""Make a GET request to Graphite with the given params"""
password_mgr = HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(
None,
settings.GRAPHITE_URL,
settings.GRAPHITE_USER,
settings.GRAPHITE_PASSWORD,
)
auth_handler = HTTPBasicAuthHandler(password_mgr)
url = '{0}/render?{1}'.format(settings.GRAPHITE_URL, params)
start = time.time()
try:
with build_opener(auth_handler).open(url) as response:
return response.read()
except HTTPError as error:
print('Warning: Graphite returned ' + str(error) + ' to ' + url)
finally:
end = time.time()
if end - start > 10:
print(
'Warning: Graphite request to {0} took {1} seconds'.format(
url, end - start))
class Fhem:
'''Connects to FHEM via socket communication with optional SSL and password
support'''
def __init__(self, server, port=7072,
use_ssl=False, protocol="telnet", username="", password="", csrf=True,
cafile="", loglevel=1):
'''
Instantiate connector object.
:param server: address of FHEM server
:param port: telnet/http(s) port of server
:param use_ssl: boolean for SSL (TLS) [https as protocol sets use_ssl=True]
:param protocol: 'telnet', 'http' or 'https'
:param username: username for http(s) basicAuth validation
:param password: (global) telnet or http(s) password
:param csrf: (http(s)) use csrf token (FHEM 5.8 and newer), default True
:param cafile: path to public certificate of your root authority, if left empty, https protocol will ignore certificate checks.
:param loglevel: deprecated, will be removed. Please use standard python logging API with logger 'Fhem'.
'''
self.log = logging.getLogger("Fhem")
validprots = ['http', 'https', 'telnet']
self.server = server
self.port = port
self.ssl = use_ssl
self.csrf = csrf
self.csrftoken = ''
self.username = username
self.password = password
self.loglevel = loglevel
self.connection = False
self.cafile = cafile
self.nolog = False
self.bsock = None
self.sock = None
self.https_handler = None
# Set LogLevel
# self.set_loglevel(loglevel)
# Check if protocol is supported
if protocol in validprots:
self.protocol = protocol
else:
self.log.error("Invalid protocol: {}".format(protocol))
# Set authenticication values if#
# the protocol is http(s) or use_ssl is True
if protocol != "telnet":
tmp_protocol = "http"
if (protocol == "https") or (use_ssl is True):
self.ssl = True
tmp_protocol = "https"
self.baseurlauth = "{}://{}:{}/".format(tmp_protocol, server, port)
self.baseurltoken = "{}fhem".format(self.baseurlauth)
self.baseurl = "{}fhem?XHR=1&cmd=".format(self.baseurlauth)
self._install_opener()
def connect(self):
'''create socket connection to server (telnet protocol only)'''
if self.protocol == 'telnet':
try:
self.log.debug("Creating socket...")
if self.ssl:
self.bsock = socket.socket(socket.AF_INET,
socket.SOCK_STREAM)
self.sock = ssl.wrap_socket(self.bsock)
self.log.info("Connecting to {}:{} with SSL (TLS)".format(
self.server, self.port))
else:
self.sock = socket.socket(socket.AF_INET,
socket.SOCK_STREAM)
self.log.info("Connecting to {}:{} without SSL".format(
self.server, self.port))
self.sock.connect((self.server, self.port))
self.connection = True
self.log.info("Connected to {}:{}".format(
self.server, self.port))
except socket.error:
self.connection = False
self.log.error("Failed to connect to {}:{}".format(
self.server, self.port))
return
if self.password != "":
# time.sleep(1.0)
# self.send_cmd("\n")
# prmpt = self._recv_nonblocking(4.0)
prmpt = self.sock.recv(32000)
self.log.debug("auth-prompt: {}".format(prmpt))
self.nolog = True
self.send_cmd(self.password)
self.nolog = False
time.sleep(0.1)
try:
po1 = self.sock.recv(32000)
self.log.debug("auth-repl1: {}".format(po1))
except socket.error:
self.log.error("Failed to recv auth reply")
self.connection = False
return
self.log.info("Auth password sent to {}".format(self.server))
else: # http(s)
if self.csrf:
dat = self.send("")
if dat is not None:
dat = dat.decode("UTF-8")
stp = dat.find("csrf_")
if stp != -1:
token = dat[stp:]
token = token[:token.find("'")]
self.csrftoken = token
self.connection = True
else:
self.log.error(
"CSRF token requested for server that doesn't know CSRF")
else:
self.log.error(
"No valid answer on send when expecting csrf.")
else:
self.connection = True
def connected(self):
'''Returns True if socket/http(s) session is connected to server.'''
return self.connection
def set_loglevel(self, level):
'''Set logging level. [Deprecated, will be removed, use python logging.setLevel]
:param level: 0: critical, 1: errors, 2: info, 3: debug
'''
self.log.warning(
"Deprecation: please set logging levels using python's standard logging for logger 'Fhem'")
if level == 0:
self.log.setLevel(logging.CRITICAL)
elif level == 1:
self.log.setLevel(logging.ERROR)
elif level == 2:
self.log.setLevel(logging.INFO)
elif level == 3:
self.log.setLevel(logging.DEBUG)
def close(self):
'''Closes socket connection. (telnet only)'''
if self.protocol == 'telnet':
if self.connected():
time.sleep(0.2)
self.sock.close()
self.connection = False
self.log.info("Disconnected from fhem-server")
else:
self.log.error("Cannot disconnect, not connected")
else:
self.connection = False
def _install_opener(self):
self.opener = None
if self.username != "":
self.password_mgr = HTTPPasswordMgrWithDefaultRealm()
self.password_mgr.add_password(None, self.baseurlauth,
self.username, self.password)
self.auth_handler = HTTPBasicAuthHandler(self.password_mgr)
if self.ssl is True:
if self.cafile == "":
self.context = ssl.create_default_context()
self.context.check_hostname = False
self.context.verify_mode = ssl.CERT_NONE
else:
self.context = ssl.create_default_context()
self.context.load_verify_locations(cafile=self.cafile)
self.context.verify_mode = ssl.CERT_REQUIRED
self.https_handler = HTTPSHandler(context=self.context)
if self.username != "":
self.opener = build_opener(self.https_handler,
self.auth_handler)
else:
self.opener = build_opener(self.https_handler)
else:
if self.username != "":
self.opener = build_opener(self.auth_handler)
if self.opener is not None:
self.log.debug("Setting up opener on: {}".format(self.baseurlauth))
install_opener(self.opener)
def send(self, buf, timeout=10):
'''Sends a buffer to server
:param buf: binary buffer'''
if len(buf) > 0:
if not self.connected():
self.log.debug("Not connected, trying to connect...")
self.connect()
if self.protocol == 'telnet':
if self.connected():
self.log.debug("Connected, sending...")
try:
self.sock.sendall(buf)
self.log.info("Sent msg, len={}".format(len(buf)))
return None
except OSError as err:
self.log.error(
"Failed to send msg, len={}. Exception raised: {}".format(len(buf), err))
self.connection = None
return None
else:
self.log.error(
"Failed to send msg, len={}. Not connected.".format(len(buf)))
return None
else: # HTTP(S)
paramdata = None
if self.csrf and len(buf) > 0:
if len(self.csrftoken) == 0:
self.log.error("CSRF token not available!")
self.connection = False
else:
datas = {'fwcsrf': self.csrftoken}
paramdata = urlencode(datas).encode('UTF-8')
try:
self.log.debug("Cmd: {}".format(buf))
cmd = quote(buf)
self.log.debug("Cmd-enc: {}".format(cmd))
if len(cmd) > 0:
ccmd = self.baseurl + cmd
else:
ccmd = self.baseurltoken
self.log.info("Request: {}".format(ccmd))
if ccmd.lower().startswith('http'):
ans = urlopen(ccmd, paramdata, timeout=timeout)
else:
self.log.error(
"Invalid URL {}, Failed to send msg, len={}, {}".format(ccmd, len(buf), err))
return None
data = ans.read()
return data
except URLError as err:
self.connection = False
self.log.error(
"Failed to send msg, len={}, {}".format(len(buf), err))
return None
except socket.timeout as err:
# Python 2.7 fix
self.log.error(
"Failed to send msg, len={}, {}".format(len(buf), err))
return None
def send_cmd(self, msg, timeout=10.0):
'''Sends a command to server.
:param msg: string with FHEM command, e.g. 'set lamp on'
:param timeout: timeout on send (sec).
'''
if not self.connected():
self.connect()
if not self.nolog:
self.log.debug("Sending: {}".format(msg))
if self.protocol == 'telnet':
if self.connection:
msg = "{}\n".format(msg)
cmd = msg.encode('utf-8')
return self.send(cmd)
else:
self.log.error(
"Failed to send msg, len={}. Not connected.".format(len(msg)))
return None
else:
return self.send(msg, timeout=timeout)
def _recv_nonblocking(self, timeout=0.1):
if not self.connected():
self.connect()
data = b''
if self.connection:
self.sock.setblocking(False)
data = b''
try:
data = self.sock.recv(32000)
except socket.error as err:
# Resource temporarily unavailable, operation did not complete are expected
if err.errno != errno.EAGAIN and err.errno!= errno.ENOENT:
self.log.debug(
"Exception in non-blocking (1). Error: {}".format(err))
time.sleep(timeout)
wok = 1
while len(data) > 0 and wok > 0:
time.sleep(timeout)
datai = b''
try:
datai = self.sock.recv(32000)
if len(datai) == 0:
wok = 0
else:
data += datai
except socket.error as err:
# Resource temporarily unavailable, operation did not complete are expected
if err.errno != errno.EAGAIN and err.errno!= errno.ENOENT:
self.log.debug(
"Exception in non-blocking (2). Error: {}".format(err))
wok = 0
self.sock.setblocking(True)
return data
def send_recv_cmd(self, msg, timeout=0.1, blocking=False):
'''
Sends a command to the server and waits for an immediate reply.
:param msg: FHEM command (e.g. 'set lamp on')
:param timeout: waiting time for reply
:param blocking: (telnet only) on True: use blocking socket communication (bool)
'''
data = b''
if not self.connected():
self.connect()
if self.protocol == 'telnet':
if self.connection:
self.send_cmd(msg)
time.sleep(timeout)
data = []
if blocking is True:
try:
# This causes failures if reply is larger!
data = self.sock.recv(64000)
except socket.error:
self.log.error("Failed to recv msg. {}".format(data))
return {}
else:
data = self._recv_nonblocking(timeout)
self.sock.setblocking(True)
else:
self.log.error(
"Failed to send msg, len={}. Not connected.".format(len(msg)))
else:
data = self.send_cmd(msg)
if data is None:
return None
if len(data) == 0:
return {}
try:
sdata = data.decode('utf-8')
jdata = json.loads(sdata)
except Exception as err:
self.log.error(
"Failed to decode json, exception raised. {} {}".format(data, err))
return {}
if len(jdata[u'Results']) == 0:
self.log.error("Query had no result.")
return {}
else:
self.log.info("JSON answer received.")
return jdata
def get_dev_state(self, dev, timeout=0.1):
self.log.warning(
"Deprecation: use get_device('device') instead of get_dev_state")
return self.get_device(dev, timeout=timeout, raw_result=True)
def get_dev_reading(self, dev, reading, timeout=0.1):
self.log.warning(
"Deprecation: use get_device_reading('device', 'reading') instead of get_dev_reading")
return self.get_device_reading(dev, reading, value_only=True, timeout=timeout)
def getDevReadings(self, dev, reading, timeout=0.1):
self.log.warning(
"Deprecation: use get_device_reading('device', ['reading']) instead of getDevReadings")
return self.get_device_reading(dev, timeout=timeout, value_only=True, raw_result=True)
def get_dev_readings(self, dev, readings, timeout=0.1):
self.log.warning(
"Deprecation: use get_device_reading('device', ['reading']) instead of get_dev_readings")
return self.get_device_reading(dev, readings, timeout=timeout, value_only=True, raw_result=True)
def get_dev_reading_time(self, dev, reading, timeout=0.1):
self.log.warning(
"Deprecation: use get_device_reading('device', 'reading', time_only=True) instead of get_dev_reading_time")
return self.get_device_reading(dev, reading, timeout=timeout, time_only=True)
def get_dev_readings_time(self, dev, readings, timeout=0.1):
self.log.warning(
"Deprecation: use get_device_reading('device', ['reading'], time_only=True) instead of get_dev_reading_time")
return self.get_device_reading(dev, readings, timeout=timeout, time_only=True)
def getFhemState(self, timeout=0.1):
self.log.warning(
"Deprecation: use get() without parameters instead of getFhemState")
return self.get(timeout=timeout, raw_result=True)
def get_fhem_state(self, timeout=0.1):
self.log.warning(
"Deprecation: use get() without parameters instead of get_fhem_state")
return self.get(timeout=timeout, raw_result=True)
@staticmethod
def _sand_down(value):
return value if len(value.values()) - 1 else list(value.values())[0]
@staticmethod
def _append_filter(name, value, compare, string, filter_list):
value_list = [value] if isinstance(value, str) else value
values = ",".join(value_list)
filter_list.append(string.format(name, compare, values))
def _response_filter(self, response, arg, value, value_only=None, time_only=None):
if len(arg) > 2:
self.log.error("Too many positional arguments")
return {}
result = {}
for r in response if 'totalResultsReturned' not in response else response['Results']:
arg = [arg[0]] if len(arg) and isinstance(arg[0], str) else arg
if value_only:
result[r['Name']] = {k: v['Value'] for k, v in r[value].items() if
'Value' in v and (not len(arg) or (len(arg) and k == arg[0]))} # k in arg[0]))} fixes #14
elif time_only:
result[r['Name']] = {k: v['Time'] for k, v in r[value].items() if
'Time' in v and (not len(arg) or (len(arg) and k == arg[0]))} # k in arg[0]))}
else:
result[r['Name']] = {k: v for k, v in r[value].items() if
(not len(arg) or (len(arg) and k == arg[0]))} # k in arg[0]))}
if not result[r['Name']]:
result.pop(r['Name'], None)
elif len(result[r['Name']].values()) == 1:
result[r['Name']] = list(result[r['Name']].values())[0]
return result
def _parse_filters(self, name, value, not_value, filter_list, case_sensitive):
compare = "=" if case_sensitive else "~"
if value:
self._append_filter(name, value, compare, "{}{}{}", filter_list)
elif not_value:
self._append_filter(name, not_value, compare,
"{}!{}{}", filter_list)
def _convert_data(self, response, k, v):
try:
test_type = unicode
except NameError:
test_type = str
if isinstance(v, test_type):
if re.findall("^[0-9]+$", v):
response[k] = int(v)
elif re.findall(r"^[0-9]+\.[0-9]+$", v):
response[k] = float(v)
elif re.findall("^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}$", v):
response[k] = datetime.datetime.strptime(
v, '%Y-%m-%d %H:%M:%S')
if isinstance(v, dict):
self._parse_data_types(response[k])
if isinstance(v, list):
self._parse_data_types(response[k])
def _parse_data_types(self, response):
if isinstance(response, dict):
for k, v in response.items():
self._convert_data(response, k, v)
if isinstance(response, list):
for i, v in enumerate(response):
self._convert_data(response, i, v)
def get(self, name=None, state=None, group=None, room=None, device_type=None, not_name=None, not_state=None, not_group=None,
not_room=None, not_device_type=None, case_sensitive=None, filters=None, timeout=0.1, blocking=False, raw_result=None):
"""
Get FHEM data of devices, can filter by parameters or custom defined filters.
All filters use regular expressions (except full match), so don't forget escaping.
Filters can be used by all other get functions.
For more information about filters, see https://FHEM.de/commandref.html#devspec
:param name: str or list, device name in FHEM
:param state: str or list, state in FHEM
:param group: str or list, filter FHEM groups
:param room: str or list, filter FHEM room
:param device_type: str or list, FHEM device type
:param not_name: not name
:param not_state: not state
:param not_group: not group
:param not_room: not room
:param not_device_type: not device_type
:param case_sensitive: bool, use case_sensitivity for all filter functions
:param filters: dict of filters - key=attribute/internal/reading, value=regex for value, e.g. {"battery": "ok"}
:param raw_result: On True: Don't convert to python types and send full FHEM response
:param timeout: timeout for reply
:param blocking: telnet socket mode, default blocking=False
:return: dict of FHEM devices
"""
if not self.connected():
self.connect()
if self.connected():
filter_list = []
self._parse_filters("NAME", name, not_name,
filter_list, case_sensitive)
self._parse_filters("STATE", state, not_state,
filter_list, case_sensitive)
self._parse_filters("group", group, not_group,
filter_list, case_sensitive)
self._parse_filters("room", room, not_room,
filter_list, case_sensitive)
self._parse_filters("TYPE", device_type,
not_device_type, filter_list, case_sensitive)
if filters:
for key, value in filters.items():
filter_list.append("{}{}{}".format(
key, "=" if case_sensitive else "~", value))
cmd = "jsonlist2 {}".format(":FILTER=".join(filter_list))
if self.protocol == 'telnet':
result = self.send_recv_cmd(
cmd, blocking=blocking, timeout=timeout)
else:
result = self.send_recv_cmd(
cmd, blocking=False, timeout=timeout)
if not result or raw_result:
return result
result = result['Results']
self._parse_data_types(result)
return result
else:
self.log.error("Failed to get fhem state. Not connected.")
return {}
def get_states(self, **kwargs):
"""
Return only device states, can use filters from get().
:param kwargs: Use keyword arguments from :py:meth:`Fhem.get` function
:return: dict of FHEM devices with states
"""
response = self.get(**kwargs)
if not response:
return response
return {r['Name']: r['Readings']['state']['Value'] for r in response if 'state' in r['Readings']}
def get_readings(self, *arg, **kwargs):
"""
Return readings of a device, can use filters from get().
:param arg: str, Get only a specified reading, return all readings of device when parameter not given
:param value_only: return only value of reading, not timestamp
:param time_only: return only timestamp of reading
:param kwargs: use keyword arguments from :py:meth:`Fhem.get` function
:return: dict of FHEM devices with readings
"""
value_only = kwargs['value_only'] if 'value_only' in kwargs else None
time_only = kwargs['time_only'] if 'time_only' in kwargs else None
kwargs.pop('value_only', None)
kwargs.pop('time_only', None)
response = self.get(**kwargs)
return self._response_filter(response, arg, 'Readings', value_only=value_only, time_only=time_only)
def get_attributes(self, *arg, **kwargs):
"""
Return attributes of a device, can use filters from get()
:param arg: str, Get only specified attribute, return all attributes of device when parameter not given
:param kwargs: use keyword arguments from :py:meth:`Fhem.get` function
:return: dict of FHEM devices with attributes
"""
response = self.get(**kwargs)
return self._response_filter(response, arg, 'Attributes')
def get_internals(self, *arg, **kwargs):
"""
Return internals of a device, can use filters from get()
:param arg: str, Get only specified internal, return all internals of device when parameter not given
:param kwargs: use keyword arguments from :py:meth:`Fhem.get` function
:return: dict of FHEM devices with internals
"""
response = self.get(**kwargs)
return self._response_filter(response, arg, 'Internals')
def get_device(self, device, **kwargs):
"""
Get all data from a device
:param device: str or list,
:param kwargs: use keyword arguments from :py:meth:`Fhem.get` function
:return: dict with data of specific FHEM device
"""
return self.get(name=device, **kwargs)
def get_device_state(self, device, **kwargs):
"""
Get state of one device
:param device: str or list,
:param kwargs: use keyword arguments from :py:meth:`Fhem.get` and :py:meth:`Fhem.get_states` functions
:return: str, int, float when only specific value requested else dict
"""
result = self.get_states(name=device, **kwargs)
return self._sand_down(result)
def get_device_reading(self, device, *arg, **kwargs):
"""
Get reading(s) of one device
:param device: str or list,
:param arg: str for one reading, list for special readings, empty for all readings
:param kwargs: use keyword arguments from :py:meth:`Fhem.get` and :py:meth:`Fhem.get_readings` functions
:return: str, int, float when only specific value requested else dict
"""
result = self.get_readings(*arg, name=device, **kwargs)
return self._sand_down(result)
def get_device_attribute(self, device, *arg, **kwargs):
"""
Get attribute(s) of one device
:param device: str or list,
:param arg: str for one attribute, list for special attributes, empty for all attributes
:param kwargs: use keyword arguments from :py:meth:`Fhem.get` function
:return: str, int, float when only specific value requested else dict
"""
result = self.get_attributes(*arg, name=device, **kwargs)
return self._sand_down(result)
def get_device_internal(self, device, *arg, **kwargs):
"""
Get internal(s) of one device
:param device: str or list,
:param arg: str for one internal value, list for special internal values, empty for all internal values
:param kwargs: use keyword arguments from :py:meth:`Fhem.get` function
:return: str, int, float when only specific value requested else dict
"""
result = self.get_internals(*arg, name=device, **kwargs)
return self._sand_down(result)
def set_authentication(self, uri, login, password):
password_manager = HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(realm=None, uri=uri, user=login, passwd=password)
self.http_opener = build_opener(HTTPBasicAuthHandler(password_manager), HTTPDigestAuthHandler(password_manager))
# 发送登陆信息
from urllib.request import HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, build_opener
from urllib.request import URLError, Request, urlopen
username = "******"
password = "******"
url = "http://localhost:5000/"
# 创建一个密码管理对象，用来保存 HTTP 请求相关的用户名和密码.
p = HTTPPasswordMgrWithDefaultRealm()
p.add_password(None, url, username, password)
# HTTPBasicAuthHandler 管理认证
auth_handler = HTTPBasicAuthHandler(p)
opener = build_opener(auth_handler)
try:
'''
result = opener.open(url)
html = result.read().decode("utf-8")
print(html)
'''
response = urlopen(url)
print(response.read().decode("utf-8"))
except URLError as e:
print(e.reason)
def download(self, url, error_message, timeout, tries):
"""
Downloads a URL and returns the contents
Uses the proxy settings from the Package Control.sublime-settings file,
however there seem to be a decent number of proxies that this code
does not work with. Patches welcome!
:param url:
The URL to download
:param error_message:
A string to include in the console error that is printed
when an error occurs
:param timeout:
The int number of seconds to set the timeout to
:param tries:
The int number of times to try and download the URL in the case of
a timeout or HTTP 503 error
:return:
The string contents of the URL, or False on error
"""
http_proxy = self.settings.get('http_proxy')
https_proxy = self.settings.get('https_proxy')
if http_proxy or https_proxy:
proxies = {}
if http_proxy:
proxies['http'] = http_proxy
if https_proxy:
proxies['https'] = https_proxy
proxy_handler = ProxyHandler(proxies)
else:
proxy_handler = ProxyHandler()
password_manager = HTTPPasswordMgrWithDefaultRealm()
proxy_username = self.settings.get('proxy_username')
proxy_password = self.settings.get('proxy_password')
if proxy_username and proxy_password:
if http_proxy:
password_manager.add_password(None, http_proxy, proxy_username,
proxy_password)
if https_proxy:
password_manager.add_password(None, https_proxy, proxy_username,
proxy_password)
handlers = [proxy_handler]
if os.name == 'nt':
ntlm_auth_handler = ProxyNtlmAuthHandler(password_manager)
handlers.append(ntlm_auth_handler)
basic_auth_handler = ProxyBasicAuthHandler(password_manager)
digest_auth_handler = ProxyDigestAuthHandler(password_manager)
handlers.extend([digest_auth_handler, basic_auth_handler])
debug = self.settings.get('debug')
if debug:
console_write(u"Urllib Debug Proxy", True)
console_write(u" http_proxy: %s" % http_proxy)
console_write(u" https_proxy: %s" % https_proxy)
console_write(u" proxy_username: %s" % proxy_username)
console_write(u" proxy_password: %s" % proxy_password)
secure_url_match = re.match('^https://([^/]+)', url)
if secure_url_match != None:
secure_domain = secure_url_match.group(1)
bundle_path = self.check_certs(secure_domain, timeout)
if not bundle_path:
return False
bundle_path = bundle_path.encode(sys.getfilesystemencoding())
handlers.append(ValidatingHTTPSHandler(ca_certs=bundle_path,
debug=debug, passwd=password_manager,
user_agent=self.settings.get('user_agent')))
else:
handlers.append(DebuggableHTTPHandler(debug=debug,
passwd=password_manager))
install_opener(build_opener(*handlers))
while tries > 0:
tries -= 1
try:
request = Request(url, headers={
"User-Agent": self.settings.get('user_agent'),
# Don't be alarmed if the response from the server does not
# select one of these since the server runs a relatively new
# version of OpenSSL which supports compression on the SSL
# layer, and Apache will use that instead of HTTP-level
# encoding.
"Accept-Encoding": "gzip,deflate"})
http_file = urlopen(request, timeout=timeout)
self.handle_rate_limit(http_file, url)
result = http_file.read()
encoding = http_file.headers.get('Content-Encoding')
return self.decode_response(encoding, result)
except (HTTPException) as e:
error_string = u'%s HTTP exception %s (%s) downloading %s.' % (
error_message, e.__class__.__name__, unicode_from_os(e), url)
console_write(error_string, True)
except (HTTPError) as e:
# Make sure we obey Github's rate limiting headers
self.handle_rate_limit(e, url)
# Bitbucket and Github return 503 a decent amount
if unicode_from_os(e.code) == '503':
error_string = u'Downloading %s was rate limited, trying again' % url
console_write(error_string, True)
continue
error_string = u'%s HTTP error %s downloading %s.' % (
error_message, unicode_from_os(e.code), url)
console_write(error_string, True)
except (URLError) as e:
# Bitbucket and Github timeout a decent amount
if unicode_from_os(e.reason) == 'The read operation timed out' \
or unicode_from_os(e.reason) == 'timed out':
error_string = u'Downloading %s timed out, trying again' % url
console_write(error_string, True)
continue
error_string = u'%s URL error %s downloading %s.' % (
error_message, unicode_from_os(e.reason), url)
console_write(error_string, True)
break
return False
from urllib.request import HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, build_opener
from urllib.error import URLError
username = '******'
password = '******'
url = 'http://localhost:5000'
p = HTTPPasswordMgrWithDefaultRealm()
# 第一个参数是realm，若设置了则必须与服务器设置WWW-Authenticate时指定的realm相同
p.add_password('localhost', url, username, password)
auth_handler = HTTPBasicAuthHandler(p)
opener = build_opener(auth_handler)
try:
result = opener.open(url)
html = result.read().decode('utf-8')
print(html)
except URLError as e:
print(e.reason)
from urllib.request import Request
from urllib.request import HTTPPasswordMgrWithDefaultRealm
from urllib.request import HTTPBasicAuthHandler
from urllib.request import build_opener
from urllib.request import install_opener
from urllib.request import urlopen
req = Request("http://example.com/")
password_manager = HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(None, "http://example.com/", 'user', 'pass')
auth_manager = HTTPBasicAuthHandler(password_manager)
opener = build_opener(auth_manager)
install_opener(opener)
handler = urlopen(req)
print(handler.getcode())
from urllib.request import HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, build_opener
from urllib.error import URLError
username = '******'
password = '******'
url = 'http://localhost:5000/'
p = HTTPPasswordMgrWithDefaultRealm()
p.add_password(None, url, username, password)
auth_handler = HTTPBasicAuthHandler(p)
opener = build_opener(auth_handler)
try:
result = opener.open(url)
html = result.read().decode('utf-8')
print(html)
except URLError as e:
print(e.reason)
#这里首先实例化HTTPBasicAuthHandler对象，其参数是HTTPPasswordMgrWithDefaultRealm对象，它利用add_password()添加进去用户名和密码，这样就建立了一个处理验证的Handler。
#接下来，利用这个Handler并使用build_opener()方法构建一个Opener，这个Opener在发送请求时就相当于已经验证成功了。
#接下来，利用Opener的open()方法打开链接，就可以完成验证了。这里获取到的结果就是验证后的页面源码内容。
def curl(url, params=None, auth=None, req_type='GET', data=None, headers=None, timeout=None, use_gzip=True):
"""
Make request to web resource
:param url: Url to endpoint
:param params: list of params after "?"
:param auth: authorization tokens
:param req_type: column_type of the request
:param data: data which need to be posted
:param headers: headers which would be posted with request
:param timeout: Request timeout
:param use_gzip: Accept gzip and deflate response from the server
:return Response object
:column_type url str
:column_type params dict
:column_type auth CURLAuth
:column_type req_type str
:column_type headers dict
:column_type timeout int
:column_type use_gzip bool
:rtype CURLResponse
"""
post_req = ["POST", "PUT"]
get_req = ["GET", "DELETE"]
if params is not None:
url += "?" + urlencode(params)
if req_type not in post_req + get_req:
raise IOError("Wrong request column_type \"%s\" passed" % req_type)
_headers = {}
handler_chain = []
req_args = {
"headers": _headers
}
# process content
if req_type in post_req and data is not None:
_data, __header = __parse_content(data)
_headers.update(__header)
_headers["Content-Length"] = len(_data)
req_args["data"] = _data
# process gzip and deflate
if use_gzip:
if "Accept-Encoding" in _headers:
if "gzip" not in _headers["Accept-Encoding"]:
_headers["Accept-Encoding"] += ", gzip, x-gzip, deflate"
else:
_headers["Accept-Encoding"] = "gzip, x-gzip, deflate"
if auth is not None and auth.force is False:
manager = HTTPPasswordMgrWithDefaultRealm()
manager.add_password(None, url, auth.user, auth.password)
handler_chain.append(HTTPBasicAuthHandler(manager))
if auth is not None and auth.force:
_headers.update(auth.headers)
if headers is not None:
_headers.update(headers)
director = build_opener(*handler_chain)
req = Request(url, **req_args)
req.get_method = lambda: req_type
try:
if timeout is not None:
return CURLResponse(director.open(req, timeout=timeout))
else:
return CURLResponse(director.open(req))
except URLError as e:
if isinstance(e, HTTPError):
raise e
else:
raise TimeoutError
def netapp_filer(host, username, password, timeout, command, parameters=None,
ssl=False):
"""
Issue a command to the NetApp filer.
Note: Change to default ssl on before we ship a release version.
"""
proto = 'http'
if ssl:
proto = 'https'
url = "%s://%s/servlets/netapp.servlets.admin.XMLrequest_filer" % \
(proto, host)
req = Request(url)
req.add_header('Content-Type', 'text/xml')
password_manager = HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(None, url, username, password)
auth_manager = HTTPBasicAuthHandler(password_manager)
opener = build_opener(auth_manager)
install_opener(opener)
# build the command and the arguments for it
p = ""
if parameters:
for k, v in list(parameters.items()):
p += "<%s>%s</%s>" % (k, param_value(v), k)
payload = "<%s>\n%s\n</%s>" % (command, p, command)
data = """<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE netapp SYSTEM "file:/etc/netapp_filer.dtd">
<netapp xmlns="http://www.netapp.com/filer/admin" version="1.1">
%s
</netapp>
""" % payload
handler = None
rc = None
try:
handler = urlopen(req, data.encode('utf-8'), float(timeout))
if handler.getcode() == 200:
rc = netapp_filer_parse_response(handler.read())
except HTTPError:
raise
except URLError as ue:
if isinstance(ue.reason, socket.timeout):
raise FilerError(Filer.ETIMEOUT, "Connection timeout")
else:
raise
except socket.timeout:
raise FilerError(Filer.ETIMEOUT, "Connection timeout")
except SSLError as sse:
# The ssl library doesn't give a good way to find specific reason.
# We are doing a string contains which is not ideal, but other than
# throwing a generic error in this case there isn't much we can do
# to be more specific.
if "timed out" in str(sse).lower():
raise FilerError(Filer.ETIMEOUT, "Connection timeout (SSL)")
else:
raise FilerError(Filer.EUNKNOWN,
"SSL error occurred (%s)", str(sse))
finally:
if handler:
handler.close()
return rc
def setup_opener(self, url, timeout):
"""
Sets up a urllib OpenerDirector to be used for requests. There is a
fair amount of custom urllib code in Package Control, and part of it
is to handle proxies and keep-alives. Creating an opener the way
below is because the handlers have been customized to send the
"Connection: Keep-Alive" header and hold onto connections so they
can be re-used.
:param url:
The URL to download
:param timeout:
The int number of seconds to set the timeout to
"""
if not self.opener:
http_proxy = self.settings.get('http_proxy')
https_proxy = self.settings.get('https_proxy')
if http_proxy or https_proxy:
proxies = {}
if http_proxy:
proxies['http'] = http_proxy
if https_proxy:
proxies['https'] = https_proxy
proxy_handler = ProxyHandler(proxies)
else:
proxy_handler = ProxyHandler()
password_manager = HTTPPasswordMgrWithDefaultRealm()
proxy_username = self.settings.get('proxy_username')
proxy_password = self.settings.get('proxy_password')
if proxy_username and proxy_password:
if http_proxy:
password_manager.add_password(None, http_proxy, proxy_username,
proxy_password)
if https_proxy:
password_manager.add_password(None, https_proxy, proxy_username,
proxy_password)
handlers = [proxy_handler]
basic_auth_handler = ProxyBasicAuthHandler(password_manager)
digest_auth_handler = ProxyDigestAuthHandler(password_manager)
handlers.extend([digest_auth_handler, basic_auth_handler])
debug = self.settings.get('debug')
if debug:
console_write(
u'''
Urllib Debug Proxy
http_proxy: %s
https_proxy: %s
proxy_username: %s
proxy_password: %s
''',
(http_proxy, https_proxy, proxy_username, proxy_password)
)
secure_url_match = re.match('^https://([^/]+)', url)
if secure_url_match is not None:
bundle_path = get_ca_bundle_path(self.settings)
bundle_path = bundle_path.encode(sys.getfilesystemencoding())
handlers.append(ValidatingHTTPSHandler(ca_certs=bundle_path,
debug=debug, passwd=password_manager,
user_agent=self.settings.get('user_agent')))
else:
handlers.append(DebuggableHTTPHandler(debug=debug,
passwd=password_manager))
self.opener = build_opener(*handlers)
def basic_auth_opener(url, username, password):
password_manager = HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(None, url, username, password)
auth_handler = PreemptiveBasicAuthHandler(password_manager)
opener = build_opener(auth_handler)
return opener
