def get_items(type):
url = '{0}/api/{1}/{2}/'.format(baseUrl, type, '%2f')
password_mgr = request.HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, baseUrl, username, password)
handler = request.HTTPBasicAuthHandler(password_mgr)
req = request.Request(url)
opener = request.build_opener(handler)
res = opener.open(req).read().decode('utf-8', errors='ignore')
items = [x['name'] for x in json.loads(res)]
return set(items)
def __init__(self,
base_url="http://teide.geophysik.uni-muenchen.de:8080",
user="******",
password="******",
timeout=10,
debug=False,
retries=3):
"""
DEPRECATED -- SeisHub database client is deprecated
Initializes the SeisHub Web service client.
:type base_url: str, optional
:param base_url: SeisHub connection string. Defaults to
'http://teide.geophysik.uni-muenchen.de:8080'.
:type user: str, optional
:param user: The user name used for identification with the Web
service. Defaults to ``'admin'``.
:type password: str, optional
:param password: A password used for authentication with the Web
service. Defaults to ``'admin'``.
:type timeout: int, optional
:param timeout: Seconds before a connection timeout is raised (default
is 10 seconds).
:type debug: bool, optional
:param debug: Enables verbose output.
:type retries: int
:param retries: Number of retries for failing requests.
"""
msg = "The module obspy.clients.seishub is deprecated and will be " + \
"removed in the next major release."
warnings.warn(msg, ObsPyDeprecationWarning)
self.base_url = base_url
#: A :class:`obspy.clients.seishub.client._WaveformMapperClient`
#: instance
self.waveform = _WaveformMapperClient(self)
#: A :class:`obspy.clients.seishub.client._StationMapperClient`
#: instance
self.station = _StationMapperClient(self)
#: An :class:`obspy.clients.seishub.client._EventMapperClient` instance
self.event = _EventMapperClient(self)
self.timeout = timeout
self.debug = debug
self.retries = retries
self.xml_seeds = {}
self.station_list = {}
# Create an OpenerDirector for Basic HTTP Authentication
password_mgr = urllib_request.HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, base_url, user, password)
auth_handler = urllib_request.HTTPBasicAuthHandler(password_mgr)
opener = urllib_request.build_opener(auth_handler)
# install globally
urllib_request.install_opener(opener)
def __build_url_opener(self, uri):
"""
Build the url opener for managing HTTP Basic Authentication
"""
# Create an OpenerDirector with support
# for Basic HTTP Authentication
auth_handler = urllib2.HTTPBasicAuthHandler()
auth_handler.add_password(None, uri, self.client_id,
self.client_secret)
opener = urllib2.build_opener(auth_handler)
return opener
def __send_request(self, uri, params=None):
url = self.__get_url(uri, params)
request = urllib_request.Request(url)
pass_man = urllib_request.HTTPPasswordMgrWithDefaultRealm()
pass_man.add_password(None, url, self.login, self.password)
auth_handler = urllib_request.HTTPBasicAuthHandler(pass_man)
try:
opener = urllib_request.build_opener(auth_handler)
data = opener.open(request).read()
return data
except IOError as e:
return e.errno
def get_page(page, username, password, filename):
url = r'http://www.pythonchallenge.com/pc/' + page
password_mgr = request.HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, url, username, password)
handler = request.HTTPBasicAuthHandler(password_mgr)
opener = request.build_opener(handler)
request.install_opener(opener)
with request.urlopen(url) as response:
result = response.read()
with open(filename, 'wb') as f:
f.write(result)
return result
def autentica_proxy():
autenticacao = get_autenticacao()
url = r'http://' + autenticacao + '@proxy.unipam.edu.br:3128'
proxy = request.ProxyHandler({'http': url})
auth = request.HTTPBasicAuthHandler()
opener = request.build_opener(proxy, auth, request.HTTPHandler)
request.install_opener(opener)
return request
def _get_data(url, login, password):
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
p = request.HTTPPasswordMgrWithDefaultRealm()
p.add_password(None, url, login, password)
ssl_handler = request.HTTPSHandler(context=ctx)
auth_handler = request.HTTPBasicAuthHandler(p)
opener = request.build_opener(ssl_handler, auth_handler)
request.install_opener(opener)
html = opener.open(url).read()
return html
def __sendRequest(self, uri, params=None):
url = self.__getUrl(uri, params)
request = urllib2.Request(url)
passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, url, self.login, self.password)
authhandler = urllib2.HTTPBasicAuthHandler(passman)
try:
opener = urllib2.build_opener(authhandler)
data = opener.open(request).read()
return data
except IOError:
return print('Какая то блять ошибка')
def call_api(self, path):
# Call the REST API and convert the results into JSON.
url = '{0}://{1}:{2}/api/{3}'.format(self.protocol, self.host_name,
self.port, path)
password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, url, self.user_name, self.password)
handler = urllib2.HTTPBasicAuthHandler(password_mgr)
logging.debug('Issue a rabbit API call to get data on ' + path +
" against " + self.host_name)
logging.debug('Full URL:' + url)
return json.loads(
urllib2.build_opener(handler).open(url).read().decode("utf-8"))
def download_file(url,
destination_dir=None,
file_name=None,
silent=False,
auth=None):
"""url lib downloader
Downloads content of url to destination dir with or without a given file name
Supports basic authentication and report hooking
Args:
url: str, download path
destination_dir: str, optional if specified file will be downloaded to destination
file_name : str, optional: file name of the downloaded data
auth: dict, user authentication {"username": your_user_name, "password": your_password}
Returns:
str: file path or None
"""
os.makedirs(destination_dir, exist_ok=True)
if not file_name:
file_name = url.split('/')[-1]
if not destination_dir:
destination = file_name
else:
destination = os.path.join(destination_dir, file_name)
if os.path.exists(destination):
return destination
if not silent:
print('[*] downloading ' + url + " to " + destination)
if auth and "username" in auth and "password" in auth:
pass_manager = urllib.HTTPPasswordMgrWithDefaultRealm()
pass_manager.add_password(None, url, auth["username"],
auth["password"])
urllib.install_opener(
urllib.build_opener(urllib.HTTPBasicAuthHandler(pass_manager)))
try:
response = urllib.urlopen(url)
_chunk_read(response,
destination,
reporthook=None if silent else _download_reporthook)
except urllib.URLError as error:
if not silent:
print("[*] error downloading", url, ":", error.reason)
return None
return destination
def main():
mem_quota = parse_options(sys.argv[1:])
settings = server_settings(CONFIG_FILE)
password_mgr = PasswordManager(settings['username'], settings['password'])
handler = urllib2.HTTPBasicAuthHandler(password_mgr)
opener = urllib2.build_opener(handler)
if mem_quota:
set_mem_quota(opener, mem_quota)
create_bucket(opener, settings, settings['bucket_prefix'])
create_bucket(opener, settings, settings['bucket_prefix'] + '_sasl', True)
def http_auth_request(url, host, user, passwd, user_agent=USER_AGENT):
"""Call an HTTP server with authorization credentials using urllib2.
"""
if DEBUG: httplib.HTTPConnection.debuglevel = 1
# Hook up handler/opener to urllib2
password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(None, host, user, passwd)
auth_handler = urllib2.HTTPBasicAuthHandler(password_manager)
opener = urllib2.build_opener(auth_handler)
urllib2.install_opener(opener)
return http_request(url, user_agent)
def send(address,
message,
requestType,
useAuth=False,
username="",
password=""):
"""
Used for sending requests that do not require message body, like GET and DELETE.
Params: address of the webservice (string).
message to the webservice (string).
request type for the message (string, GET or DELETE).
"""
try:
if not address.startswith("http://"):
address = "http://" + address
url = address + message
if useAuth:
password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, url, username, password)
handler = urllib2.HTTPBasicAuthHandler(password_mgr)
opener = urllib2.build_opener(handler)
request = urllib2.Request(url)
request.get_method = lambda: requestType
response = opener.open(request)
else:
opener = urllib2.build_opener(urllib2.HTTPHandler)
request = urllib2.Request(url)
request.get_method = lambda: requestType
response = opener.open(request)
data = response.read()
data = data.replace('\n'.encode(), ' '.encode())
except urllib2.HTTPError as err:
data = traceback.format_exc()
if err.code == 401:
data = "Error: HTTP Status Code 401. Authentication with the Web Service failed. Please ensure that the authentication credentials are set, are correct, and that authentication mode is enabled."
else:
data = err.read()
try:
data = json.loads(data)
except:
pass
return data
def download(url, des, overwrite=False, username='******', password='******'):
""" Download and save a file from ftp
Args:
url (str): the link to the file
des (str): destination to save the file
overwrite (bool): overwrite or not
username (str): username
password (str): password
Returns:
0: successful
1: error due to des
2: error due to url
"""
# check if file already exists
if (not overwrite) and os.path.isfile(des):
log.warning('{} already exists.'.format(des.split('/')[-1]))
return 1
# download file
try:
if not username == 'NA' and not password == 'NA':
pw_manager = urllib.HTTPPasswordMgrWithDefaultRealm()
pw_manager.add_password(None, "https://urs.earthdata.nasa.gov",
username, password)
cookie_jar = cj()
opener = urllib.build_opener(
urllib.HTTPBasicAuthHandler(pw_manager),
urllib.HTTPCookieProcessor(cookie_jar))
urllib.install_opener(opener)
req = urllib.Request(url)
response = urllib.urlopen(req)
except:
log.error('{}...failed.'.format(url))
return 2
else:
with open(des, 'wb') as f:
while True:
data = response.read(cons.CHUNK)
if not data:
break
else:
f.write(data)
# complete download
log.info('{}...completed.'.format(url.split('/')[-1]))
# done
return 0
def get_fn(url, login, password):
url += '/control/rcontrol?action=gettext&message=%24(ID.NAM)'
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
p = request.HTTPPasswordMgrWithDefaultRealm()
p.add_password(None, url, login, password)
ssl_handler = request.HTTPSHandler(context=ctx)
auth_handler = request.HTTPBasicAuthHandler(p)
opener = request.build_opener(ssl_handler, auth_handler)
request.install_opener(opener)
result = opener.open(url)
data = result.read()
return data.decode()
def __init__(self, categoryName):
self.__categoryName = categoryName
self.__filePath = r"H:/GIT/Python/Spider/CartoonSpider/file/categories/" + categoryName + '/list/'
self.__baseUrl = 'http://cartoon.baidu.com/comic/'
user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
# 设置代理 IP，http 不行，使用 https
proxy = req.ProxyHandler({'https': 's1firewall:8080'})
auth = req.HTTPBasicAuthHandler()
# 构造 opener
opener = req.build_opener(proxy, auth, req.HTTPHandler)
# 添加 header
opener.addheaders = [('User-Agent', user_agent)]
# 安装 opener
req.install_opener(opener)
def __init__(self):
self.__baseUrl = r'http://wufazhuce.com/'
self.__workPath = os.path.abspath('..') + r'\temp\OEBPS\images'
self.__imageName = 'cover.png'
user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
# 设置代理 IP，http 不行，使用 https
proxy = req.ProxyHandler({'https': 's1firewall:8080'})
auth = req.HTTPBasicAuthHandler()
# 构造 opener
opener = req.build_opener(proxy, auth, req.HTTPHandler)
# 添加 header
opener.addheaders = [('User-Agent', user_agent)]
# 安装 opener
req.install_opener(opener)
def _connect(self):
"""
Internal function connecting to the server.
"""
username = self._username
password = self._password
if username and password:
passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, self._url, username, password)
self.authhandler = urllib2.HTTPBasicAuthHandler(passman)
self.passman = passman
self.opener = urllib2.build_opener(self.authhandler)
else:
self.opener = urllib2.build_opener()
def _get_raw_content(self):
"""returns all values as a list"""
password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, self.host, self.username,
self.password)
handler = urllib2.HTTPBasicAuthHandler(password_mgr)
opener = urllib2.build_opener(handler)
opener.open(self.host)
urllib2.install_opener(opener)
response = urllib2.urlopen(self.host)
root = html.fromstring(response.read().strip())
data = [v.text.strip() for v in root.xpath("//td[@bgcolor='#FFFFFF']")]
return data
def download(config, ambari):
url = get_url(config, ambari)
password_manager = urllib2.HTTPPasswordMgrWithPriorAuth()
password_manager.add_password(None, url, ambari.credentials.username, ambari.credentials.password, is_authenticated=True)
auth_manager = urllib2.HTTPBasicAuthHandler(password_manager)
opener = urllib2.build_opener(auth_manager)
file_name = get_file_name(config)
f = open(file_name, 'wb')
f.write(opener.open(url).read())
f.close()
print('Success load ' + config)
return file_name
def install_opener(baseurl, login, password):
password_mgr = request.HTTPPasswordMgrWithDefaultRealm()
# Add the username and password.
# If we knew the realm, we could use it instead of None.
password_mgr.add_password(None, baseurl, login, password)
handler = request.HTTPBasicAuthHandler(password_mgr)
# create "opener" (OpenerDirector instance)
opener = request.build_opener(handler)
# use the opener to fetch a URL
opener.open(baseurl)
# Install the opener.
# Now all calls to urllib.request.urlopen use our opener.
request.install_opener(opener)
return
def get_img(url, login, password, x, y, q):
url += '/cgi-bin/image.jpg?size=%sx%s&quality=%s'.format(x, y, q)
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
p = request.HTTPPasswordMgrWithDefaultRealm()
p.add_password(None, url, login, password)
ssl_handler = request.HTTPSHandler(context=ctx)
auth_handler = request.HTTPBasicAuthHandler(p)
opener = request.build_opener(ssl_handler, auth_handler)
request.install_opener(opener)
result = opener.open(url)
data = result.read()
return data
def get_url(url, **kwargs):
''' Returns the content of website or data file as string '''
if isrequests:
rr = requests.get(url, **kwargs)
return rr.text
else:
if 'proxies' in kwargs:
if kwargs['proxies'] is not None:
proxy = liburl.ProxyHandler(proxies)
auth = liburl.HTTPBasicAuthHandler()
opener = liburl.build_opener(proxy, auth, liburl.HTTPHandler)
liburl.install_opener(opener)
rr = liburl.urlopen(url)
return rr.read().decode('utf-8')
def http_fetch(url: str,
fail_on_404: bool = True,
verify_cert: bool = True,
username: str = None,
pwd: str = '') -> bytes:
"""Fetches and returns data at the specified URL. If server returned an error, an exception is raised.
:param url: URL to fetch.
:param fail_on_404: Whether to raise an HTTPError on HTTP 404 error.
:param verify_cert: Whether to enforce SSL certificate check.
:param username: Optional username to use for authentication. If None, authentication is not used.
:param pwd: Optional password to use for authentication. Ignored if username is None.
:return: Fetched [binary] data as bytes.
"""
try:
handlers = []
# If certificate errors are to be ignored, create a non-verified context
if not verify_cert:
context = ssl.create_default_context()
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
# Add an HTTPS handler
handlers.append(request.HTTPSHandler(context=context))
# If credentials are provided
if username is not None:
# Create a password manager
password_mgr = request.HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, url, username, pwd)
# Add a basic authentication handler
handlers.append(request.HTTPBasicAuthHandler(password_mgr))
# Create an "opener" (OpenerDirector instance)
opener = request.build_opener(*handlers)
# Use the opener to fetch the URL
response = opener.open(url)
except HTTPError as e:
# Special handling of 'File not found' errors
if e.code == 404:
if fail_on_404:
raise HttpError(
'Failed to fetch {} (file not found)'.format(url))
return None
raise
# If succeeded, read the data and return
return response.read()
def getFile(cls, getfile, compressed=False):
if cls.getProxy():
proxy = req.ProxyHandler({'http': cls.getProxy(), 'https': cls.getProxy()})
auth = req.HTTPBasicAuthHandler()
opener = req.build_opener(proxy, auth, req.HTTPHandler)
req.install_opener(opener)
if not compressed:
return req.urlopen(getfile)
else:
response = req.urlopen(getfile + '.gz')
data = None
if 'gzip' in response.info().get('Content-Type'):
buf = BytesIO(response.read())
data = gzip.GzipFile(fileobj=buf)
return (data, response)
def switch(self):
self.current_proxy_index += 1
if self.current_proxy_index >= len(self.urls):
self.current_proxy_index = 0
self.proxies = {
'http': 'http://' + self.urls[self.current_proxy_index]
}
proxy = request.ProxyHandler(
{'http': self.urls[self.current_proxy_index]})
auth = request.HTTPBasicAuthHandler()
opener = request.build_opener(proxy, auth, request.HTTPHandler)
request.install_opener(opener)
print('switch to ' + self.urls[self.current_proxy_index])
def getNodes(data):
try:
NODES_URL = RABBITMQ_SERVER + RABBITMQ_NODES_URI
if RABBITMQ_USERNAME and RABBITMQ_PASSWORD:
password_mgr = connector.HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(REALM, NODES_URL, RABBITMQ_USERNAME,
RABBITMQ_PASSWORD)
auth_handler = connector.HTTPBasicAuthHandler(password_mgr)
opener = connector.build_opener(auth_handler)
connector.install_opener(opener)
response = connector.urlopen(NODES_URL, timeout=10)
byte_responseData = response.read()
str_responseData = byte_responseData.decode('UTF-8')
rabbit_nodes_dict = json.loads(str_responseData)
nodes_dict = rabbit_nodes_dict[0]
if nodes_dict:
if 'mem_used' in nodes_dict:
value = convertBytesToMB(nodes_dict['mem_used'])
data['mem_used'] = value
if 'fd_used' in nodes_dict:
data['fd_used'] = nodes_dict['fd_used']
if 'run_queue' in nodes_dict:
data['run_queue'] = nodes_dict['run_queue']
if 'sockets_used' in nodes_dict:
data['sockets_used'] = nodes_dict['sockets_used']
if 'proc_used' in nodes_dict:
data['proc_used'] = nodes_dict['proc_used']
if 'processors' in nodes_dict:
data['processors'] = nodes_dict['processors']
if 'fd_total' in nodes_dict:
data['fd_total'] = nodes_dict['fd_total']
if 'sockets_total' in nodes_dict:
data['sockets_total'] = nodes_dict['sockets_total']
if 'disk_free_limit' in nodes_dict:
value = convertBytesToMB(nodes_dict['disk_free_limit'])
data['disk_free_limit'] = value
if 'partitions' in nodes_dict:
partitions = nodes_dict['partitions']
data['partitions'] = len(partitions)
except Exception as e:
data['status'] = 0
data['msg'] = str(e)
def __add_openers(self,
user=None,
password=None,
ss_cert=None,
proxy=None,
url=None):
user = user or self.user
password = password or self.password
ss_cert = ss_cert or self.ss_cert
proxy = proxy or self.proxy
handlers = list()
# HTTP Basic Auth handler
if all(
[user, password,
isinstance(user, str),
isinstance(password, str)]):
url = url or self.url
url_parse = urlparse(url)
top_level_url = '://'.join([url_parse.scheme, url_parse.netloc])
passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, top_level_url, user, password)
handlers.append(urllib2.HTTPBasicAuthHandler(passman))
self.debug("Enabling HTTP basic auth")
# HTTPS handler
# Self-signed certificate ignore
if ss_cert:
try:
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
except AttributeError:
self.error('HTTPS self-signed certificate ignore not enabled')
else:
handlers.append(urllib2.HTTPSHandler(context=ctx))
self.debug("Enabling HTTP self-signed certificate ignore")
# PROXY handler
if proxy and isinstance(proxy, str) and not ss_cert:
handlers.append(urllib2.ProxyHandler(dict(http=proxy)))
self.debug("Enabling HTTP proxy handler (%s)" % proxy)
opener = urllib2.build_opener(*handlers)
return opener
def _get_html(url, login, password):
url = url + '/admin/m1cam.cfg'
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
p = request.HTTPPasswordMgrWithDefaultRealm()
p.add_password(None, url, login, password)
ssl_handler = request.HTTPSHandler(context=ctx)
auth_handler = request.HTTPBasicAuthHandler(p)
opener = request.build_opener(ssl_handler, auth_handler)
request.install_opener(opener)
result = opener.open(url)
info = result.info()['Content-Disposition']
pattern = r'attachment; filename=\"(.*)\"'
m = re.findall(pattern, info)
fn = m[0]
return result, fn
def auth_nei_wang():
# 1，用户名密码
user = "******"
pwd = "admin123"
nei_url = "http://192.168.66.68"
# 2,创建密码管理器
pwd_manager = request.HTTPPasswordMgrWithPriorAuth()
pwd_manager.add_password(None,nei_url,user,pwd)
# 3，创建认证处理器
auth_handler = request.HTTPBasicAuthHandler(pwd_manager)
opener = request.build_opener(auth_handler)
response = opener.open(nei_url)
print(response)