c.request('GET', path, headers=myHeaders)
res = c.getresponse()
out = res.read()
print("does resource on" + iFlowName + " exist? " + filename)
code = res.getcode()
print("exists:" + str(code))
if code == 200:
return True
else:
return False
print(out)
#This sets up the https connection
c = HTTPSConnection("658fa002trial.it-cpitrial.cfapps.us10.hana.ondemand.com")
#we need to base 64 encode it
#and then decode it to acsii as python 3 stores it as a byte string
#username = sys.argv[1]
#password = sys.argv[2]
userAndPass = base64.b64encode(b"$(username):$(password)").decode("ascii")
#userAndPass = base64.b64encode(bytes(username + ':' + password, "utf-8")).decode("ascii")
headers = {'Authorization': 'Basic %s' % userAndPass, 'X-CSRF-Token': 'Fetch'}
#then connect
c.request('GET', '/api/v1/MessageProcessingLogs', headers=headers)
#get the response back
res = c.getresponse()
res.read()
xsrftoken = res.getheader("X-CSRF-Token")
print("token fetch status:" + str(res.getcode()))
myHeaders = {}
def connectOpenAPIServer():
global conn, server
conn = HTTPSConnection(server)
conn.set_debuglevel(1)
def open(readonly):
# Parse out the username from the output_conn URL.
parsed = urlparse(params['output_conn'])
username = parsed.username or "admin@internal"
# Read the password from file.
with builtins.open(params['output_password'], 'r') as fp:
password = fp.read()
password = password.rstrip()
# Connect to the server.
connection = sdk.Connection(
url=params['output_conn'],
username=username,
password=password,
ca_file=params['rhv_cafile'],
log=logging.getLogger(),
insecure=params['insecure'],
)
system_service = connection.system_service()
# Create the disk.
disks_service = system_service.disks_service()
if params['disk_format'] == "raw":
disk_format = types.DiskFormat.RAW
else:
disk_format = types.DiskFormat.COW
disk = disks_service.add(disk=types.Disk(
name=params['disk_name'],
description="Uploaded by virt-v2v",
format=disk_format,
initial_size=params['disk_size'],
provisioned_size=params['disk_size'],
# XXX Ignores params['output_sparse'].
# Handling this properly will be complex, see:
# https://www.redhat.com/archives/libguestfs/2018-March/msg00177.html
sparse=True,
storage_domains=[types.StorageDomain(name=params['output_storage'], )],
))
# Wait till the disk is up, as the transfer can't start if the
# disk is locked:
disk_service = disks_service.disk_service(disk.id)
debug("disk.id = %r" % disk.id)
endt = time.time() + timeout
while True:
time.sleep(5)
disk = disk_service.get()
if disk.status == types.DiskStatus.OK:
break
if time.time() > endt:
raise RuntimeError("timed out waiting for disk to become unlocked")
# Get a reference to the transfer service.
transfers_service = system_service.image_transfers_service()
# Create a new image transfer, using the local host is possible.
host = find_host(connection) if params['rhv_direct'] else None
transfer = transfers_service.add(
types.ImageTransfer(
disk=types.Disk(id=disk.id),
host=host,
inactivity_timeout=3600,
))
debug("transfer.id = %r" % transfer.id)
# Get a reference to the created transfer service.
transfer_service = transfers_service.image_transfer_service(transfer.id)
# After adding a new transfer for the disk, the transfer's status
# will be INITIALIZING. Wait until the init phase is over. The
# actual transfer can start when its status is "Transferring".
endt = time.time() + timeout
while True:
time.sleep(5)
transfer = transfer_service.get()
if transfer.phase != types.ImageTransferPhase.INITIALIZING:
break
if time.time() > endt:
raise RuntimeError("timed out waiting for transfer status " +
"!= INITIALIZING")
# Now we have permission to start the transfer.
if params['rhv_direct']:
if transfer.transfer_url is None:
raise RuntimeError("direct upload to host not supported, " +
"requires ovirt-engine >= 4.2 and only works " +
"when virt-v2v is run within the oVirt/RHV " +
"environment, eg. on an oVirt node.")
destination_url = urlparse(transfer.transfer_url)
else:
destination_url = urlparse(transfer.proxy_url)
context = ssl.create_default_context()
context.load_verify_locations(cafile=params['rhv_cafile'])
http = HTTPSConnection(destination_url.hostname,
destination_url.port,
context=context)
# The first request is to fetch the features of the server.
# Authentication was needed only for GET and PUT requests when
# communicating with old imageio-proxy.
needs_auth = not params['rhv_direct']
can_flush = False
can_trim = False
can_zero = False
unix_socket = None
http.request("OPTIONS", destination_url.path)
r = http.getresponse()
data = r.read()
if r.status == 200:
# New imageio never needs authentication.
needs_auth = False
j = json.loads(data)
can_flush = "flush" in j['features']
can_trim = "trim" in j['features']
can_zero = "zero" in j['features']
unix_socket = j.get('unix_socket')
# Old imageio servers returned either 405 Method Not Allowed or
# 204 No Content (with an empty body). If we see that we leave
# all the features as False and they will be emulated.
elif r.status == 405 or r.status == 204:
pass
else:
raise RuntimeError("could not use OPTIONS request: %d: %s" %
(r.status, r.reason))
debug("imageio features: flush=%r trim=%r zero=%r unix_socket=%r" %
(can_flush, can_trim, can_zero, unix_socket))
# If we are connected to imageio on the local host and the
# transfer features a unix_socket then we can reconnect to that.
if host is not None and unix_socket is not None:
try:
http = UnixHTTPConnection(unix_socket)
except Exception as e:
# Very unlikely failure, but we can recover by using the https
# connection.
debug("cannot create unix socket connection, using https: %s" % e)
else:
debug("optimizing connection using unix socket %r" % unix_socket)
# Save everything we need to make requests in the handle.
return {
'can_flush': can_flush,
'can_trim': can_trim,
'can_zero': can_zero,
'connection': connection,
'disk': disk,
'disk_service': disk_service,
'failed': False,
'highestwrite': 0,
'http': http,
'needs_auth': needs_auth,
'path': destination_url.path,
'transfer': transfer,
'transfer_service': transfer_service,
}
def find_speeches_by_year(host, this_url, print_test=False):
'''
Takes the host and a url for a given year
and returns infromation about the speeches and links to the web site
containing the text of the speeches. This function is used to create
the list of all web sites that contain the individual speeches that
need to be scraped.
INPUTS:
host the host (for the Federal Reserve 'www.federalreserve.gov)
this_url the path to the speeches for a given year
print_test an optional field that will print out summary statistics
OUTPUT:
date_lst list of speech dates
speaker_lst list of speaker names
title_lst list containing titles of speeches
link_lst list of htm links to the actual speeches
NOTES:
1. There are video links on some of the urls that we need to removed.
These videos are represented by the 'watchLive' class.
'''
conn = HTTPSConnection(host = host)
conn.request(method='GET', url = this_url)
resp = conn.getresponse()
body = resp.read()
# check that we received the correct response code
if resp.status != 200:
print('Error from Web Site! Response code: ', resp.status)
else:
soup=BeautifulSoup(body, 'html.parser')
event_list = soup.find('div', class_='row eventlist')
# creating the list of dates, titles, speakers and html articles from web page
date_lst =[]
title_lst = []
speaker_lst = []
link_lst = []
for row in event_list.find_all('div', class_='row'):
tmp_date= [x.text for x in row.find_all('time')]
date_lst.append(tmp_date)
tmp_speaker = [x.text for x in row.find_all('p', class_='news__speaker')]
speaker_lst.append(tmp_speaker)
tmp_title = [x.text for x in row.find_all('em')]
title_lst.append(tmp_title)
# some of the links include video with the transcript. We are deleteing these here
for link in event_list.find_all('a', href=True, class_ = lambda x: x != 'watchLive'):
link_lst.append(link['href'])
if print_test:
print('length of dates: ', len(date_lst))
print('length of speakers: ', len(speaker_lst))
print('length of titles: ', len(title_lst))
print('length of href: ', len(link_lst))
return date_lst, speaker_lst, title_lst, link_lst
def _request(self,
url,
method=None,
body=None,
output=None,
host=None,
token=None,
auth=False):
if method is None:
if body is None:
method = "GET"
else:
method = "PUT"
if host is None:
host = self._host
# Build our headers.
headers = {}
headers["Accept"] = "application/json"
headers["Content-Type"] = "application/json"
if token is not None:
headers["Authorization"] = "Token %s" % token
elif auth and self._auth is not None:
headers["Authorization"] = "Basic %s" % self._auth
headers["X-Docker-Token"] = "true"
# Open the requested URL.
url = "https://%s/%s" % (host, url)
con = HTTPSConnection(host)
con.request(method, url, body=body, headers=headers)
resp = con.getresponse()
if int(resp.status / 100) != 2:
# Return the given error.
raise Exception(resp.read())
if auth:
# This is a special case where we are authenticating
# with an index prior to querying the underlying registry.
token = resp.getheader("X-Docker-Token")
endpoints = resp.getheader("X-Docker-Endpoints")
if endpoints:
endpoint = random.choice(endpoints.split(","))
else:
endpoint = self._host
return (token, endpoint)
if output is not None:
# Write out to the given stream.
while True:
content = resp.read(1024 * 1024)
if not content:
break
output.write(content)
output.flush()
else:
# Interpret as JSON (since we requested it).
content = resp.read()
if content:
return json.loads(content)
def cli_tls_http_server_tests(tmp_dir):
if not check_for_command("tls_http_server"):
return
try:
from http.client import HTTPSConnection
except ImportError:
try:
from httplib import HTTPSConnection
except ImportError:
return
import ssl
server_port = random_port_number()
priv_key = os.path.join(tmp_dir, 'priv.pem')
ca_cert = os.path.join(tmp_dir, 'ca.crt')
crt_req = os.path.join(tmp_dir, 'crt.req')
server_cert = os.path.join(tmp_dir, 'server.crt')
test_cli("keygen",
["--algo=ECDSA", "--params=secp384r1", "--output=" + priv_key],
"")
test_cli("gen_self_signed", [
priv_key, "CA", "--ca", "--country=VT", "--dns=ca.example",
"--hash=SHA-384", "--output=" + ca_cert
], "")
test_cli("gen_pkcs10", "%s localhost --output=%s" % (priv_key, crt_req))
test_cli(
"sign_cert",
"%s %s %s --output=%s" % (ca_cert, priv_key, crt_req, server_cert))
tls_server = subprocess.Popen([
CLI_PATH, 'tls_http_server', '--max-clients=2',
'--port=%d' % (server_port), server_cert, priv_key
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
time.sleep(.5)
context = ssl.create_default_context(cafile=ca_cert)
conn = HTTPSConnection('localhost', port=server_port, context=context)
conn.request("GET", "/")
resp = conn.getresponse()
if resp.status != 200:
logging.error('Unexpected response status %d' % (resp.status))
body = str(resp.read())
if body.find('TLS negotiation with Botan 2.') < 0:
logging.error('Unexpected response body')
conn.request("POST", "/logout")
resp = conn.getresponse()
if resp.status != 405:
logging.error('Unexpected response status %d' % (resp.status))
if sys.version_info.major >= 3:
rc = tls_server.wait(5) # pylint: disable=too-many-function-args
else:
rc = tls_server.wait()
if rc != 0:
logging.error("Unexpected return code from https_server %d", rc)
from http.client import HTTPSConnection
from telegram import args_from
from logging import out
import json
import os
DNDBEYOND_URL = 'www.dndbeyond.com'
CONNECTION = HTTPSConnection(DNDBEYOND_URL)
CACHE_PATH = '/cache/dndbeyond-cache.json'
SET_CHARACTER_HELP = '/setcharacter <name> <id> - track a D&D Beyond character'
STATS_HELP = '/stats <name> - get character stats'
HELP = '*D&D Beyond commands*\n{}\n{}'.format(SET_CHARACTER_HELP, STATS_HELP)
def update_cache(cache):
cache_file = open(CACHE_PATH, 'w+')
cache_file.write(json.dumps(cache))
def init_cache():
if not os.path.isfile(CACHE_PATH):
cache = CACHE_SCHEMA
update_cache(cache)
else:
cache_file = open(CACHE_PATH, 'r')
cache = json.load(cache_file)
return cache
def write_rec(self, d):
common = {
'timestamp': d['timestamp'].strftime(self.dbinfo['fmtdatetime'])
}
if 'rel_time' in d:
common['rel_time'] = d['rel_time']
res = {}
slowvals = []
if 'meas_thp' in d or 'meas_sc' in d:
# Temperature
temper = {
dbkey: d[esskey]
for esskey, dbkey in self.TEMP_KEYS if esskey in d
}
if temper:
res['temperature'] = temper
# SlowValues
for uubnum in self.uubnums:
if uubnum is None:
continue
vals = {}
for keytemplate, dbname in self.SLOW_KEYS:
key = keytemplate.format(u=uubnum)
if key in d:
vals[dbname] = d[key]
if vals:
vals['uubnum'] = uubnum
slowvals.append(vals)
if slowvals:
res['slowvals'] = slowvals
# UUB data
uubdata = []
for uubnum in self.uubnums:
if uubnum is None:
continue
for typ in ('pede', 'pedemean', 'pedestdev', 'cutoff', 'noise',
'noisemean', 'noisestdev', 'gain'):
vals = self._collect(d, uubnum, typ)
if vals is not None:
uubdata.append({
'typ': self._keymap(typ),
'uubnum': uubnum,
'values': vals
})
for flabel in self.flabels:
vals = self._collect(d, uubnum, 'fgain', flabel)
if vals is not None:
uubdata.append({
'uubnum': uubnum,
'typ': 'freqgain',
'flabel': flabel,
'values': vals
})
if uubdata:
res['uubdata'] = uubdata
# send results if any
if res:
res['common'] = common
payload = json.dumps(res)
self.logger.debug('Sending results to grafana')
url = self.dbinfo['urlWriteRec'].format(runid=self.runid)
conn = HTTPSConnection(self.dbinfo['host_addr'],
port=self.dbinfo['host_port'],
context=self.sslctx)
# conn.set_debuglevel(3)
conn.request("POST",
url,
body=bytes(payload, 'ascii'),
headers={'Content-Type': 'application/json'})
resp = conn.getresponse()
self.logger.debug('Received status %d', resp.status)
if resp.status != 204:
self.logger.warning(resp.read().decode('utf-8'))
conn.close()
def test_replicate_manifest_list(docker_registry_secure: DockerRegistrySecure,
manifest_list: str):
"""Tests that manifest lists can be replicated."""
image_name = ImageName.parse(manifest_list)
# Try to retrieve credentials first ...
auth_header_src = get_auth_headers(docker_registry_secure, image_name)
if not auth_header_src:
pytest.skip("Unable to retrieve credentials for: %s",
image_name.endpoint)
# Replicate all of the manifests referenced in the manifest list ...
for image in [
"library/busybox@sha256:4fe8827f51a5e11bb83afa8227cbccb402df840d32c6b633b7ad079bc8144100",
"library/busybox@sha256:abc043b5132f825e44eefffc35535b1f24bd3f1bb60b11943863563a46795fdc",
"library/busybox@sha256:07717dd5f074de0cf4f7ca8f635cb63aef63d789f15a22ab482a3d27a0a1f881",
"library/busybox@sha256:8dfe92e22300734a185375b6316d01aa1a2b0623d425a5e6e406771ba5642bf1",
"library/busybox@sha256:3bdba83255bf7c575e31e129b2ddf1c0c32382e112cb051af6c5143c24a5ddbd",
"library/busybox@sha256:bb87f507b42a6efe6f1d5382c826f914673a065f4d777b54b52f5414d688837a",
"library/busybox@sha256:a09f03056efb5d3facb5077a9e58e83e9bba74ad4d343b2afa92c70b5ae01e2b",
"library/busybox@sha256:0b671b6a323d86aa6165883f698b557ca257c3a3ffa1e3152ffb6467e7ac11b3",
]:
img_name = ImageName.parse(image)
replicate_image(
docker_registry_secure.docker_client,
img_name,
docker_registry_secure.endpoint,
)
# Replicate the manifest list ...
replicate_manifest_list(
image_name,
docker_registry_secure.endpoint,
auth_header_dest=docker_registry_secure.auth_header,
auth_header_src=auth_header_src,
ssl_context_dest=docker_registry_secure.ssl_context,
)
media_type = "application/vnd.docker.distribution.manifest.list.v2+json"
https_connection = HTTPSConnection(
context=docker_registry_secure.ssl_context,
host=docker_registry_secure.endpoint)
if image_name.digest:
https_connection.request(
"GET",
url=f"/v2/{image_name.image}/manifests/{image_name.digest}",
headers={
"Accept": media_type,
**docker_registry_secure.auth_header
},
)
verify_http_response(https_connection, image_name, media_type)
if image_name.tag:
# TODO: Figure out why HTTPSConnection cannot perform multiple requests ?!?
https_connection = HTTPSConnection(
context=docker_registry_secure.ssl_context,
host=docker_registry_secure.endpoint,
)
https_connection.request(
"GET",
url=f"/v2/{image_name.image}/manifests/{image_name.tag}",
headers={
"Accept": media_type,
**docker_registry_secure.auth_header
},
)
verify_http_response(https_connection, image_name, media_type)
def _new_conn(self):
# Performs the NTLM handshake that secures the connection. The socket
# must be kept open while requests are performed.
self.num_connections += 1
log.debug(
"Starting NTLM HTTPS connection no. %d: https://%s%s",
self.num_connections,
self.host,
self.authurl,
)
headers = {"Connection": "Keep-Alive"}
req_header = "Authorization"
resp_header = "www-authenticate"
conn = HTTPSConnection(host=self.host, port=self.port)
# Send negotiation message
headers[
req_header] = f"NTLM {ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser)}"
log.debug("Request headers: %s", headers)
conn.request("GET", self.authurl, None, headers)
res = conn.getresponse()
reshdr = dict(res.getheaders())
log.debug("Response status: %s %s", res.status, res.reason)
log.debug("Response headers: %s", reshdr)
log.debug("Response data: %s [...]", res.read(100))
# Remove the reference to the socket, so that it can not be closed by
# the response object (we want to keep the socket open)
res.fp = None
# Server should respond with a challenge message
auth_header_values = reshdr[resp_header].split(", ")
auth_header_value = None
for s in auth_header_values:
if s[:5] == "NTLM ":
auth_header_value = s[5:]
if auth_header_value is None:
raise Exception(
f"Unexpected {resp_header} response header: {reshdr[resp_header]}"
)
# Send authentication message
ServerChallenge, NegotiateFlags = ntlm.parse_NTLM_CHALLENGE_MESSAGE(
auth_header_value)
auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE(
ServerChallenge, self.user, self.domain, self.pw, NegotiateFlags)
headers[req_header] = f"NTLM {auth_msg}"
log.debug("Request headers: %s", headers)
conn.request("GET", self.authurl, None, headers)
res = conn.getresponse()
log.debug("Response status: %s %s", res.status, res.reason)
log.debug("Response headers: %s", dict(res.getheaders()))
log.debug("Response data: %s [...]", res.read()[:100])
if res.status != 200:
if res.status == 401:
raise Exception(
"Server rejected request: wrong username or password")
raise Exception(
f"Wrong server response: {res.status} {res.reason}")
res.fp = None
log.debug("Connection established")
return conn
from flask import Flask, render_template, request, url_for
import urllib
import json
from http.client import HTTPSConnection
app = Flask(__name__)
headers = {
'Authorization': "Basic YWRtaW46TGYwYll4dGo4UTdt",
'X-ID-TENANT-NAME': "nyc093",
'Content-Type': 'application/json',
'Accept': 'application/json'
}
#This sets up the https connection
c = HTTPSConnection("yuuvis.io")
def build(word):
host = "/api/dms/objects/search"
payload = {
"query": {
"statement":
"SELECT * FROM enaio:object where contains('%s')" % word,
"skipCount": 0,
"maxItems": 50
}
}
json_load = json.dumps(payload)
c.request('POST', host, json_load, headers=headers)
#get the response back
res = c.getresponse()
def tryhttpsconnectionwithcontext(url):
conn = HTTPSConnection(urlparse(url).netloc,
context=ssl.create_default_context())
conn.request("GET", "/")
return conn.getresponse().read()
def tryhttpsconnection(url):
conn = HTTPSConnection(urlparse(url).netloc)
conn.request("GET", "/")
return conn.getresponse().read()
def Image_DayMovie(TitleText):
global DayMovieDoc
global MovieChart
global ChartFlag
ImgURL = []
image = []
for i in range(0, 5):
if ChartFlag == True:
server = "openapi.naver.com"
client_id = "iEV22cE2b1ZJnyGDYXtc"
client_secret = "xJB_PnOFmw"
text = urllib.parse.quote(TitleText[i])
conn = HTTPSConnection(server)
conn.request(
"GET",
"/v1/search/movie.xml?query=" + text + "&display=10&start=1",
None, {
"X-Naver-Client-Id": client_id,
"X-Naver-Client-Secret": client_secret
})
req = conn.getresponse()
if int(req.status) == 200:
ImageDoc = parseString(req.read())
ImgURL.append(getJpgURL(ImageDoc, TitleText[i]))
else:
server = "openapi.naver.com"
client_id = "iEV22cE2b1ZJnyGDYXtc"
client_secret = "xJB_PnOFmw"
text = urllib.parse.quote(TitleText[i + 5])
conn = HTTPSConnection(server)
conn.request(
"GET",
"/v1/search/movie.xml?query=" + text + "&display=10&start=1",
None, {
"X-Naver-Client-Id": client_id,
"X-Naver-Client-Secret": client_secret
})
req = conn.getresponse()
if int(req.status) == 200:
ImageDoc = parseString(req.read())
ImgURL.append(getJpgURL(ImageDoc, TitleText[i + 5]))
for i in range(0, len(ImgURL)):
#print(ImgURL[i])
if ImgURL[i] != None:
with urllib.request.urlopen(ImgURL[i]) as u:
raw_data = u.read()
tmp = Image.open(BytesIO(raw_data))
tmp = tmp.resize((160, 208), Image.ANTIALIAS)
tmp = ImageTk.PhotoImage(tmp)
image.append(tmp)
JPG = Label(MovieChart, image=image[i], bg="Black")
JPG.place(x=55 + (180 * i), y=100)
else:
image.append(None)
MovieChart.mainloop()
def check_url(self, url, frags=None, local_html=None):
"""Check a url
Returns (final status code(s) in tuple of string, final redirect url)
>>> c = Checker()
>>> c.check_url('http://example.com') # doctest: +SKIP
(('200',), 'http://www.iana.org/domains/example')
"""
MAX_REDIRS = 10
redirs = 0
method = 'HEAD'
start_url = url
status = ''
statuses = []
while not status:
if url.startswith('//'):
url = 'http:' + url
url_comp = urlparse(url)
if url_comp.scheme == 'http':
conn = HTTPConnection(url_comp.netloc)
elif url_comp.scheme == 'https':
if self.unverified_certificates:
context = ssl._create_unverified_context()
else:
context = ssl.create_default_context()
conn = HTTPSConnection(url_comp.netloc, context=context)
else:
if not url and frags and local_html:
pairs = product(frags, (local_html, ))
statuses = tuple(map(self._check_url_frag, pairs))
break
status = 'SCH'
if url_comp.scheme in ('about', 'javascript'):
status = 'SKP'
break
try:
# attempt to decide whether lnkckr should quote the URL.
# if there is '%' in URL, then don't quote it.
p = url_comp.path if '%' in url_comp.path else quote(
url_comp.path)
if url_comp.query:
p += '?' + url_comp.query
if frags:
method = 'GET'
conn.request(method, p, headers=self.HEADERS)
resp = conn.getresponse()
if resp.status == 200 and frags != ('', ):
# any non text/html result ### as such fragment isn't found
if not resp.getheader('Content-Type',
'').startswith('text/html'):
status = '###'
break
# assume characters in fragment are valid ASCII
rbody = resp.read().decode('ascii', 'ignore')
pairs = product(frags, (rbody, ))
statuses = tuple(map(self._check_url_frag, pairs))
break
elif 300 <= resp.status < 400:
if redirs >= MAX_REDIRS:
status = 'RRR'
else:
redirs += 1
url = urljoin(url, resp.getheader('location'))
method = 'HEAD'
elif resp.status == 405 and method == 'HEAD':
method = 'GET'
else:
status = str(resp.status)
conn.close()
except socket.error:
status = '000'
except Exception:
traceback.print_exc()
status = 'XXX'
if start_url == url and not redirs:
url = None
if not statuses:
statuses = (status, ) * len(frags)
return statuses, url
def upload_document(path,
progress_tracker=None,
file_id=0,
post_url='/upload_file3'):
"""
:param path:
:param progress_tracker:
:param file_id: 0 if uploading a new document, not 0 if repplacing an existing document.
:param post_url:
:return:
"""
mr = None
if progress_tracker:
mr = Wrap(progress_tracker)
else:
mr = MultiRead()
mr.add_field('file_id',
str(file_id)) # 0 == It's a new document, > 0 == overwrite
# We store the filename here 'cos the one encoded in the file part
# must be ASCII (IETF, RFC 2183, section 2.3 : Current [RFC 2045] grammar
# restricts parameter values (and hence Content-Disposition filenames)
# to US-ASCII. We recognize the great desirability of allowing
# arbitrary character sets in filenames, but it is beyond the
# scope of this document to define the necessary mechanisms.
mr.add_field('encoding_safe_filename', os.path.split(path)[-1])
mr.add_file_part(path)
mr.close_parts()
host, port = extract_host_port(
configuration.get("DownloadSite", "base_url"))
mainlog.debug(
u"Upload to {}:{}{} (determined from DownloadSite/base_url : {})".
format(host, port, post_url,
configuration.get("DownloadSite", "base_url")))
if configuration.get("DownloadSite", "base_url").startswith('https'):
h = HTTPSConnection(host, port)
else:
h = HTTPConnection(host, port)
h.putrequest('POST', post_url)
h.putheader('content-type', mr.content_type())
h.putheader('content-length', str(mr.total_size()))
h.putheader('x-filesize', str(mr.total_size()))
h.endheaders()
mr.open()
h.send(mr)
mr.close()
server_response = h.getresponse()
if server_response.status == OK:
server_response.getheaders(
) # Skip headers (is this really necessary ?)
t = server_response.read()
file_id = int(t)
mainlog.debug("Successfully uploaded {} bytes".format(mr.total_size()))
h.close()
return file_id
else:
raise Exception(
"Unable to upload, server response status was {}".format(
server_response.status))
def create_channel(self, method, headers):
if self._rpcProxyUrl.scheme == 'http':
self.__channels[method] = HTTPConnection(self._rpcProxyUrl.netloc)
else:
try:
uv_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
self.__channels[method] = HTTPSConnection(
self._rpcProxyUrl.netloc, context=uv_context)
except AttributeError:
self.__channels[method] = HTTPSConnection(
self._rpcProxyUrl.netloc)
auth = ntlm.getNTLMSSPType1(domain=self.__domain)
auth_headers = headers.copy()
auth_headers['Content-Length'] = '0'
auth_headers['Authorization'] = b'NTLM ' + base64.b64encode(
auth.getData())
self.__channels[method].request(method,
self._rpcProxyUrl.path,
headers=auth_headers)
res = self.__channels[method].getresponse()
res.read()
if res.status != 401:
raise RPCProxyClientException(
'Status code returned: %d. Authentication does not seem required for url %s'
% (res.status, self._rpcProxyUrl.path))
if res.getheader('WWW-Authenticate') is None:
raise RPCProxyClientException(
'No authentication requested by the server for url %s' %
self._rpcProxyUrl.path)
if 'NTLM' not in res.getheader('WWW-Authenticate'):
raise RPCProxyClientException(
'NTLM Auth not offered by URL, offered protocols: %s' %
res.getheader('WWW-Authenticate'))
try:
serverChallengeBase64 = re.search(
'NTLM ([a-zA-Z0-9+/]+={0,2})',
res.getheader('WWW-Authenticate')).group(1)
serverChallenge = base64.b64decode(serverChallengeBase64)
except (IndexError, KeyError, AttributeError):
raise RPCProxyClientException(
'No NTLM challenge returned from server for url %s' %
self._rpcProxyUrl.path)
# Default ACL in HKLM\SOFTWARE\Microsoft\Rpc\ValidPorts allows connections only by NetBIOS name of the server.
# If remoteName is empty we assume the target is the rpcproxy server, and get its NetBIOS name from NTLMSSP.
#
# Interestingly, if Administrator renames the server, the ACL remains the original.
if not self.__ntlmssp_info:
challenge = ntlm.NTLMAuthChallenge(serverChallenge)
self.__ntlmssp_info = ntlm.AV_PAIRS(challenge['TargetInfoFields'])
if not self.__remoteName:
self.__remoteName = self.__ntlmssp_info[
ntlm.NTLMSSP_AV_HOSTNAME][1].decode('utf-16le')
self._stringbinding.set_network_address(self.__remoteName)
if not self._rpcProxyUrl.query:
query = self.__remoteName + ':' + str(self.__dstport)
self._rpcProxyUrl = self._rpcProxyUrl._replace(query=query)
type3, exportedSessionKey = ntlm.getNTLMSSPType3(
auth, serverChallenge, self.__username, self.__password,
self.__domain, self.__lmhash, self.__nthash)
headers['Authorization'] = b'NTLM ' + base64.b64encode(type3.getData())
self.__channels[method].request(method,
self._rpcProxyUrl.path + '?' +
self._rpcProxyUrl.query,
headers=headers)
auth_resp = self.__channels[method].sock.recv(8192)
if auth_resp != b'HTTP/1.1 100 Continue\r\n\r\n':
try:
auth_resp = auth_resp.split(b'\r\n')[0].decode(
"utf-8", errors='replace')
raise RPCProxyClientException(
'RPC Proxy authentication failed in %s channel' % method,
proxy_error=auth_resp)
except (IndexError, KeyError, AttributeError):
raise RPCProxyClientException(
'RPC Proxy authentication failed in %s channel' % method)
class IuguRequests(IuguApi):
"""
All request to API pass by here. Use the HTTP verbs for each request. For
each method (get, post, put and delete) is need an URN and a list of fields
its passed as list of tuples that is encoded by urlencode (e.g:
[("field_api", "value")]
URN: is relative path of URL http://api.iugu.com/ARG1/ARG2 where
URN = "/ARG1/ARG2"
All methods appends an api_token that is encoded in url params. The
api_token is given in config.py its useful to work in sandbox mode.
:method get: make a GET request
:method post: make a POST request
:method put: make a PUT request
:method delete: make a DELETE request
"""
headers = {"Content-Type": "application/x-www-form-urlencoded"}
__conn = HTTPSConnection(config.API_HOSTNAME) # not put in instance
__conn.timeout = 10
def __init__(self, **options):
super(IuguRequests, self).__init__(**options)
if self.is_debug():
# set debuglevel to HTTPSConnection
self.__conn.set_debuglevel(2)
def __validation(self, response, msg=None):
"""
Validates if data returned by API contains errors json. The API returns
by default a json with errors as field {errors: XXX}
=> http://iugu.com/referencias/api#erros
"""
results = json_load(response)
try:
err = results["errors"]
except:
err = None
if err:
raise errors.IuguGeneralException(value=err)
else:
return results
def __reload_conn(self):
"""
Wrapper to keep TCP connection ESTABLISHED. Rather the connection go to
CLOSE_WAIT and raise errors CannotSendRequest or the server reply with
empty and it raise BadStatusLine
"""
self.__conn = HTTPSConnection(config.API_HOSTNAME) # reload
self.__conn.timeout = 10
def __conn_request(self, http_verb, urn, params):
"""
Wrapper to request/response of httplib's context, reload a
connection if presume that error will occurs and returns the response
"""
try:
self.__conn.request(http_verb, urn, params, self.headers)
except CannotSendRequest:
self.__reload_conn()
self.__conn.request(http_verb, urn, params, self.headers)
try:
response = self.__conn.getresponse()
except (IOError, BadStatusLine):
self.__reload_conn()
self.__conn.request(http_verb, urn, params, self.headers)
response = self.__conn.getresponse()
return response
def get(self, urn, fields):
fields.append(("api_token", self.API_TOKEN))
params = urlencode(fields, True)
response = self.__conn_request("GET", urn, params)
return self.__validation(response)
def post(self, urn, fields):
fields.append(("api_token", self.API_TOKEN))
params = urlencode(fields, True)
response = self.__conn_request("POST", urn, params)
return self.__validation(response)
def put(self, urn, fields):
fields.append(("api_token", self.API_TOKEN))
params = urlencode(fields, True)
response = self.__conn_request("PUT", urn, params)
return self.__validation(response)
def delete(self, urn, fields):
fields.append(("api_token", self.API_TOKEN))
params = urlencode(fields, True)
response = self.__conn_request("DELETE", urn, params)
return self.__validation(response)
def cli_tls_proxy_tests(tmp_dir):
# pylint: disable=too-many-locals,too-many-statements
if not check_for_command("tls_proxy"):
return
try:
from http.client import HTTPSConnection
except ImportError:
try:
from httplib import HTTPSConnection
except ImportError:
return
try:
from http.server import HTTPServer, BaseHTTPRequestHandler
except ImportError:
try:
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
except ImportError:
return
import ssl
import threading
server_port = random_port_number()
proxy_port = random_port_number()
while server_port == proxy_port:
proxy_port = random_port_number()
priv_key = os.path.join(tmp_dir, 'priv.pem')
ca_cert = os.path.join(tmp_dir, 'ca.crt')
crt_req = os.path.join(tmp_dir, 'crt.req')
server_cert = os.path.join(tmp_dir, 'server.crt')
test_cli("keygen",
["--algo=ECDSA", "--params=secp384r1", "--output=" + priv_key],
"")
test_cli("gen_self_signed", [
priv_key, "CA", "--ca", "--country=VT", "--dns=ca.example",
"--hash=SHA-384", "--output=" + ca_cert
], "")
test_cli("gen_pkcs10", "%s localhost --output=%s" % (priv_key, crt_req))
test_cli(
"sign_cert",
"%s %s %s --output=%s" % (ca_cert, priv_key, crt_req, server_cert))
tls_proxy = subprocess.Popen([
CLI_PATH, 'tls_proxy',
str(proxy_port), '127.0.0.1',
str(server_port), server_cert, priv_key, '--output=/tmp/proxy.err',
'--max-clients=2'
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
time.sleep(.5)
server_response = binascii.hexlify(os.urandom(32))
def run_http_server():
class Handler(BaseHTTPRequestHandler):
def do_GET(self): # pylint: disable=invalid-name
self.send_response(200)
self.end_headers()
self.wfile.write(server_response)
httpd = HTTPServer(('', server_port), Handler)
httpd.serve_forever()
http_thread = threading.Thread(target=run_http_server)
http_thread.daemon = True
http_thread.start()
time.sleep(.5)
context = ssl.create_default_context(cafile=ca_cert)
for _i in range(2):
conn = HTTPSConnection('localhost', port=proxy_port, context=context)
conn.request("GET", "/")
resp = conn.getresponse()
if resp.status != 200:
logging.error('Unexpected response status %d' % (resp.status))
body = resp.read()
if body != server_response:
logging.error('Unexpected response from server %s' % (body))
if sys.version_info.major >= 3:
rc = tls_proxy.wait(5) # pylint: disable=too-many-function-args
else:
rc = tls_proxy.wait()
if rc != 0:
logging.error('Unexpected return code %d', rc)
def perform_discovery(app: str, user: str, passwd: str, prefix: str,
device_filter: str, properties_filter: bool) -> dict:
if app in SECRET_ID_MAP:
app_prefix = SECRET_ID_MAP[app]
else:
app_prefix = 'a-Hisense-{}-field'.format(app)
if app in SECRET_ID_EXTRA_MAP:
app_id = '-'.join((app_prefix, SECRET_ID_EXTRA_MAP[app], 'id'))
else:
app_id = '-'.join((app_prefix, 'id'))
secret = base64.b64encode(
SECRET_MAP[app]).decode('utf-8').rstrip('=').replace('+', '-').replace(
'/', '_')
app_secret = '-'.join((app_prefix, secret))
# Extract the region from the app ID (and fallback to US)
region = app[-2:]
if region not in AYLA_USER_SERVERS:
region = 'us'
user_server = AYLA_USER_SERVERS[region]
devices_server = AYLA_DEVICES_SERVERS[region]
ssl_context = ssl.SSLContext()
ssl_context.verify_mode = ssl.CERT_NONE
ssl_context.check_hostname = False
ssl_context.load_default_certs()
access_token = _sign_in(user, passwd, user_server, app_id, app_secret,
ssl_context)
result = []
conn = HTTPSConnection(devices_server, context=ssl_context)
headers = {
'Accept': 'application/json',
'Connection': 'Keep-Alive',
'Authorization': 'auth_token ' + access_token,
'User-Agent': _USER_AGENT,
'Host': devices_server,
'Accept-Encoding': 'gzip'
}
devices = _get_devices(devices_server, access_token, headers, conn)
logging.debug('Found devices: %r', devices)
for device in devices:
device_data = device['device']
if device_filter and device_filter != device_data['product_name']:
continue
dsn = device_data['dsn']
lanip = _get_lanip(dsn, headers, conn)
properties_text = ''
if properties_filter:
props = _get_device_properties(dsn, headers, conn)
device_data['properties'] = props
properties_text = 'Properties:\n%s', json.dumps(props, indent=2)
print(
'Device {} has:\nIP address: {}\nlanip_key: {}\nlanip_key_id: {}\n{}\n'
.format(device_data['product_name'], device_data['lan_ip'],
lanip['lanip_key'], lanip['lanip_key_id'],
properties_text))
device_data['lanip_key'] = lanip['lanip_key']
device_data['lanip_key_id'] = lanip['lanip_key_id']
result.append(device_data)
conn.close()
return result
class Mqtt4App(Client):
"""Classe de monitoramento e envio de informacoes ao back4app."""
BACK4APP_CON = HTTPSConnection(host="parseapi.back4app.com", port=443)
def __init__(self, back_db_name=None, back_app_id=None, back_rest_id=None,
topics=[], qos=0, *args, **kwargs):
"""Atributos essenciais para o projeto.
Parameters
----------
back_db_name: str
Nome do banco de dados back4app.
back_app_id: str
Chave app ID disponibilizada pelo back4app.
back_rest_id: str
Chave rest ID disponibilizada pelo back4app.
"""
super().__init__(*args, **kwargs)
self.back_app_id = back_app_id
self.back_rest_id = back_rest_id
self.back_db_name = back_db_name
self.qos = qos
self.topics = self._convert_to_tuple_list(topics)
self.con_id = self.get_json_connection_id()
self.classpath = self._get_class_path()
def _convert_to_tuple_list(self, topics):
"""Modificar uma lista de topicos para uma lista de tuplas."""
new_topics = []
for topic in topics:
new_topics.append((topic, self.qos))
return new_topics
def start_connection(self, address, port):
"""Inicia conexao com o Broker.
Parameters
----------
address: str
Endereco do servidor Broker.
"""
msg = f"Conexao com {address}:{port} estabelecida com sucesso."
self.connect(address, port)
self.loop_forever()
def on_connect(self, client, userdata, flags, rc):
"""Realiza a inscricao de topicos."""
msg = (
f"{len(self.topics)} foram cadastrados "
f"na conexao {client._host}."
)
client.subscribe(self.topics)
def on_disconnect(self, client, userdata, rc):
"""Realiza a reconexao, caso se torne indisponivel."""
msg = f"A conexao com {self._host} foi perdida, reconectando..."
client.reconnect()
def on_message(self, client, userdata, msg):
"""Manipula a mensagem e o topico."""
payload = float(msg.payload)
topic = msg.topic
print(topic, payload)
json_data = self.get_json_data(payload, topic)
self._send_data_to_back4app(json_data)
def get_json_data(self, payload, topic):
"""Gerar JSON com a estrutura de envio para o trabalho.
Parameters
----------
payload: str
Mensagem obtida pelo protocolo.
topic: str
Topico obtido pelo protocolo.
Returns
----------
str
Mensagem convertida para o formato JSON.
"""
return json.dumps(
{
"topic": f"{topic}",
"payload": payload
}
)
def get_json_connection_id(self):
"""Informacoes essenciais para a conexao com o Back4App.
Returns
----------
dict
Cabecalho com dados essenciais para a conexao com o back4app.
"""
return {
"X-Parse-Application-Id": self.back_app_id,
"X-Parse-REST-API-Key": self.back_rest_id,
"Content-Type": "application/json"
}
def _get_class_path(self):
"""Obter o caminho do banco de dados no back4app.
Returns
----------
str
Caminho do banco de dados criado no back4app.
"""
return f'/classes/{self.back_db_name}/'
def _send_data_to_back4app(self, data):
"""Realizar conexao com o back4app.
Parameters
----------
classpath: str
Localizacao do banco NoSQL.
data: str
JSON com informacoes relacionadas ao topico.
ex: '{"topico": "abc", "valor": 123}'
"""
back_ids = self.back_app_id and self.back_rest_id
if not back_ids or not self.back_db_name:
return
try:
self.BACK4APP_CON.connect()
self.BACK4APP_CON.request(
'POST',
self.classpath,
data,
self.con_id
)
response = json.loads(self.BACK4APP_CON.getresponse().read())
print(response)
except ConnectionRefusedError as err:
print("A conexao foi recusada: ", err)
def client_factory():
ctx = ssl.create_default_context(cafile="./unittest.crt")
ctx.check_hostname = False
return MagicMock(return_value=HTTPSConnection(
host="localhost", port=50510, timeout=10, context=ctx))
def _new_conn(self):
# Performs the NTLM handshake that secures the connection. The socket
# must be kept open while requests are performed.
self.num_connections += 1
log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s',
self.num_connections, self.host, self.authurl)
headers = {}
headers['Connection'] = 'Keep-Alive'
req_header = 'Authorization'
resp_header = 'www-authenticate'
conn = HTTPSConnection(host=self.host, port=self.port)
# Send negotiation message
headers[req_header] = (
'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser))
log.debug('Request headers: %s', headers)
conn.request('GET', self.authurl, None, headers)
res = conn.getresponse()
reshdr = dict(res.getheaders())
log.debug('Response status: %s %s', res.status, res.reason)
log.debug('Response headers: %s', reshdr)
log.debug('Response data: %s [...]', res.read(100))
# Remove the reference to the socket, so that it can not be closed by
# the response object (we want to keep the socket open)
res.fp = None
# Server should respond with a challenge message
auth_header_values = reshdr[resp_header].split(', ')
auth_header_value = None
for s in auth_header_values:
if s[:5] == 'NTLM ':
auth_header_value = s[5:]
if auth_header_value is None:
raise Exception('Unexpected %s response header: %s' %
(resp_header, reshdr[resp_header]))
# Send authentication message
ServerChallenge, NegotiateFlags = \
ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value)
auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE(
ServerChallenge, self.user, self.domain, self.pw, NegotiateFlags)
headers[req_header] = 'NTLM %s' % auth_msg
log.debug('Request headers: %s', headers)
conn.request('GET', self.authurl, None, headers)
res = conn.getresponse()
log.debug('Response status: %s %s', res.status, res.reason)
log.debug('Response headers: %s', dict(res.getheaders()))
log.debug('Response data: %s [...]', res.read()[:100])
if res.status != 200:
if res.status == 401:
raise Exception('Server rejected request: wrong '
'username or password')
raise Exception('Wrong server response: %s %s' %
(res.status, res.reason))
res.fp = None
log.debug('Connection established')
return conn
import shutil
from base64 import b64encode
from http.client import HTTPSConnection
import pathlib
import yaml
import requests
from dotenv import load_dotenv
load_dotenv()
email = os.environ['EMAIL']
password = os.environ['PASSWORD']
c = HTTPSConnection("www.paprikaapp.com")
userAndPass = b64encode(bytes(email + ":" + password, 'utf-8')).decode("ascii")
headers = {'Authorization': 'Basic %s' % userAndPass}
def check_and_run():
pathlib.Path('_data').mkdir(parents=True, exist_ok=True)
try:
with open(r'./_data/recipes_status.json', 'rb') as file:
old_data = file.read()
except IOError as error:
open(r'./_data/recipes_status.json', 'wb+').close()
old_data = "{}"
c.request('GET', '/api/v1/sync/status/', headers=headers)
res = c.getresponse()
def request(method,
urlString,
body='',
headers={},
auth=None,
verbose=False,
https_proxy=os.getenv('https_proxy', None),
timeout=60):
url = urlparse(urlString)
if url.scheme == 'http':
conn = HTTPConnection(url.netloc, timeout=timeout)
else:
if httpRequestProps['secure'] or not hasattr(
ssl, '_create_unverified_context'):
conn = HTTPSConnection(
url.netloc if https_proxy is None else https_proxy,
timeout=timeout)
else:
conn = HTTPSConnection(
url.netloc if https_proxy is None else https_proxy,
context=ssl._create_unverified_context(),
timeout=timeout)
if https_proxy:
conn.set_tunnel(url.netloc)
if auth is not None:
auth = base64.b64encode(auth.encode()).decode()
headers['Authorization'] = 'Basic %s' % auth
if verbose:
print('========')
print('REQUEST:')
print('%s %s' % (method, urlString))
print('Headers sent:')
print(getPrettyJson(headers))
if body != '':
print('Body sent:')
print(body)
try:
conn.request(method, urlString, body, headers)
res = conn.getresponse()
body = ''
try:
body = res.read()
except IncompleteRead as e:
body = e.partial
# patch the read to return just the body since the normal read
# can only be done once
res.read = lambda: body
if verbose:
print('--------')
print('RESPONSE:')
print('Got response with code %s' % res.status)
print('Body received:')
print(res.read())
print('========')
return res
except socket.timeout:
return ErrorResponse(status=500,
error='request timed out at %d seconds' % timeout)
except Exception as e:
return ErrorResponse(status=500, error=str(e))
def request(method,
urlString,
body='',
headers={},
auth=None,
verbose=False,
https_proxy=os.getenv('https_proxy', None),
timeout=60):
url = urlparse(urlString)
if url.scheme == 'http':
conn = HTTPConnection(url.netloc, timeout=timeout)
else:
if httpRequestProps['secure'] or not hasattr(
ssl, '_create_unverified_context'):
conn = HTTPSConnection(
url.netloc if https_proxy is None else https_proxy,
timeout=timeout)
else:
conn = HTTPSConnection(
url.netloc if https_proxy is None else https_proxy,
context=ssl._create_unverified_context(),
timeout=timeout)
if https_proxy:
conn.set_tunnel(url.netloc)
if auth is not None:
auth = base64.b64encode(auth.encode()).decode()
headers['Authorization'] = 'Basic %s' % auth
if verbose:
print('========')
print('REQUEST:')
print('%s %s' % (method, urlString))
print('Headers sent:')
print(getPrettyJson(headers))
if body != '':
print('Body sent:')
print(body)
try:
# Add random network delay
time.sleep(random.expovariate(40))
conn.request(method, urlString, body, headers)
# read reuse buffer
global rb
body_store = body
if USE_REUSE_BUFFER:
l.acquire()
matched_item = rb.loc[(rb['Url'] == urlString)
& (rb['Body'] == body_store)]
l.release()
if len(matched_item.index) > 0:
# print("Find Matched!")
if len(matched_item.index) > 1:
raise Exception("Two matched items in edge reuse buffer!!")
res = matched_item["Output"]
t_end = time.time()
global hit_count
hit_count += 1
if hit_count % 20 == 0:
print("Buffer Usage: " + str(rb.memory_usage().sum()) +
"/" + str(MAX_BUFFER_SIZE))
print("Hit Times: " + str(hit_count))
return res
# Get response
res = conn.getresponse()
body = ''
try:
body = res.read()
except IncompleteRead as e:
body = e.partial
# patch the read to return just the body since the normal read
# can only be done once
res.read = lambda: body
if verbose:
print('--------')
print('RESPONSE:')
print('Got response with code %s' % res.status)
print('Body received:')
print(res.read())
print('========')
# store Reuse Buffer
if USE_REUSE_BUFFER:
l.acquire()
# Need to judge whether there is matched item again before writing
matched_item = rb.loc[(rb['Url'] == urlString)
& (rb['Body'] == body_store)]
if len(matched_item.index) > 0:
l.release()
return res.read().decode()
rb = rb.append(
{
'Url': urlString,
'Body': body_store,
'Output': res.read().decode()
},
ignore_index=True)
# Need to judge whether to kick unused items out
# If the buffer is full, kick the first a few elements out of the buffer
if (not rb.empty) and rb.memory_usage().sum() > MAX_BUFFER_SIZE:
idx = 0
while (not rb.empty
) and rb[idx:].memory_usage().sum() > MAX_BUFFER_SIZE:
idx += 1
rb = rb[idx:]
l.release()
return res.read().decode()
except socket.timeout:
return ErrorResponse(status=500,
error='request timed out at %d seconds' % timeout)
except Exception as e:
return ErrorResponse(status=500, error=str(e))
print()
clusterIP = input("Enter the target cluster IP: ")
print()
cvmUserID = input("Enter cluster administrator ID: ")
print()
cvmPassword = input(" Enter cluster administrator password: "******"Enter Prism Central IP: ")
print()
pcUserID = input("Enter Prism Central administrator ID: ")
print()
pcPassword = input("Enter Prism Central administrator password: "******":" + (pcPassword)
buserAndPass = b64encode(userpass.encode("ascii"))
authKey = (buserAndPass.decode("ascii"))
headers = {
'Content-Type': "application/json",
'Authorization': "Basic " + authKey,
'cache-control': "no-cache"
}
# # Defines base url for API calls
def execute(self):
# Build the request URL
url = self.api_root + self.path
if len(self.parameters):
url = '%s?%s' % (url, urlencode(self.parameters))
# Query the cache if one is available
# and this request uses a GET method.
if self.use_cache and self.api.cache and self.method == 'GET':
cache_result = self.api.cache.get(url)
# if cache result found and not expired, return it
if cache_result:
# must restore api reference
if isinstance(cache_result, list):
for result in cache_result:
if isinstance(result, Model):
result._api = self.api
else:
if isinstance(cache_result, Model):
cache_result._api = self.api
return cache_result
# Continue attempting request until successful
# or maximum number of retries is reached.
retries_performed = 0
while retries_performed < self.retry_count + 1:
# Open connection
# FIXME: add timeout
if self.api.secure:
if self.api.proxy_host:
conn = HTTPSConnection(self.api.proxy_host,
self.api.proxy_port)
else:
conn = HTTPSConnection(self.host)
else:
if self.api.proxy_host:
conn = HTTPConnection(self.api.proxy_host,
self.api.proxy_port)
else:
conn = HTTPConnection(self.host)
# Apply authentication
if self.api.auth:
self.api.auth.apply_auth(self.scheme + self.host + url,
self.method, self.headers,
self.parameters)
#MAJOR HACK FOR PYTHON 3
#FIXME: Find a better way.
if PY_MAJOR_VERSION == 3:
head_auth = self.headers['Authorization'].split(',')
# Execute request
try:
_url = url
if self.api.proxy_host:
_url = self.scheme + self.host + url
if self.method == "POST":
if self.post_data:
self.headers["Content-Length"] = len(
self.post_data)
else:
self.headers["Content-Length"] = "0"
conn.request(self.method,
_url,
headers=self.headers,
body=self.post_data)
resp = conn.getresponse()
except Exception as e:
raise TweepError('Failed to send request: %s' % e)
# Exit request loop if non-retry error code
if self.retry_errors:
if resp.status not in self.retry_errors: break
else:
if resp.status == 200: break
# Sleep before retrying request again
time.sleep(self.retry_delay)
retries_performed += 1
# If an error was returned, throw an exception
self.api.last_response = resp
if resp.status != 200:
try:
error_msg = self.api.parser.parse_error(resp.read())
except Exception:
error_msg = "Twitter error response: status code = %s" % resp.status
raise TweepError(error_msg, resp)
# Parse the response payload
result = self.api.parser.parse(self, resp)
conn.close()
# Store result into cache if one is available.
if self.use_cache and self.api.cache and self.method == 'GET' and result:
self.api.cache.store(url, result)
return result
# -*- coding:utf-8 -*-
from urllib.parse import quote
from http.client import HTTPSConnection
from getNaverNews import resBody
from bs4 import BeautifulSoup
# 웹사이트 소스
w = quote("천창원")
huc = HTTPSConnection("book.naver.com")
huc.request(
"GET", "/search/search.nhn?sm=sta_hty.book&sug=&where=nexearch&query=" + w)
resBody = huc.getresponse().read()
print(resBody.decode())
#############################################
# HTML 파싱
# Java : jsoup.jar
# Python : bs4.py(BeautifulSoup)
# cmd
# pip3 install bs4
# 내장된HTML파서이름
naverBook = BeautifulSoup(resBody, "html.parser", from_encoding="utf-8")
# naverBook.select("CSS선택자")
books = naverBook.select("#searchBiblioList li")
for b in books:
print(b.select("dl a")[0].text) # 제목
print(b.select("dl a")[1].text) # 저자
print(b.select("dd")[2].text.strip()) # desc strip() 문자열 앞뒤공백 없애기
print("------")
def _test_connection_to_URL(self, url_str):
"""
Internal function that tests the connection to a URL via a simple
HTTP/1.1 HEAD request. If a test connection cannot be established,
raises an exception with a helpful diagnostic message (e.g. the
HTTP Response status code & reason).
:param url_str: full string for the URL to test a connection to
:type url_str: str
:return: the value of the HTTP Response 'Content-Type' header,
or None if no successful test connection was made
"""
content_type_header = None
try:
url = urlparse(url_str)
test_conn = None
if url.scheme == 'https':
test_conn = HTTPSConnection(url.netloc)
elif url.scheme == 'http':
test_conn = HTTPConnection(url.netloc)
else:
raise RuntimeError(" Error: Cannot connect to URL '{}'\n" \
" Unsupported network protocol scheme " \
"'{}' specified\n" \
" Supported schemes: http | https".format(
url_str, url.scheme))
test_conn.request('HEAD', url.path)
response = test_conn.getresponse()
content_type_header = response.getheader('Content-Type')
if response.status != OK: # i.e. "HTTP/1.1 200 OK"
raise RuntimeError(" Error: Cannot connect to URL '{}'\n" \
" HTTP Response status code & reason: " \
"{} {}".format(url_str, response.status,
response.reason))
except ValueError as ve: # Raised by urlparse(...)
raise RuntimeError(" Error: Cannot connect to URL '{}'\n" \
" URL string may be invalid.".format(url_str))
except HTTPException as he: # Raised by HTTPSConnection(...)
# or HTTPConnection(...)
raise RuntimeError(" Error: Cannot connect to URL '{}'\n" \
" HTTPException: {}\n" \
" url.scheme : '{}'\n" \
" url.netloc : '{}'\n" \
" url.path : '{}'\n".format(
url_str, str(he),
url.scheme, url.netloc, url.path))
except OSError as oe: # Catches any low-level socket errors
# raised by test_conn.request(...)
# -> _socket (socketmodule.c)
raise RuntimeError(" Error: Cannot connect to URL '{}'\n" \
" Socket Error: {}: {}\n" \
" url.scheme : '{}'\n" \
" url.netloc : '{}'\n" \
" url.path : '{}'\n".format(
url_str, type(oe).__name__, str(oe),
url.scheme, url.netloc, url.path))
finally:
if isinstance(test_conn, HTTPConnection):
test_conn.close()
return content_type_header