def __init__(self,
hostname,
auth=None,
port=None,
verify=True,
reauthenticate_handler=None):
""" Initialize new connection and setup authentication
`hostname` - include protocol, e.g. "https://host.com"
`auth` - authentication object, see below
`port` - optional port to use for connection
`verify` - require SSL certificate validation.
Authentication:
For simple basic auth, passing a tuple of (user, pass) is
sufficient as a shortcut to an instance of HTTPBasicAuth.
This auth method will trigger a check to ensure
the protocol is using SSL to connect (though cert verification
may still be turned off to avoid errors with self-signed certs).
OAuth2 will require the ``requests-oauthlib`` package and
an instance of the `OAuth2Session` object.
netrc config files will be checked if auth is left as None.
If no authentication is provided for the hostname in the
netrc file, or no file exists, an error will be raised
when trying to connect.
"""
p = parse_url(hostname)
if p.port and port and p.port != port:
raise RvbdException('Mismatched ports provided.')
elif not p.port and port:
hostname = hostname + ':' + str(port)
if not p.scheme:
# default to https, except when port 80 specified
if parse_url(hostname).port == '80':
logger.info("Connection defaulting to 'http://' scheme.")
hostname = 'http://' + hostname
else:
logger.info("Connection defaulting to 'https://' scheme.")
hostname = 'https://' + hostname
self.hostname = hostname
self._ssladapter = False
if self.HTTPLIB_DEBUGLEVEL > 0:
self.set_debuglevel()
self.conn = requests.session()
self.conn.auth = auth
self.conn.verify = verify
self._reauthenticate_handler = reauthenticate_handler
# store last full response
self.response = None
logger.debug("Connection initialized for %s" % self.hostname)
def __init__(self, hostname, auth=None, port=None, verify=True,
reauthenticate_handler=None):
""" Initialize new connection and setup authentication
`hostname` - include protocol, e.g. "https://host.com"
`auth` - authentication object, see below
`port` - optional port to use for connection
`verify` - require SSL certificate validation.
Authentication:
For simple basic auth, passing a tuple of (user, pass) is
sufficient as a shortcut to an instance of HTTPBasicAuth.
This auth method will trigger a check to ensure
the protocol is using SSL to connect (though cert verification
may still be turned off to avoid errors with self-signed certs).
OAuth2 will require the ``requests-oauthlib`` package and
an instance of the `OAuth2Session` object.
netrc config files will be checked if auth is left as None.
If no authentication is provided for the hostname in the
netrc file, or no file exists, an error will be raised
when trying to connect.
"""
p = parse_url(hostname)
if p.port and port and p.port != port:
raise RvbdException('Mismatched ports provided.')
elif not p.port and port:
hostname = hostname + ':' + str(port)
if not p.scheme:
# default to https, except when port 80 specified
if parse_url(hostname).port == '80':
logger.info("Connection defaulting to 'http://' scheme.")
hostname = 'http://' + hostname
else:
logger.info("Connection defaulting to 'https://' scheme.")
hostname = 'https://' + hostname
self.hostname = hostname
self._ssladapter = False
if self.HTTPLIB_DEBUGLEVEL > 0:
self.set_debuglevel()
self.conn = requests.session()
self.conn.auth = auth
self.conn.verify = verify
self._reauthenticate_handler = reauthenticate_handler
# store last full response
self.response = None
logger.debug("Connection initialized for %s" % self.hostname)
def resolve(url):
parts = parse_url(url)
if not parts.port and parts.scheme == 'https':
port = 443
elif not parts.port and parts.scheme == 'http':
port = 80
else:
port = parts.port
original = parts.host
resolved = gethostbyname(parts.host)
# Don't use a resolved hostname for SSL requests otherwise the
# certificate will not match the IP address (resolved)
host = resolved if parts.scheme != 'https' else parts.host
netloc = '%s:%d' % (host, port) if port else host
if port not in (443, 80):
host += ':%d' % port
original += ':%d' % port
return (urlparse.urlunparse((parts.scheme, netloc, parts.path or '',
'', parts.query or '',
parts.fragment or '')),
original, host)
def get_server_info(method='GET'):
global TEST_URL
def _make_request(self, conn, method, url, **kwargs):
response = self._old_make_request(conn, method, url, **kwargs)
sock = getattr(conn, 'sock', False)
if sock:
setattr(response, 'peer', sock.getpeername())
else:
setattr(response, 'peer', None)
return response
HTTPConnectionPool._old_make_request = HTTPConnectionPool._make_request
HTTPConnectionPool._make_request = _make_request
try:
res = requests.get(TEST_URL)
print('Server IP: %s' % res.raw._original_response.peer[0])
parts = parse_url(TEST_URL)
if not parts.port and parts.scheme == 'https':
port = 443
elif not parts.port and parts.scheme == 'http':
port = 80
else:
port = parts.port
print(
'Server Software: %s' %
res.headers.get('server'))
if str(res.history) == '[]':
res.history = ''
print('Status Code: %s %s' % (res.status_code, res.history))
print('Server Port: %s' % port)
except requests.exceptions.ConnectionError as e:
pass
def build_environ(method, path, host='localhost',
accept_type='text/html', content_type=None,
query=None, form=None, files=None, cookies=None):
if '://' in host:
url = host.rstrip('/') + '/' + path.lstrip('/')
else:
url = 'http://' + host.strip('/') + '/' + path.lstrip('/')
request = Request(method, url, None, files, form, query, cookies)
prepared = request.prepare()
parsed_url = parse_url(prepared.url)
environ = {
'HTTP_HOST': parsed_url.host,
'PATH_INFO': parsed_url.path,
'REQUEST_METHOD': prepared.method,
'HTTP_ACCEPT': accept_type,
'QUERY_STRING': parsed_url.query or '',
}
for key, value in iteritems(prepared.headers):
key = underscore(key)
if key not in ['content_type', 'content_length']:
key = 'http_' + key
environ[key.upper()] = value
if content_type is not None:
environ['CONTENT_TYPE'] = content_type
environ['wsgi.input'] = BytesIO(prepared.body)
return environ
def _request(self, method, uri, body=None, params=None,
extra_headers=None):
p = parse_url(uri)
if not p.host:
uri = self.get_url(uri)
try:
r = self.conn.request(method, uri, data=body, params=params,
headers=extra_headers, timeout=self.timeout)
except (requests.exceptions.SSLError,
requests.exceptions.ConnectionError) as e:
if self._ssladapter:
# If we've already applied an adapter, this is another problem
# Raise the corresponding sleepwaker exception.
raise ConnectionError("Could not connect to uri %s: %s",
uri, e)
# Otherwise, mount adapter and retry the request
self.conn.mount('https://', SSLAdapter(ssl.PROTOCOL_TLSv1))
self._ssladapter = True
r = self.conn.request(method, uri, data=body, params=params,
headers=extra_headers, timeout=self.timeout)
self.response = r
# check if good status response otherwise raise exception
if not r.ok:
HTTPError.raise_by_status(r)
return r
def _request(self, method, uri, body=None, params=None,
extra_headers=None):
p = parse_url(uri)
if not p.host:
uri = self.get_url(uri)
try:
r = self.conn.request(method, uri, data=body, params=params,
headers=extra_headers, timeout=self.timeout)
except (requests.exceptions.SSLError,
requests.exceptions.ConnectionError) as e:
if self._ssladapter:
# If we've already applied an adapter, this is another problem
# Raise the corresponding sleepwaker exception.
raise ConnectionError("Could not connect to uri %s: %s",
uri, e)
# Otherwise, mount adapter and retry the request
self.conn.mount('https://', SSLAdapter(ssl.PROTOCOL_TLSv1))
self._ssladapter = True
r = self.conn.request(method, uri, data=body, params=params,
headers=extra_headers, timeout=self.timeout)
self.response = r
# check if good status response otherwise raise exception
if not r.ok:
HTTPError.raise_by_status(r)
return r
def sign_url(input_url=None, secret=None):
""" Sign a request URL with a Crypto Key.
Usage:
from urlsigner import sign_url
signed_url = sign_url(input_url=my_url,
client_id=CLIENT_ID,
client_secret=CLIENT_SECRET)
Args:
input_url - The URL to sign
client_id - Your Client ID
client_secret - Your Crypto Key
Returns:
The signed request URL
"""
# Return if any parameters aren't given
if not input_url or not secret:
return None
# Add the Client ID to the URL
#input_url += "&client=%s" % (client_id)
url = parse_url(input_url)
# We only need to sign the path+query part of the string
url_to_sign = url.path + "?" + url.query
# Decode the private key into its binary format
# We need to decode the URL-encoded private key
decoded_key = base64.urlsafe_b64decode(secret)
# Create a signature using the private key and the URL-encoded
# string using HMAC SHA1. This signature will be binary.
signature = hmac.new(decoded_key, url_to_sign.encode(), hashlib.sha1)
# Encode the binary signature into base64 for use within a URL
encoded_signature = base64.urlsafe_b64encode(signature.digest())
original_url = url.scheme + "://" + url.netloc + url.path + "?" + url.query
# Return signed URL
return original_url + "&signature=" + encoded_signature.decode()
def _request(self, method, path, body=None, params=None,
extra_headers=None, **kwargs):
p = parse_url(path)
if not p.host:
path = self.get_url(path)
try:
logger.debug('Issuing %s request to: %s' % (method, str(path)))
#logger.debug('Body: %s' % (body))
flag = body is not None and '"password":'******'<username and password hidden>')
r = self.conn.request(method, path, data=body, params=params,
headers=extra_headers, **kwargs)
logger.debug('Response for %s request to %s: %s, %d' %
(method, str(path), r.status_code, len(r.content)))
if flag:
self.set_debuglevel()
except (requests.exceptions.SSLError,
requests.exceptions.ConnectionError):
if self._ssladapter:
# If we've already applied an adapter, this is another problem
raise
# Otherwise, mount adapter and retry the request
# See #152536 - Versions of openssl cause handshake failures
self.conn.mount('https://', SSLAdapter(ssl.PROTOCOL_TLSv1))
self._ssladapter = True
logger.debug('SSL error -- retrying with TLSv1')
r = self.conn.request(method, path, data=body,
params=params, headers=extra_headers)
# check if good status response otherwise raise exception
if not r.ok:
exc = RvbdHTTPException(r, r.text, method, path)
if (self._reauthenticate_handler is not None and
exc.error_id in ('AUTH_INVALID_SESSION',
'AUTH_EXPIRED_TOKEN')):
logger.debug('session timed out -- reauthenticating')
handler = self._reauthenticate_handler
self._reauthenticate_handler = None
handler()
logger.debug('session reauthentication succeeded -- retrying')
r = self._request(method, path, body, params,
extra_headers, **kwargs)
# successful connection, reset token if previously unset
self._reauthenticate_handler = handler
return r
else:
raise exc
return r
def parse_target_exchange(self) -> Union[str, None]:
"""Check the URL for its scheme and extract an exchange name, if any.
If the url starts with http/https we set :attr:`BitexRequest.exchange`
to `None`. Otherwise we store the `exchange` in said attribute.
"""
scheme = parse_url(self.url).scheme
if scheme and not scheme.startswith("http"):
return scheme
return None
def validate_url(url):
"""Asserts that the url string has a valid protocol scheme.
:param url: ``str`` url we want to validate
:raises OptimoError: When we can't deduce a valid protocol scheme
"""
_url = parse_url(url)
if not _url.scheme:
raise OptimoError("The url: '{}' does not define a protocol scheme"
.format(url))
def validate_url(url):
"""Asserts that the url string has a valid protocol scheme.
:param url: ``str`` url we want to validate
:raises OptimoError: When we can't deduce a valid protocol scheme
"""
_url = parse_url(url)
if not _url.scheme:
raise OptimoError(
"The url: '{}' does not define a protocol scheme".format(url))
def __init__(self, hostname, auth=None, port=None, verify=True,
timeout=None):
""" Initialize new connection and setup authentication
`hostname` - include protocol, e.g. 'https://host.com'
`auth` - authentication object, see below
`port` - optional port to use for connection
`verify` - require SSL certificate validation.
`timeout` - float connection timeout in seconds, or tuple
(connect timeout, read timeout)
Authentication:
For simple basic auth, passing a tuple of (user, pass) is
sufficient as a shortcut to an instance of HTTPBasicAuth.
This auth method will trigger a check to ensure
the protocol is using SSL to connect (though cert verification
may still be turned off to avoid errors with self-signed certs).
netrc config files will be checked if auth is left as None.
If no authentication is provided for the hostname in the
netrc file, or no file exists, an error will be raised
when trying to connect.
"""
p = parse_url(hostname)
if not p.scheme:
raise URLError('Scheme must be provided (e.g. https:// '
'or http://).')
else:
if p.port and port and p.port != port:
raise URLError('Mismatched ports provided.')
elif not p.port and port:
hostname = hostname + ':' + str(port)
# since the system re-tries, the effective timeout
# will be 2 times the connection timeout specified, so divide
# it in half so the connection timeout is what the caller expects
if timeout is None:
self.timeout = None
elif isinstance(timeout, Iterable):
if len(timeout) != 2:
raise ValueError('timeout tuple must be 2 float entries')
self.timeout = tuple([float(timeout[0]) / 2.0,
float(timeout[1])])
else:
self.timeout = float(timeout) / 2.0
self.hostname = hostname
self._ssladapter = False
self.conn = requests.session()
self.conn.auth = auth
self.conn.verify = verify
# store last full response
self.response = None
def __init__(self, hostname, auth=None, port=None, verify=True,
timeout=None):
""" Initialize new connection and setup authentication
`hostname` - include protocol, e.g. 'https://host.com'
`auth` - authentication object, see below
`port` - optional port to use for connection
`verify` - require SSL certificate validation.
`timeout` - float connection timeout in seconds, or tuple
(connect timeout, read timeout)
Authentication:
For simple basic auth, passing a tuple of (user, pass) is
sufficient as a shortcut to an instance of HTTPBasicAuth.
This auth method will trigger a check to ensure
the protocol is using SSL to connect (though cert verification
may still be turned off to avoid errors with self-signed certs).
netrc config files will be checked if auth is left as None.
If no authentication is provided for the hostname in the
netrc file, or no file exists, an error will be raised
when trying to connect.
"""
p = parse_url(hostname)
if not p.scheme:
raise URLError('Scheme must be provided (e.g. https:// '
'or http://).')
else:
if p.port and port and p.port != port:
raise URLError('Mismatched ports provided.')
elif not p.port and port:
hostname = hostname + ':' + str(port)
# since the system re-tries, the effective timeout
# will be 2 times the connection timeout specified, so divide
# it in half so the connection timeout is what the caller expects
if timeout is None:
self.timeout = None
elif isinstance(timeout, Iterable):
if len(timeout) != 2:
raise ValueError('timeout tuple must be 2 float entries')
self.timeout = tuple([float(timeout[0]) / 2.0,
float(timeout[1])])
else:
self.timeout = float(timeout) / 2.0
self.hostname = hostname
self._ssladapter = False
self.conn = requests.session()
self.conn.auth = auth
self.conn.verify = verify
# store last full response
self.response = None
def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None):
global _httpie_timings
scheme, auth, host, port, path, query, fragment = parse_url(request.url)
if port is None:
port = port_by_scheme[scheme]
_httpie_timings['request-start'] = time.time()
response = super(TimingHTTPAdapter, self).send(request, stream, timeout, verify, cert, proxies)
_httpie_timings['request-finish'] = time.time()
_httpie_timings['request-duration'] = _httpie_timings['request-finish'] - _httpie_timings['request-start']
return response
def check_url(input_url=None, client_secret=None):
""" Check a signed request URL with a Crypto Key.
Usage:
from urlsigner import check_url
is_valid = check_url(input_url=my_url,
client_secret=CLIENT_SECRET)
Args:
input_url - The signed URL
client_secret - Your Crypto Key
Returns:
True or False
"""
# Error if any parameters aren't given
if not input_url or not client_secret:
raise ValueError("Expected arguments input_url and client_secret")
url = parse_url(input_url)
# Extract signature
parts = url.query.rsplit("&signature=", 1)
if len(parts) != 2:
return False
# If signature was not the last part of the url, reorder it:
if "&" in parts[1]:
parts[0] += "&" + parts[1].split("&", 1)[1]
parts[1] = parts[1].split("&", 1)[0]
query = parts[0]
actual_signature = parts[1]
# We only need to sign the path+query part of the string
url_to_sign = url.path + "?" + query
# Decode the private key into its binary format
# We need to decode the URL-encoded private key
decoded_key = base64.urlsafe_b64decode(client_secret)
# Create a signature using the private key and the URL-encoded
# string using HMAC SHA1. This signature will be binary.
signature = hmac.new(decoded_key, url_to_sign.encode(), hashlib.sha1)
# Encode the binary signature into base64 for use within a URL
encoded_signature = base64.urlsafe_b64encode(signature.digest())
return encoded_signature.decode() == actual_signature
def resolve(url):
"""
parse the url to dns resolution
"""
parts = parse_url(url)
if not parts.port and parts.scheme == 'https':
port = 443
if not parts.port and parts.scheme == 'http':
port = 80
else:
port = parts.port
original = parts.host
resolved = gethostbyname(parts.host)
host = resolved if parts.scheme != 'https' else parts.host
netloc = '%s:%d' % (host, port) if port else host
return (urlparse.urlunparse((parts.scheme, netloc, parts.path or '', '', parts.query or '',\
parts.fragment or '')), original, host)
def set_debuglevel(self, level=None):
if level is None:
level = self.HTTPLIB_DEBUGLEVEL
logger.debug("Setting HTTPLIB_DEBUGLEVEL to %d" % level)
if parse_url(self.hostname).scheme == 'https':
httplib.HTTPSConnection.debuglevel = level
else:
httplib.HTTPConnection.debuglevel = level
requests_logger = logging.getLogger('requests.packages.urllib3')
requests_logger.propagate = True
if int(level) > 0:
requests_logger.setLevel(logging.DEBUG)
else:
requests_logger.setLevel(logging.INFO)
def set_debuglevel(self, level=None):
if level is None:
level = self.HTTPLIB_DEBUGLEVEL
logger.debug("Setting HTTPLIB_DEBUGLEVEL to %d" % level)
if parse_url(self.hostname).scheme == 'https':
httplib.HTTPSConnection.debuglevel = level
else:
httplib.HTTPConnection.debuglevel = level
requests_logger = logging.getLogger('requests.packages.urllib3')
requests_logger.propagate = True
if int(level) > 0:
requests_logger.setLevel(logging.DEBUG)
else:
requests_logger.setLevel(logging.INFO)
def show_host_info(url):
response = requests.head(url)
url_parts = parse_url(url)
server_software = response.headers.get('server', 'UnKnown')
server_host = url_parts.host or "Unknown"
server_port = None
server_path = url_parts.path or '/'
if not url_parts.port and url_parts.scheme == 'https':
server_port = 443
if not url_parts.port and url_parts.scheme == 'http':
server_port = 80
if url_parts.port:
server_port = url_parts.port
print("""
Server Software: \t {}
Server Hostname: \t {}
Server Port: \t {}
Document Path: \t {}
""".format(server_software, server_host, server_port, server_path))
def resolve(self):
requests = parse_url(self.args.url)
if not requests.port and requests.scheme == 'https':
port = 443
elif not requests.port and requests.scheme == 'http':
port = 80
else:
port = requests.port
orignal_host = requests.host
resolved_host = gethostbyname(orignal_host)
if requests.scheme != 'https':
host = resolved_host
else:
host = orignal_host
http_location = '%s:%d'%(host, port)
url = urlparse.urlunparse((requests.scheme, http_location,
requests.path or '', '', requests.query or '',requests.fragment or ''))
return (url, orignal_host, host)
def _request(self, method, path, body=None, params=None,
extra_headers=None, raw_json=None, stream=False,
**kwargs):
p = parse_url(path)
if not p.host:
path = self.get_url(path)
try:
rest_logger.info('%s %s' % (method, str(path)))
if params:
rest_logger.info('Parameters: ')
for k, v in params.iteritems():
rest_logger.info('... %s: %s' % (k, v))
if self.REST_DEBUG >= 1 and extra_headers:
rest_logger.info('Extra request headers: ')
for k, v in extra_headers.iteritems():
rest_logger.info('... %s: %s' % (k, v))
if self.REST_DEBUG >= 2 and body:
rest_logger.info('Request body: ')
if raw_json:
if path.endswith('login'):
debug_body = json.dumps(
scrub_passwords(raw_json), indent=2,
cls=self.JsonEncoder)
else:
logger.debug("raw_json: %s" % raw_json)
debug_body = json.dumps(
raw_json, indent=2, cls=self.JsonEncoder)
else:
debug_body = body
lines = debug_body.split('\n')
for line in lines[:self.REST_BODY_LINES]:
rest_logger.info('... %s' % line)
if len(lines) > self.REST_BODY_LINES:
rest_logger.info('... <truncated %d lines>'
% (len(lines) - 20))
r = self.conn.request(method, path, data=body, params=params,
headers=extra_headers,
stream=stream,
cookies=self.cookies, **kwargs)
if r.request.url != path:
rest_logger.info('Full URL: %s' % r.request.url)
if self.REST_DEBUG >= 1 and extra_headers:
rest_logger.info('Request headers: ')
for k, v in scrub_passwords(r.request.headers).iteritems():
rest_logger.info('... %s: %s' % (k, v))
if stream:
rest_logger.info('Response Status %s, streaming content' %
r.status_code)
else:
rest_logger.info('Response Status %s, %d bytes' %
(r.status_code, len(r.content)))
if self.REST_DEBUG >= 1 and extra_headers:
rest_logger.info('Response headers: ')
for k, v in r.headers.iteritems():
rest_logger.info('... %s: %s' % (k, v))
if self.REST_DEBUG >= 2 and not stream and r.text:
rest_logger.info('Response body: ')
try:
debug_body = json.dumps(r.json(), indent=2,
cls=self.JsonEncoder)
lines = debug_body.split('\n')
except:
lines = r.text.split('\n')
for line in lines[:self.REST_BODY_LINES]:
rest_logger.info('... %s' % line)
if len(lines) > self.REST_BODY_LINES:
rest_logger.info('... <truncated %d lines>'
% (len(lines) - 20))
except (requests.exceptions.SSLError,
requests.exceptions.ConnectionError):
if self._ssladapter:
# If we've already applied an adapter, this is another problem
raise
# Otherwise, mount adapter and retry the request
# See #152536 - Versions of openssl cause handshake failures
self.conn.mount('https://', SSLAdapter(ssl.PROTOCOL_TLSv1))
self._ssladapter = True
logger.info('SSL error -- retrying with TLSv1')
r = self.conn.request(method, path, data=body,
params=params, headers=extra_headers,
cookies=self.cookies)
# check if good status response otherwise raise exception
if not r.ok:
exc = RvbdHTTPException(r, r.text, method, path)
if (self._reauthenticate_handler is not None and
exc.error_id in ('AUTH_REQUIRED',
'AUTH_INVALID_SESSION',
'AUTH_EXPIRED_TOKEN')):
logger.debug('session timed out -- reauthenticating')
# clean any stale cookies from session
self._clear_cookies()
handler = self._reauthenticate_handler
self._reauthenticate_handler = None
handler()
logger.debug('session reauthentication succeeded -- retrying')
r = self._request(method, path, body, params,
extra_headers, raw_json=raw_json, **kwargs)
# successful connection, reset token if previously unset
self._reauthenticate_handler = handler
return r
else:
raise exc
return r
def _request(self,
method,
path,
body=None,
params=None,
extra_headers=None,
raw_json=None,
stream=False,
files=None,
**kwargs):
p = parse_url(path)
if not p.host:
path = self.get_url(path)
try:
rest_logger.info('%s %s' % (method, str(path)))
if params:
rest_logger.info('Parameters: ')
for k, v in params.items():
rest_logger.info('... %s: %s' % (k, v))
if self.REST_DEBUG >= 1 and extra_headers:
rest_logger.info('Extra request headers: ')
for k, v in extra_headers.items():
rest_logger.info('... %s: %s' % (k, v))
if self.REST_DEBUG >= 2 and body:
rest_logger.info('Request body: ')
if raw_json:
if path.endswith('login'):
debug_body = json.dumps(scrub_passwords(raw_json),
indent=2,
cls=self.JsonEncoder)
else:
logger.debug("raw_json: %s" % raw_json)
debug_body = json.dumps(raw_json,
indent=2,
cls=self.JsonEncoder)
else:
if files and isinstance(body, dict):
debug_body = json.dumps(body,
indent=2,
cls=self.JsonEncoder)
else:
debug_body = body
lines = debug_body.split('\n')
for line in lines[:self.REST_BODY_LINES]:
rest_logger.info('... %s' % line)
if len(lines) > self.REST_BODY_LINES:
rest_logger.info('... <truncated %d lines>' %
(len(lines) - 20))
r = self.conn.request(method,
path,
data=body,
params=params,
headers=extra_headers,
stream=stream,
files=files,
cookies=self.cookies,
**kwargs)
if r.request.url != path:
rest_logger.info('Full URL: %s' % r.request.url)
if self.REST_DEBUG >= 1 and extra_headers:
rest_logger.info('Request headers: ')
for k, v in scrub_passwords(r.request.headers).items():
rest_logger.info('... %s: %s' % (k, v))
if stream:
rest_logger.info('Response Status %s, streaming content' %
r.status_code)
else:
rest_logger.info('Response Status %s, %d bytes' %
(r.status_code, len(r.content)))
if self.REST_DEBUG >= 1 and extra_headers:
rest_logger.info('Response headers: ')
for k, v in r.headers.items():
rest_logger.info('... %s: %s' % (k, v))
if self.REST_DEBUG >= 2 and not stream and r.text:
rest_logger.info('Response body: ')
try:
debug_body = json.dumps(r.json(),
indent=2,
cls=self.JsonEncoder)
lines = debug_body.split('\n')
except:
lines = r.text.split('\n')
for line in lines[:self.REST_BODY_LINES]:
rest_logger.info('... %s' % line)
if len(lines) > self.REST_BODY_LINES:
rest_logger.info('... <truncated %d lines>' %
(len(lines) - 20))
except (requests.exceptions.SSLError,
requests.exceptions.ConnectionError):
if self._ssladapter:
# If we've already applied an adapter, this is another problem
raise
# Otherwise, mount adapter and retry the request
# See #152536 - Versions of openssl cause handshake failures
self.conn.mount('https://', SSLAdapter(ssl.PROTOCOL_TLSv1))
self._ssladapter = True
logger.info('SSL error -- retrying with TLSv1')
r = self.conn.request(method,
path,
data=body,
params=params,
headers=extra_headers,
cookies=self.cookies,
files=files)
# check if good status response otherwise raise exception
if not r.ok:
exc = RvbdHTTPException(r, r.text, method, path)
if (self._reauthenticate_handler is not None and exc.error_id
in ('AUTH_REQUIRED', 'AUTH_INVALID_SESSION',
'AUTH_EXPIRED_TOKEN', 'AUTH_INVALID_CREDENTIALS')):
logger.debug('session timed out -- reauthenticating')
# clean any stale cookies from session
self._clear_cookies()
handler = self._reauthenticate_handler
self._reauthenticate_handler = None
handler()
logger.debug('session reauthentication succeeded -- retrying')
r = self._request(method,
path,
body=body,
params=params,
extra_headers=extra_headers,
raw_json=raw_json,
files=files,
**kwargs)
# successful connection, reset token if previously unset
self._reauthenticate_handler = handler
return r
else:
raise exc
return r
def check_url(url):
url_part = parse_url(url)
return url_part.scheme and url_part.netloc
def __init__(self, scraped_url, source_url):
self.scraped_url = scraped_url
self.source_url = source_url
self.parsed_scraped_url = parse_url(self.scraped_url)
self.parsed_source_url = parse_url(self.source_url)
def save_item(request):
"""
Will carry a json payload with these possible fields
:id (required): the id
:data (optional): the new value for the data
:metadata (optional): new values for the metadata fields.
Any whose values are None will be deleted not set to None! Absent ones will be left alone
:nullout (optional): which metadata fields to set to None
"""
# The nullout is a bit of a temporary copout until we can make module_edit.coffee and the metadata editors a
# little smarter and able to pass something more akin to {unset: [field, field]}
try:
item_location = request.json['id']
except KeyError:
import inspect
log.exception(
'''Request missing required attribute 'id'.
Request info:
%s
Caller:
Function %s in file %s
''', request.META,
inspect.currentframe().f_back.f_code.co_name,
inspect.currentframe().f_back.f_code.co_filename)
return JsonResponse(
{"error": "Request missing required attribute 'id'."}, 400)
try:
old_item = modulestore().get_item(item_location)
except (ItemNotFoundError, InvalidLocationError):
log.error("Can't find item by location.")
return JsonResponse({"error": "Can't find item by location"}, 404)
# check permissions for this user within this course
if not has_access(request.user, item_location):
raise PermissionDenied()
store = get_modulestore(Location(item_location))
if request.json.get('data'):
data = request.json['data']
store.update_item(item_location, data)
if request.json.get('children') is not None:
children = request.json['children']
store.update_children(item_location, children)
# cdodge: also commit any metadata which might have been passed along
if request.json.get('nullout') is not None or request.json.get(
'metadata') is not None:
# the postback is not the complete metadata, as there's system metadata which is
# not presented to the end-user for editing. So let's fetch the original and
# 'apply' the submitted metadata, so we don't end up deleting system metadata
existing_item = modulestore().get_item(item_location)
for metadata_key in request.json.get('nullout', []):
setattr(existing_item, metadata_key, None)
# update existing metadata with submitted metadata (which can be partial)
# IMPORTANT NOTE: if the client passed 'null' (None) for a piece of metadata that means 'remove it'. If
# the intent is to make it None, use the nullout field
for metadata_key, value in request.json.get('metadata', {}).items():
field = existing_item.fields[metadata_key]
if value is None:
field.delete_from(existing_item)
else:
try:
value = field.from_json(value)
except ValueError:
return JsonResponse({"error": "Invalid data"}, 400)
field.write_to(existing_item, value)
if existing_item.category == 'video':
allowedSchemes = ['https']
# The entire site is served from https, so browsers with good
# security will reject non-https URLs anyway.
# Also, following video module specific code is here, because front-end
# metadata fields doesn't support validation.
if metadata_key == 'html5_sources' and not all(
[parse_url(u).scheme in allowedSchemes for u in value]):
raise ValidationError(
u'HTML5 video sources support following protocols: {0}.'
.format(' '.join(allowedSchemes)))
# Save the data that we've just changed to the underlying
# MongoKeyValueStore before we update the mongo datastore.
existing_item.save()
# commit to datastore
store.update_metadata(item_location, own_metadata(existing_item))
if existing_item.category == 'video':
manage_video_subtitles_save(old_item, existing_item)
return JsonResponse()
def save_item(request):
"""
Will carry a json payload with these possible fields
:id (required): the id
:data (optional): the new value for the data
:metadata (optional): new values for the metadata fields.
Any whose values are None will be deleted not set to None! Absent ones will be left alone
:nullout (optional): which metadata fields to set to None
"""
# The nullout is a bit of a temporary copout until we can make module_edit.coffee and the metadata editors a
# little smarter and able to pass something more akin to {unset: [field, field]}
try:
item_location = request.json['id']
except KeyError:
import inspect
log.exception(
'''Request missing required attribute 'id'.
Request info:
%s
Caller:
Function %s in file %s
''',
request.META,
inspect.currentframe().f_back.f_code.co_name,
inspect.currentframe().f_back.f_code.co_filename
)
return JsonResponse({"error": "Request missing required attribute 'id'."}, 400)
try:
old_item = modulestore().get_item(item_location)
except (ItemNotFoundError, InvalidLocationError):
log.error("Can't find item by location.")
return JsonResponse({"error": "Can't find item by location"}, 404)
# check permissions for this user within this course
if not has_access(request.user, item_location):
raise PermissionDenied()
store = get_modulestore(Location(item_location))
if request.json.get('data'):
data = request.json['data']
store.update_item(item_location, data)
if request.json.get('children') is not None:
children = request.json['children']
store.update_children(item_location, children)
# cdodge: also commit any metadata which might have been passed along
if request.json.get('nullout') is not None or request.json.get('metadata') is not None:
# the postback is not the complete metadata, as there's system metadata which is
# not presented to the end-user for editing. So let's fetch the original and
# 'apply' the submitted metadata, so we don't end up deleting system metadata
existing_item = modulestore().get_item(item_location)
for metadata_key in request.json.get('nullout', []):
setattr(existing_item, metadata_key, None)
# update existing metadata with submitted metadata (which can be partial)
# IMPORTANT NOTE: if the client passed 'null' (None) for a piece of metadata that means 'remove it'. If
# the intent is to make it None, use the nullout field
for metadata_key, value in request.json.get('metadata', {}).items():
field = existing_item.fields[metadata_key]
if value is None:
field.delete_from(existing_item)
else:
try:
value = field.from_json(value)
except ValueError:
return JsonResponse({"error": "Invalid data"}, 400)
field.write_to(existing_item, value)
if existing_item.category == 'video':
allowedSchemes = ['https']
# The entire site is served from https, so browsers with good
# security will reject non-https URLs anyway.
# Also, following video module specific code is here, because front-end
# metadata fields doesn't support validation.
if metadata_key == 'html5_sources' and not all([parse_url(u).scheme.lower() in allowedSchemes for u in value]):
raise ValidationError(u'HTML5 video sources support following protocols: {0}.'.format(' '.join(allowedSchemes)))
# Save the data that we've just changed to the underlying
# MongoKeyValueStore before we update the mongo datastore.
existing_item.save()
# commit to datastore
store.update_metadata(item_location, own_metadata(existing_item))
if existing_item.category == 'video':
manage_video_subtitles_save(old_item, existing_item)
return JsonResponse()
def request(method, url, access_key, secret_key, **kwargs):
"""Sends request with different method
This is to construct a whole headers by adding basic header
e.g. Date/Authorization besides user defined.
And also sort params by its name for signature.
"""
if access_key is None:
access_key = cfg["ACCESS_KEY"]
if secret_key is None:
secret_key = cfg["ACCESS_SECRET"]
logger.debug("access_key: %s, secret_key: %s", access_key, secret_key)
if "headers" in kwargs:
headers = kwargs["headers"].copy()
else:
headers = {}
if not find_matching_headers(constants.DATE, headers):
headers[constants.DATE] = formatdate(usegmt=True)
string_to_sign = "%s\n" % method
if find_matching_headers(constants.CONTENT_MD5, headers):
string_to_sign += "%s\n" % headers[constants.CONTENT_MD5]
else:
string_to_sign += "\n"
if find_matching_headers(constants.CONTENT_TYPE, headers):
string_to_sign += "%s\n" % headers[constants.CONTENT_TYPE]
else:
string_to_sign += "\n"
string_to_sign += headers[constants.DATE] + "\n"
# Sort canonicalizedAmzHeaders by name
amz_keys = [h for h in headers if h.lower().startswith("x-amz-")]
amz_keys.sort()
for key in amz_keys:
string_to_sign += "%s:%s\n" % (key, headers[key].strip())
canonicalized_resource = ""
# The subresources that must be included
# when constructing the CanonicalizedResource
# for calculating signature.
subresources = [
"acl",
"lifecycle",
"location",
"logging",
"notification",
"partNumber",
"policy",
"requestPayment",
"torrent",
"uploadId",
"uploads",
"versionId",
"versioning",
"versions",
"website",
]
_, _, _, _, path, _, _ = parse_url(url)
if not path:
path = "/"
canonicalized_resource += path
if "params" in kwargs:
params = kwargs["params"].copy()
else:
params = {}
subresource_without_value = []
subresource_to_sign = []
# Sort subresource by name, otherwise signature will not match.
# If the value of key in dict params is none, e.g. ?acl,
# it will not be appended to query string by requests parsing.
# So need to pick it up from params to query string by manual.
params = sorted(params.items(), key=lambda d: d[0])
for key, value in params:
if key in subresources:
# subresource has value
if value is not None:
subresource_to_sign.append("%s=%s" % (key, value))
else:
subresource_to_sign.append("%s" % key)
subresource_without_value.append("%s" % key)
else:
if value is None:
subresource_without_value.append("%s" % key)
if subresource_to_sign:
canonicalized_resource += "?"
canonicalized_resource += "&".join(subresource_to_sign)
if subresource_without_value:
url += "?"
url += "&".join(subresource_without_value)
string_to_sign += canonicalized_resource
logger.debug("string to sign:\n%s", string_to_sign)
logger.debug("request url:\n%s", url)
headers[constants.AUTHORIZATION] = "AWS %s:%s" % (access_key, utils.get_signature(secret_key, string_to_sign))
logger.debug("headers is " + repr(headers))
if "verify" not in kwargs:
kwargs["verify"] = False
kwargs["timeout"] = cfg["REQUEST_TIMEOUT"]
kwargs["headers"] = headers
logger.debug("kwargs headers is " + repr(kwargs["headers"]))
kwargs["params"] = params
logger.debug("kwargs params is " + repr(kwargs["params"]))
session = requests.Session()
response = session.request(method, url, **kwargs)
return response
def _request(self,
method,
path,
body=None,
params=None,
extra_headers=None,
**kwargs):
p = parse_url(path)
if not p.host:
path = self.get_url(path)
try:
logger.debug('Issuing %s request to: %s' % (method, str(path)))
#logger.debug('Body: %s' % (body))
flag = body is not None and '"password":'******'<username and password hidden>')
r = self.conn.request(method,
path,
data=body,
params=params,
headers=extra_headers,
**kwargs)
logger.debug('Response for %s request to %s: %s, %d' %
(method, str(path), r.status_code, len(r.content)))
if flag:
self.set_debuglevel()
except (requests.exceptions.SSLError,
requests.exceptions.ConnectionError):
if self._ssladapter:
# If we've already applied an adapter, this is another problem
raise
# Otherwise, mount adapter and retry the request
# See #152536 - Versions of openssl cause handshake failures
self.conn.mount('https://', SSLAdapter(ssl.PROTOCOL_TLSv1))
self._ssladapter = True
logger.debug('SSL error -- retrying with TLSv1')
r = self.conn.request(method,
path,
data=body,
params=params,
headers=extra_headers)
# check if good status response otherwise raise exception
if not r.ok:
exc = RvbdHTTPException(r, r.text, method, path)
if (self._reauthenticate_handler is not None and exc.error_id
in ('AUTH_INVALID_SESSION', 'AUTH_EXPIRED_TOKEN')):
logger.debug('session timed out -- reauthenticating')
handler = self._reauthenticate_handler
self._reauthenticate_handler = None
handler()
logger.debug('session reauthentication succeeded -- retrying')
r = self._request(method, path, body, params, extra_headers,
**kwargs)
# successful connection, reset token if previously unset
self._reauthenticate_handler = handler
return r
else:
raise exc
return r
def host(self):
if self.type is requests.models.PreparedRequest:
return parse_url(self.request.url).host
if self.type is dict:
return self.request['host']