def upload_cloudinary(user_id, filename):
if not user_id:
raise httplib.CannotSendRequest(
"Cloudinary upload called with no user_id.")
if not filename:
raise httplib.CannotSendRequest(
"Cloudinary upload called with no filename.")
req_url = "{}/{}/{}".format(CLOUDINARY_METHOD, user_id, filename)
try:
# Use HEAD since no data is required
httpConn.request("HEAD", req_url)
except Exception as e:
raise httplib.CannotSendRequest(
"Cloudinary request failed for {}.".format(req_url))
cloudinary_return = False
cloudinary_attempts = 0
# Retry Cloudinary trigger until success or at most 3 times
while not cloudinary_return and cloudinary_attempts < 3:
cloudinary_attempts += 1
try:
resp = httpConn.getresponse()
if resp.status != 200:
# Cloudinary failed, return from method
raise httplib.BadStatusLine(
"Cloudinary HTTP HEAD status {} reason {} for {}.".format(
resp.status, resp.reason, req_url))
else:
# Read the response to enabled next request
resp.read()
cloudinary_return = True
except httplib.ResponseNotReady as e:
logger.info(
"Cloudinary HTTP response for {} not ready after attempt {}, retrying..."
.format(req_url, cloudinary_attempts))
# The request never returned
if not cloudinary_return:
raise httplib.IncompleteRead(
"Cloudinary HTTP request for {} never returned.".format(req_url))
def request(self, *args, **kwargs):
for retry in xrange(10):
try:
self.conn.request(*args, **kwargs)
res = self.conn.getresponse()
return res
except Exception as e:
sys.stderr.write(
'Cannot send request ({0}).. retry {1}\n'.format(e, retry))
time.sleep(0.1)
self.conn = self._create_http_connection()
continue
raise httplib.CannotSendRequest()
def read(self):
ret = self._prompt.read()
begin = self._oldTime = datetime.now()
while (len(ret) == 0) and not (self.is_timer_expired(begin)):
time.sleep(0.2)
ret = self._prompt.read()
if ("Rex::TimeoutError Operation timed out." in ret) or self.is_timer_expired(begin):
self._toolbox.session_kill(self._index)
raise SessionTimedOutException("SessionTimedOutException: session {} is dead.".format(self._index))
if ("Broken pipe from" in ret):
raise httplib.CannotSendRequest()
return ret
def open_connection(url):
"""Open an Http Connection and return the connection object.
:param url:
:return conn:
"""
try:
if url.scheme == 'https':
conn = httplib.HTTPSConnection(url.netloc)
else:
conn = httplib.HTTPConnection(url.netloc)
except httplib.InvalidURL as exc:
msg = 'ERROR: Making connection to %s\nREASON:\t %s' % (url, exc)
raise httplib.CannotSendRequest(msg)
else:
if ARGS.get('debug'):
conn.set_debuglevel(1)
return conn
def request(self,
method,
path,
body=None,
headers=None,
expect_continue=False):
"""Send a request to the server.
For increased flexibility, this does not return the response
object. Future versions of HTTPConnection that juggle multiple
sockets will be able to send (for example) 5 requests all at
once, and then let the requests arrive as data is
available. Use the `getresponse()` method to retrieve the
response.
"""
if headers is None:
headers = {}
method = _ensurebytes(method)
path = _ensurebytes(path)
if self.busy():
raise httplib.CannotSendRequest(
'Can not send another request before '
'current response is read!')
self._current_response_taken = False
logger.info('sending %s request for %s to %s on port %s', method, path,
self.host, self.port)
hdrs = _foldheaders(headers)
# Figure out headers that have to be computed from the request
# body.
chunked = False
if body and HDR_CONTENT_LENGTH not in hdrs:
if getattr(body, '__len__', False):
hdrs[HDR_CONTENT_LENGTH] = (HDR_CONTENT_LENGTH,
b'%d' % len(body))
elif getattr(body, 'read', False):
hdrs[HDR_XFER_ENCODING] = (HDR_XFER_ENCODING,
XFER_ENCODING_CHUNKED)
chunked = True
else:
raise BadRequestData('body has no __len__() nor read()')
# Figure out expect-continue header
if hdrs.get('expect', ('', ''))[1].lower() == b'100-continue':
expect_continue = True
elif expect_continue:
hdrs['expect'] = (b'Expect', b'100-Continue')
# httplib compatibility: if the user specified a
# proxy-authorization header, that's actually intended for a
# proxy CONNECT action, not the real request, but only if
# we're going to use a proxy.
pheaders = dict(self._proxy_headers)
if self._proxy_host and self.ssl:
pa = hdrs.pop('proxy-authorization', None)
if pa is not None:
pheaders['proxy-authorization'] = pa
# Build header data
outgoing_headers = self._buildheaders(method, path, hdrs,
self.http_version)
# If we're reusing the underlying socket, there are some
# conditions where we'll want to retry, so make a note of the
# state of self.sock
fresh_socket = self.sock is None
self._connect(pheaders)
response = None
first = True
while ((outgoing_headers or body)
and not (response and response.complete())):
select_timeout = self.timeout
out = outgoing_headers or body
blocking_on_continue = False
if expect_continue and not outgoing_headers and not (
response and (response.headers or response.continued)):
logger.info(
'waiting up to %s seconds for'
' continue response from server', self.continue_timeout)
select_timeout = self.continue_timeout
blocking_on_continue = True
out = False
if out:
w = [self.sock]
else:
w = []
r, w, x = select.select([self.sock], w, [], select_timeout)
# if we were expecting a 100 continue and it's been long
# enough, just go ahead and assume it's ok. This is the
# recommended behavior from the RFC.
if r == w == x == []:
if blocking_on_continue:
expect_continue = False
logger.info('no response to continue expectation from '
'server, optimistically sending request body')
else:
raise HTTPTimeoutException('timeout sending data')
was_first = first
# incoming data
if r:
try:
try:
data = r[0].recv(INCOMING_BUFFER_SIZE)
except ssl.SSLError as e:
if e.args[0] != ssl.SSL_ERROR_WANT_READ:
raise
logger.debug('SSL_ERROR_WANT_READ while sending '
'data, retrying...')
continue
if not data:
logger.info('socket appears closed in read')
self.sock = None
self._current_response = None
if response is not None:
# We're a friend of the response class, so let
# us use the private attribute.
# pylint: disable=W0212
response._close()
# This if/elif ladder is a bit subtle,
# comments in each branch should help.
if response is not None and response.complete():
# Server responded completely and then
# closed the socket. We should just shut
# things down and let the caller get their
# response.
logger.info('Got an early response, '
'aborting remaining request.')
break
elif was_first and response is None:
# Most likely a keepalive that got killed
# on the server's end. Commonly happens
# after getting a really large response
# from the server.
logger.info(
'Connection appeared closed in read on first'
' request loop iteration, will retry.')
self._reconnect('read', pheaders)
continue
else:
# We didn't just send the first data hunk,
# and either have a partial response or no
# response at all. There's really nothing
# meaningful we can do here.
raise HTTPStateError(
'Connection appears closed after '
'some request data was written, but the '
'response was missing or incomplete!')
logger.debug('read %d bytes in request()', len(data))
if response is None:
response = self.response_class(r[0], self.timeout,
method)
# We're a friend of the response class, so let us
# use the private attribute.
# pylint: disable=W0212
response._load_response(data)
# Jump to the next select() call so we load more
# data if the server is still sending us content.
continue
except socket.error as e:
if e[0] != errno.EPIPE and not was_first:
raise
# outgoing data
if w and out:
try:
if getattr(out, 'read', False):
# pylint guesses the type of out incorrectly here
# pylint: disable=E1103
data = out.read(OUTGOING_BUFFER_SIZE)
if not data:
continue
if len(data) < OUTGOING_BUFFER_SIZE:
if chunked:
body = b'0' + EOL + EOL
else:
body = None
if chunked:
# This encode is okay because we know
# hex() is building us only 0-9 and a-f
# digits.
asciilen = hex(len(data))[2:].encode('ascii')
out = asciilen + EOL + data + EOL
else:
out = data
amt = w[0].send(out)
except socket.error as e:
if e[0] == ssl.SSL_ERROR_WANT_WRITE and self.ssl:
# This means that SSL hasn't flushed its buffer into
# the socket yet.
# TODO: find a way to block on ssl flushing its buffer
# similar to selecting on a raw socket.
continue
if e[0] == errno.EWOULDBLOCK or e[0] == errno.EAGAIN:
continue
elif (e[0] not in (errno.ECONNRESET, errno.EPIPE)
and not first):
raise
self._reconnect('write', pheaders)
amt = self.sock.send(out)
logger.debug('sent %d', amt)
first = False
if out is body:
body = out[amt:]
else:
outgoing_headers = out[amt:]
# End of request-sending loop.
# close if the server response said to or responded before eating
# the whole request
if response is None:
response = self.response_class(self.sock, self.timeout, method)
if not fresh_socket:
if not response._select():
# This means the response failed to get any response
# data at all, and in all probability the socket was
# closed before the server even saw our request. Try
# the request again on a fresh socket.
logger.debug('response._select() failed during request().'
' Assuming request needs to be retried.')
self.sock = None
# Call this method explicitly to re-try the
# request. We don't use self.request() because
# some tools (notably Mercurial) expect to be able
# to subclass and redefine request(), and they
# don't have the same argspec as we do.
#
# TODO restructure sending of requests to avoid
# this recursion
return HTTPConnection.request(
self,
method,
path,
body=body,
headers=headers,
expect_continue=expect_continue)
data_left = bool(outgoing_headers or body)
if data_left:
logger.info('stopped sending request early, '
'will close the socket to be safe.')
response.will_close = True
if response.will_close:
# The socket will be closed by the response, so we disown
# the socket
self.sock = None
self._current_response = response
def request(self,
method,
path,
body=None,
headers={},
expect_continue=False):
"""Send a request to the server.
For increased flexibility, this does not return the response
object. Future versions of HTTPConnection that juggle multiple
sockets will be able to send (for example) 5 requests all at
once, and then let the requests arrive as data is
available. Use the `getresponse()` method to retrieve the
response.
"""
if self.busy():
raise httplib.CannotSendRequest(
'Can not send another request before '
'current response is read!')
self._current_response_taken = False
logger.info('sending %s request for %s to %s on port %s', method, path,
self.host, self.port)
hdrs = dict((k.lower(), (k, v)) for k, v in headers.iteritems())
if hdrs.get('expect', ('', ''))[1].lower() == '100-continue':
expect_continue = True
elif expect_continue:
hdrs['expect'] = ('Expect', '100-Continue')
chunked = False
if body and HDR_CONTENT_LENGTH not in hdrs:
if getattr(body, '__len__', False):
hdrs[HDR_CONTENT_LENGTH] = (HDR_CONTENT_LENGTH, len(body))
elif getattr(body, 'read', False):
hdrs[HDR_XFER_ENCODING] = (HDR_XFER_ENCODING,
XFER_ENCODING_CHUNKED)
chunked = True
else:
raise BadRequestData('body has no __len__() nor read()')
# If we're reusing the underlying socket, there are some
# conditions where we'll want to retry, so make a note of the
# state of self.sock
fresh_socket = self.sock is None
self._connect()
outgoing_headers = self._buildheaders(method, path, hdrs,
self.http_version)
response = None
first = True
while ((outgoing_headers or body)
and not (response and response.complete())):
select_timeout = self.timeout
out = outgoing_headers or body
blocking_on_continue = False
if expect_continue and not outgoing_headers and not (
response and (response.headers or response.continued)):
logger.info(
'waiting up to %s seconds for'
' continue response from server', self.continue_timeout)
select_timeout = self.continue_timeout
blocking_on_continue = True
out = False
if out:
w = [self.sock]
else:
w = []
r, w, x = select.select([self.sock], w, [], select_timeout)
# if we were expecting a 100 continue and it's been long
# enough, just go ahead and assume it's ok. This is the
# recommended behavior from the RFC.
if r == w == x == []:
if blocking_on_continue:
expect_continue = False
logger.info('no response to continue expectation from '
'server, optimistically sending request body')
else:
raise HTTPTimeoutException('timeout sending data')
was_first = first
# incoming data
if r:
try:
try:
data = r[0].recv(INCOMING_BUFFER_SIZE)
except socket.sslerror, e:
if e.args[0] != socket.SSL_ERROR_WANT_READ:
raise
logger.debug('SSL_ERROR_WANT_READ while sending '
'data, retrying...')
continue
if not data:
logger.info('socket appears closed in read')
self.sock = None
self._current_response = None
if response is not None:
# We're a friend of the response class, so let
# us use the private attribute.
# pylint: disable=W0212
response._close()
# This if/elif ladder is a bit subtle,
# comments in each branch should help.
if response is not None and response.complete():
# Server responded completely and then
# closed the socket. We should just shut
# things down and let the caller get their
# response.
logger.info('Got an early response, '
'aborting remaining request.')
break
elif was_first and response is None:
# Most likely a keepalive that got killed
# on the server's end. Commonly happens
# after getting a really large response
# from the server.
logger.info(
'Connection appeared closed in read on first'
' request loop iteration, will retry.')
self._reconnect('read')
continue
else:
# We didn't just send the first data hunk,
# and either have a partial response or no
# response at all. There's really nothing
# meaningful we can do here.
raise HTTPStateError(
'Connection appears closed after '
'some request data was written, but the '
'response was missing or incomplete!')
logger.debug('read %d bytes in request()', len(data))
if response is None:
response = self.response_class(r[0], self.timeout,
method)
# We're a friend of the response class, so let us
# use the private attribute.
# pylint: disable=W0212
response._load_response(data)
# Jump to the next select() call so we load more
# data if the server is still sending us content.
continue
except socket.error, e:
if e[0] != errno.EPIPE and not was_first:
raise
def side_effect(mo, **kwargs):
self.assertEqual(managed_object, mo._type)
self.assertEqual(managed_object, mo.value)
raise httplib.CannotSendRequest()
if self.result_obj: break
time.sleep(1)
except httplib.CannotSendRequest,v:
err_msg("Could not connect. Make sure certificate is still valid")
err_msg("%s: %s"%(repr(v), dir(v)))
connected=0
raise
return
except httplib.ResponseNotReady:
pass
except httplib.BadStatusLine:
self.conn=httplib.HTTPSConnection(self.host, self.port, self.keyfile, self.certfile)
self.nb_dispatch(method,args)
i=i+1
if not self.result_obj:
raise httplib.CannotSendRequest("Could not connect. Make sure certificate is still valid")
new_info=self.result_obj.getheader('set-cookie')
if new_info:
c=Cookie.SimpleCookie(new_info)
if c.has_key('clarens_username'):
self.conn_user=c.get('clarens_username').value
self.conn_passwd=c.get('clarens_password').value
# Return data in file or as string to user
if not self.deserialize:
if not self.filename:
s=StringIO.StringIO()
else:
s=open(filename,"w")
try: