Example #1
0
 def get(self, http, locator):
     # http is an httplib2.Http object.
     # locator is a KeepLocator object.
     url = self.root + str(locator)
     _logger.debug("Request: GET %s", url)
     try:
         with timer.Timer() as t:
             result = http.request(url.encode('utf-8'),
                                   'GET',
                                   headers=self.get_headers)
     except self.HTTP_ERRORS as e:
         _logger.debug("Request fail: GET %s => %s: %s", url, type(e),
                       str(e))
         self.last_result = e
     else:
         self.last_result = result
         self.success_flag = retry.check_http_response_success(result)
         content = result[1]
         _logger.info("%s response: %s bytes in %s msec (%.3f MiB/sec)",
                      self.last_status(), len(content), t.msecs,
                      (len(content) / (1024.0 * 1024)) / t.secs)
         if self.success_flag:
             resp_md5 = hashlib.md5(content).hexdigest()
             if resp_md5 == locator.md5sum:
                 return content
             _logger.warning("Checksum fail: md5(%s) = %s", url,
                             resp_md5)
     return None
Example #2
0
 def get(self, locator, timeout=None):
     # locator is a KeepLocator object.
     url = self.root + str(locator)
     _logger.debug("Request: GET %s", url)
     try:
         with timer.Timer() as t:
             result = self.session.get(url.encode('utf-8'),
                                   headers=self.get_headers,
                                   timeout=timeout)
     except self.HTTP_ERRORS as e:
         _logger.debug("Request fail: GET %s => %s: %s",
                       url, type(e), str(e))
         self.last_result = e
     else:
         self.last_result = result
         self.success_flag = retry.check_http_response_success(result)
         content = result.content
         _logger.info("%s response: %s bytes in %s msec (%.3f MiB/sec)",
                      self.last_status(), len(content), t.msecs,
                      (len(content)/(1024.0*1024))/t.secs if t.secs > 0 else 0)
         if self.success_flag:
             resp_md5 = hashlib.md5(content).hexdigest()
             if resp_md5 == locator.md5sum:
                 return content
             _logger.warning("Checksum fail: md5(%s) = %s",
                             url, resp_md5)
     return None
Example #3
0
 def put(self, hash_s, body, timeout=None):
     url = self.root + hash_s
     _logger.debug("Request: PUT %s", url)
     curl = self._get_user_agent()
     try:
         self._headers = {}
         body_reader = cStringIO.StringIO(body)
         response_body = cStringIO.StringIO()
         curl.setopt(pycurl.NOSIGNAL, 1)
         curl.setopt(pycurl.OPENSOCKETFUNCTION, self._socket_open)
         curl.setopt(pycurl.URL, url.encode('utf-8'))
         # Using UPLOAD tells cURL to wait for a "go ahead" from the
         # Keep server (in the form of a HTTP/1.1 "100 Continue"
         # response) instead of sending the request body immediately.
         # This allows the server to reject the request if the request
         # is invalid or the server is read-only, without waiting for
         # the client to send the entire block.
         curl.setopt(pycurl.UPLOAD, True)
         curl.setopt(pycurl.INFILESIZE, len(body))
         curl.setopt(pycurl.READFUNCTION, body_reader.read)
         curl.setopt(pycurl.HTTPHEADER, [
             '{}: {}'.format(k, v)
             for k, v in self.put_headers.iteritems()
         ])
         curl.setopt(pycurl.WRITEFUNCTION, response_body.write)
         curl.setopt(pycurl.HEADERFUNCTION, self._headerfunction)
         self._setcurltimeouts(curl, timeout)
         try:
             curl.perform()
         except Exception as e:
             raise arvados.errors.HttpError(0, str(e))
         self._result = {
             'status_code': curl.getinfo(pycurl.RESPONSE_CODE),
             'body': response_body.getvalue(),
             'headers': self._headers,
             'error': False,
         }
         ok = retry.check_http_response_success(
             self._result['status_code'])
         if not ok:
             self._result['error'] = arvados.errors.HttpError(
                 self._result['status_code'],
                 self._headers.get('x-status-line', 'Error'))
     except self.HTTP_ERRORS as e:
         self._result = {
             'error': e,
         }
         ok = False
     self._usable = ok != False  # still usable if ok is True or None
     if self._result.get('status_code', None):
         # Client is functional. See comment in get().
         self._put_user_agent(curl)
     else:
         curl.close()
     if not ok:
         _logger.debug("Request fail: PUT %s => %s: %s", url,
                       type(self._result['error']),
                       str(self._result['error']))
         return False
     return True
Example #4
0
 def put(self, hash_s, body, timeout=None):
     url = self.root + hash_s
     _logger.debug("Request: PUT %s", url)
     curl = self._get_user_agent()
     try:
         self._headers = {}
         body_reader = cStringIO.StringIO(body)
         response_body = cStringIO.StringIO()
         curl.setopt(pycurl.NOSIGNAL, 1)
         curl.setopt(pycurl.OPENSOCKETFUNCTION, self._socket_open)
         curl.setopt(pycurl.URL, url.encode('utf-8'))
         # Using UPLOAD tells cURL to wait for a "go ahead" from the
         # Keep server (in the form of a HTTP/1.1 "100 Continue"
         # response) instead of sending the request body immediately.
         # This allows the server to reject the request if the request
         # is invalid or the server is read-only, without waiting for
         # the client to send the entire block.
         curl.setopt(pycurl.UPLOAD, True)
         curl.setopt(pycurl.INFILESIZE, len(body))
         curl.setopt(pycurl.READFUNCTION, body_reader.read)
         curl.setopt(pycurl.HTTPHEADER, [
             '{}: {}'.format(k,v) for k,v in self.put_headers.iteritems()])
         curl.setopt(pycurl.WRITEFUNCTION, response_body.write)
         curl.setopt(pycurl.HEADERFUNCTION, self._headerfunction)
         self._setcurltimeouts(curl, timeout)
         try:
             curl.perform()
         except Exception as e:
             raise arvados.errors.HttpError(0, str(e))
         self._result = {
             'status_code': curl.getinfo(pycurl.RESPONSE_CODE),
             'body': response_body.getvalue(),
             'headers': self._headers,
             'error': False,
         }
         ok = retry.check_http_response_success(self._result['status_code'])
         if not ok:
             self._result['error'] = arvados.errors.HttpError(
                 self._result['status_code'],
                 self._headers.get('x-status-line', 'Error'))
     except self.HTTP_ERRORS as e:
         self._result = {
             'error': e,
         }
         ok = False
     self._usable = ok != False # still usable if ok is True or None
     if self._result.get('status_code', None):
         # Client is functional. See comment in get().
         self._put_user_agent(curl)
     else:
         curl.close()
     if not ok:
         _logger.debug("Request fail: PUT %s => %s: %s",
                       url, type(self._result['error']), str(self._result['error']))
         return False
     return True
Example #5
0
 def put(self, http, hash_s, body):
     url = self.root + hash_s
     _logger.debug("Request: PUT %s", url)
     try:
         result = http.request(url.encode('utf-8'),
                               'PUT',
                               headers=self.put_headers,
                               body=body)
     except self.HTTP_ERRORS as e:
         _logger.debug("Request fail: PUT %s => %s: %s", url, type(e),
                       str(e))
         self.last_result = e
     else:
         self.last_result = result
         self.success_flag = retry.check_http_response_success(result)
     return self.success_flag
Example #6
0
 def put(self, hash_s, body, timeout=None):
     url = self.root + hash_s
     _logger.debug("Request: PUT %s", url)
     try:
         result = self.session.put(url.encode('utf-8'),
                               data=body,
                               headers=self.put_headers,
                               timeout=timeout)
     except self.HTTP_ERRORS as e:
         _logger.debug("Request fail: PUT %s => %s: %s",
                       url, type(e), str(e))
         self.last_result = e
     else:
         self.last_result = result
         self.success_flag = retry.check_http_response_success(result)
     return self.success_flag
Example #7
0
 def results_map(self, *codes):
     for code in codes:
         yield code, arv_retry.check_http_response_success(code)
Example #8
0
        def get(self, locator, method="GET", timeout=None):
            # locator is a KeepLocator object.
            url = self.root + str(locator)
            _logger.debug("Request: %s %s", method, url)
            curl = self._get_user_agent()
            ok = None
            try:
                with timer.Timer() as t:
                    self._headers = {}
                    response_body = cStringIO.StringIO()
                    curl.setopt(pycurl.NOSIGNAL, 1)
                    curl.setopt(pycurl.OPENSOCKETFUNCTION, self._socket_open)
                    curl.setopt(pycurl.URL, url.encode('utf-8'))
                    curl.setopt(pycurl.HTTPHEADER, [
                        '{}: {}'.format(k,v) for k,v in self.get_headers.iteritems()])
                    curl.setopt(pycurl.WRITEFUNCTION, response_body.write)
                    curl.setopt(pycurl.HEADERFUNCTION, self._headerfunction)
                    if method == "HEAD":
                        curl.setopt(pycurl.NOBODY, True)
                    self._setcurltimeouts(curl, timeout)

                    try:
                        curl.perform()
                    except Exception as e:
                        raise arvados.errors.HttpError(0, str(e))
                    self._result = {
                        'status_code': curl.getinfo(pycurl.RESPONSE_CODE),
                        'body': response_body.getvalue(),
                        'headers': self._headers,
                        'error': False,
                    }

                ok = retry.check_http_response_success(self._result['status_code'])
                if not ok:
                    self._result['error'] = arvados.errors.HttpError(
                        self._result['status_code'],
                        self._headers.get('x-status-line', 'Error'))
            except self.HTTP_ERRORS as e:
                self._result = {
                    'error': e,
                }
            self._usable = ok != False
            if self._result.get('status_code', None):
                # The client worked well enough to get an HTTP status
                # code, so presumably any problems are just on the
                # server side and it's OK to reuse the client.
                self._put_user_agent(curl)
            else:
                # Don't return this client to the pool, in case it's
                # broken.
                curl.close()
            if not ok:
                _logger.debug("Request fail: GET %s => %s: %s",
                              url, type(self._result['error']), str(self._result['error']))
                return None
            if method == "HEAD":
                _logger.info("HEAD %s: %s bytes",
                         self._result['status_code'],
                         self._result.get('content-length'))
                return True

            _logger.info("GET %s: %s bytes in %s msec (%.3f MiB/sec)",
                         self._result['status_code'],
                         len(self._result['body']),
                         t.msecs,
                         (len(self._result['body'])/(1024.0*1024))/t.secs if t.secs > 0 else 0)

            if self.download_counter:
                self.download_counter.add(len(self._result['body']))
            resp_md5 = hashlib.md5(self._result['body']).hexdigest()
            if resp_md5 != locator.md5sum:
                _logger.warning("Checksum fail: md5(%s) = %s",
                                url, resp_md5)
                self._result['error'] = arvados.errors.HttpError(
                    0, 'Checksum fail')
                return None
            return self._result['body']
Example #9
0
 def results_map(self, *codes):
     for code in codes:
         response = fake_requests_response(code, None)
         yield code, arv_retry.check_http_response_success(response)
Example #10
0
        def get(self, locator, method="GET", timeout=None):
            # locator is a KeepLocator object.
            url = self.root + str(locator)
            _logger.debug("Request: %s %s", method, url)
            curl = self._get_user_agent()
            ok = None
            try:
                with timer.Timer() as t:
                    self._headers = {}
                    response_body = BytesIO()
                    curl.setopt(pycurl.NOSIGNAL, 1)
                    curl.setopt(
                        pycurl.OPENSOCKETFUNCTION,
                        lambda *args, **kwargs: self._socket_open(
                            *args, **kwargs))
                    curl.setopt(pycurl.URL, url.encode('utf-8'))
                    curl.setopt(pycurl.HTTPHEADER, [
                        '{}: {}'.format(k, v)
                        for k, v in self.get_headers.items()
                    ])
                    curl.setopt(pycurl.WRITEFUNCTION, response_body.write)
                    curl.setopt(pycurl.HEADERFUNCTION, self._headerfunction)
                    if method == "HEAD":
                        curl.setopt(pycurl.NOBODY, True)
                    self._setcurltimeouts(curl, timeout)

                    try:
                        curl.perform()
                    except Exception as e:
                        raise arvados.errors.HttpError(0, str(e))
                    finally:
                        if self._socket:
                            self._socket.close()
                            self._socket = None
                    self._result = {
                        'status_code': curl.getinfo(pycurl.RESPONSE_CODE),
                        'body': response_body.getvalue(),
                        'headers': self._headers,
                        'error': False,
                    }

                ok = retry.check_http_response_success(
                    self._result['status_code'])
                if not ok:
                    self._result['error'] = arvados.errors.HttpError(
                        self._result['status_code'],
                        self._headers.get('x-status-line', 'Error'))
            except self.HTTP_ERRORS as e:
                self._result = {
                    'error': e,
                }
            self._usable = ok != False
            if self._result.get('status_code', None):
                # The client worked well enough to get an HTTP status
                # code, so presumably any problems are just on the
                # server side and it's OK to reuse the client.
                self._put_user_agent(curl)
            else:
                # Don't return this client to the pool, in case it's
                # broken.
                curl.close()
            if not ok:
                _logger.debug("Request fail: GET %s => %s: %s", url,
                              type(self._result['error']),
                              str(self._result['error']))
                return None
            if method == "HEAD":
                _logger.info("HEAD %s: %s bytes", self._result['status_code'],
                             self._result.get('content-length'))
                return True

            _logger.info(
                "GET %s: %s bytes in %s msec (%.3f MiB/sec)",
                self._result['status_code'], len(self._result['body']),
                t.msecs, 1.0 * len(self._result['body']) / 2**20 /
                t.secs if t.secs > 0 else 0)

            if self.download_counter:
                self.download_counter.add(len(self._result['body']))
            resp_md5 = hashlib.md5(self._result['body']).hexdigest()
            if resp_md5 != locator.md5sum:
                _logger.warning("Checksum fail: md5(%s) = %s", url, resp_md5)
                self._result['error'] = arvados.errors.HttpError(
                    0, 'Checksum fail')
                return None
            return self._result['body']
Example #11
0
 def results_map(self, *codes):
     for code in codes:
         yield code, arv_retry.check_http_response_success(code)
Example #12
0
 def results_map(self, *codes):
     for code in codes:
         response = (fake_httplib2_response(code), None)
         yield code, arv_retry.check_http_response_success(response)
Example #13
0
 def put(self, hash_s, body, timeout=None):
     url = self.root + hash_s
     _logger.debug("Request: PUT %s", url)
     curl = self._get_user_agent()
     ok = None
     try:
         with timer.Timer() as t:
             self._headers = {}
             body_reader = cStringIO.StringIO(body)
             response_body = cStringIO.StringIO()
             curl.setopt(pycurl.NOSIGNAL, 1)
             curl.setopt(pycurl.OPENSOCKETFUNCTION, self._socket_open)
             curl.setopt(pycurl.URL, url.encode("utf-8"))
             # Using UPLOAD tells cURL to wait for a "go ahead" from the
             # Keep server (in the form of a HTTP/1.1 "100 Continue"
             # response) instead of sending the request body immediately.
             # This allows the server to reject the request if the request
             # is invalid or the server is read-only, without waiting for
             # the client to send the entire block.
             curl.setopt(pycurl.UPLOAD, True)
             curl.setopt(pycurl.INFILESIZE, len(body))
             curl.setopt(pycurl.READFUNCTION, body_reader.read)
             curl.setopt(pycurl.HTTPHEADER, ["{}: {}".format(k, v) for k, v in self.put_headers.iteritems()])
             curl.setopt(pycurl.WRITEFUNCTION, response_body.write)
             curl.setopt(pycurl.HEADERFUNCTION, self._headerfunction)
             self._setcurltimeouts(curl, timeout)
             try:
                 curl.perform()
             except Exception as e:
                 raise arvados.errors.HttpError(0, str(e))
             self._result = {
                 "status_code": curl.getinfo(pycurl.RESPONSE_CODE),
                 "body": response_body.getvalue(),
                 "headers": self._headers,
                 "error": False,
             }
         ok = retry.check_http_response_success(self._result["status_code"])
         if not ok:
             self._result["error"] = arvados.errors.HttpError(
                 self._result["status_code"], self._headers.get("x-status-line", "Error")
             )
     except self.HTTP_ERRORS as e:
         self._result = {"error": e}
     self._usable = ok != False  # still usable if ok is True or None
     if self._result.get("status_code", None):
         # Client is functional. See comment in get().
         self._put_user_agent(curl)
     else:
         curl.close()
     if not ok:
         _logger.debug(
             "Request fail: PUT %s => %s: %s", url, type(self._result["error"]), str(self._result["error"])
         )
         return False
     _logger.info(
         "PUT %s: %s bytes in %s msec (%.3f MiB/sec)",
         self._result["status_code"],
         len(body),
         t.msecs,
         (len(body) / (1024.0 * 1024)) / t.secs if t.secs > 0 else 0,
     )
     if self.upload_counter:
         self.upload_counter.add(len(body))
     return True