def create(self, env, vrs, account, container, versions=None): """ Perform a container PUT request :param env: WSGI environment for original request :param vrs: API version, e.g. "v1" :param account: account in which to create the container :param container: container name :param versions: value for X-Versions-Location header (for container versioning) :returns: None :raises: HTTPException on failure (non-2xx response) """ env = env.copy() env['REQUEST_METHOD'] = 'PUT' env["PATH_INFO"] = "/%s/%s/%s" % (vrs, account, container) if versions: env['HTTP_X_VERSIONS_LOCATION'] = versions resp_iter = self._app_call(env) # The body of a PUT response is either empty or very short (e.g. error # message), so we can get away with slurping the whole thing. body = ''.join(resp_iter) close_if_possible(resp_iter) status_int = int(self._response_status.split(' ', 1)[0]) if not http.is_success(status_int): raise swob.HTTPException(status=self._response_status, headers=self._response_headers, body=friendly_error(body))
def test_delete_object(self): x = expirer.ObjectExpirer({}, logger=self.logger) actual_obj = 'actual_obj' timestamp = int(time()) reclaim_ts = timestamp - x.reclaim_age account = 'account' container = 'container' obj = 'obj' http_exc = { resp_code: internal_client.UnexpectedResponse( str(resp_code), swob.HTTPException(status=resp_code)) for resp_code in {404, 412, 500} } exc_other = Exception() def check_call_to_delete_object(exc, ts, should_pop): x.logger.clear() start_reports = x.report_objects with mock.patch.object(x, 'delete_actual_object', side_effect=exc) as delete_actual: with mock.patch.object(x, 'pop_queue') as pop_queue: x.delete_object(actual_obj, ts, account, container, obj, False) delete_actual.assert_called_once_with(actual_obj, ts, False) log_lines = x.logger.get_lines_for_level('error') if should_pop: pop_queue.assert_called_once_with(account, container, obj) self.assertEqual(start_reports + 1, x.report_objects) self.assertFalse(log_lines) else: self.assertFalse(pop_queue.called) self.assertEqual(start_reports, x.report_objects) self.assertEqual(1, len(log_lines)) if isinstance(exc, internal_client.UnexpectedResponse): self.assertEqual( log_lines[0], 'Unexpected response while deleting object ' 'account container obj: %s' % exc.resp.status_int) else: self.assertTrue(log_lines[0].startswith( 'Exception while deleting object ' 'account container obj')) # verify pop_queue logic on exceptions for exc, ts, should_pop in [(None, timestamp, True), (http_exc[404], timestamp, False), (http_exc[412], timestamp, False), (http_exc[500], reclaim_ts, False), (exc_other, reclaim_ts, False), (http_exc[404], reclaim_ts, True), (http_exc[412], reclaim_ts, True)]: try: check_call_to_delete_object(exc, ts, should_pop) except AssertionError as err: self.fail("Failed on %r at %f: %s" % (exc, ts, err))
def handle404(self, reqorig, url, container, obj): """ Return a swob.Response which fetches the thumbnail from the thumb host and returns it. Note also that the thumb host might write it out to Swift so it won't 404 next time. """ # upload doesn't like our User-agent, otherwise we could call it # using urllib2.url() thumbor_opener = urllib2.build_opener(DumbRedirectHandler()) # Pass on certain headers from Varnish to Thumbor thumbor_opener.addheaders = [] if reqorig.headers.get('User-Agent') is not None: thumbor_opener.addheaders.append( ('User-Agent', reqorig.headers.get('User-Agent'))) else: thumbor_opener.addheaders.append(('User-Agent', self.user_agent)) for header_to_pass in [ 'X-Forwarded-For', 'X-Forwarded-Proto', 'Accept', 'Accept-Encoding', 'X-Original-URI' ]: if reqorig.headers.get(header_to_pass) is not None: header = (header_to_pass, reqorig.headers.get(header_to_pass)) thumbor_opener.addheaders.append(header) # At least in theory, we shouldn't be handing out links to originals # that we don't have (or in the case of thumbs, can't generate). # However, someone may have a formerly valid link to a file, so we # should do them the favor of giving them a 404. try: thumbor_encodedurl = self.thumborify_url(reqorig, self.thumborhost) upcopy = thumbor_opener.open(thumbor_encodedurl) except urllib2.HTTPError as error: # Wrap the urllib2 HTTPError into a swob HTTPException status = error.code body = error.fp.read() headers = error.hdrs.items() if status not in swob.RESPONSE_REASONS: # Generic status description in case of unknown status reasons. status = "%s Error" % status return swob.HTTPException(status=status, body=body, headers=headers) except urllib2.URLError as error: msg = 'There was a problem while contacting the thumbnailing service: %s' % \ error.reason return swob.HTTPServiceUnavailable(msg) # We successfully generated a thumbnail on the active DC, send the same request # blindly to the inactive DC to populate Swift there, not waiting for the response inactivedc_encodedurl = self.thumborify_url( reqorig, self.inactivedc_thumborhost) eventlet.spawn(self.inactivedc_request, thumbor_opener, inactivedc_encodedurl) # get the Content-Type. uinfo = upcopy.info() c_t = uinfo.gettype() resp = swob.Response(app_iter=upcopy, content_type=c_t) headers_whitelist = [ 'Content-Length', 'Content-Disposition', 'Last-Modified', 'Accept-Ranges', 'XKey', 'Thumbor-Engine', 'Server', 'Nginx-Request-Date', 'Nginx-Response-Date', 'Thumbor-Processing-Time', 'Thumbor-Processing-Utime', 'Thumbor-Request-Id', 'Thumbor-Request-Date' ] # add in the headers if we've got them for header in headers_whitelist: if (uinfo.getheader(header) != ''): resp.headers[header] = uinfo.getheader(header) # also add CORS; see also our CORS middleware resp.headers['Access-Control-Allow-Origin'] = '*' return resp
def handle404(self, reqorig, url, container, obj): """ Return a swob.Response which fetches the thumbnail from the thumb host and returns it. Note also that the thumb host might write it out to Swift so it won't 404 next time. """ # go to the thumb media store for unknown files reqorig.host = self.thumbhost # upload doesn't like our User-agent, otherwise we could call it # using urllib2.url() proxy_handler = urllib2.ProxyHandler({'http': self.thumbhost}) redirect_handler = DumbRedirectHandler() opener = urllib2.build_opener(redirect_handler, proxy_handler) # Thumbor doesn't need (and doesn't like) the proxy thumbor_opener = urllib2.build_opener(redirect_handler) # Pass on certain headers from the caller squid to the scalers opener.addheaders = [] if reqorig.headers.get('User-Agent') is not None: opener.addheaders.append( ('User-Agent', reqorig.headers.get('User-Agent'))) else: opener.addheaders.append(('User-Agent', self.user_agent)) for header_to_pass in [ 'X-Forwarded-For', 'X-Forwarded-Proto', 'Accept', 'Accept-Encoding', 'X-Original-URI' ]: if reqorig.headers.get(header_to_pass) is not None: opener.addheaders.append( (header_to_pass, reqorig.headers.get(header_to_pass))) thumbor_opener.addheaders = opener.addheaders # At least in theory, we shouldn't be handing out links to originals # that we don't have (or in the case of thumbs, can't generate). # However, someone may have a formerly valid link to a file, so we # should do them the favor of giving them a 404. try: # break apach the url, url-encode it, and put it back together urlobj = list(urlparse.urlsplit(reqorig.url)) # encode the URL but don't encode %s and /s urlobj[2] = urllib2.quote(urlobj[2], '%/') encodedurl = urlparse.urlunsplit(urlobj) # Thumbor never needs URL mangling and it needs a different host if self.thumborhost: thumbor_reqorig = swob.Request(reqorig.environ.copy()) thumbor_reqorig.host = self.thumborhost thumbor_urlobj = list(urlparse.urlsplit(thumbor_reqorig.url)) thumbor_urlobj[2] = urllib2.quote(thumbor_urlobj[2], '%/') thumbor_encodedurl = urlparse.urlunsplit(thumbor_urlobj) # if sitelang, we're supposed to mangle the URL so that # http://upload.wm.o/wikipedia/commons/thumb/a/a2/Foo_.jpg/330px-Foo_.jpg # changes to # http://commons.wp.o/w/thumb_handler.php/a/a2/Foo_.jpg/330px-Foo_.jpg if self.backend_url_format == 'sitelang': match = re.match( r'^http://(?P<host>[^/]+)/(?P<proj>[^-/]+)/(?P<lang>[^/]+)/thumb/(?P<path>.+)', encodedurl) if match: proj = match.group('proj') lang = match.group('lang') # and here are all the legacy special cases, imported from thumb_handler.php if (proj == 'wikipedia'): if (lang in ['meta', 'commons', 'internal', 'grants']): proj = 'wikimedia' if (lang in ['mediawiki']): lang = 'www' proj = 'mediawiki' hostname = '%s.%s.%s' % (lang, proj, self.tld) if (proj == 'wikipedia' and lang == 'sources'): # yay special case hostname = 'wikisource.%s' % self.tld # ok, replace the URL with just the part starting with thumb/ # take off the first two parts of the path # (eg /wikipedia/commons/); make sure the string starts # with a / encodedurl = 'http://%s/w/thumb_handler.php/%s' % ( hostname, match.group('path')) # add in the X-Original-URI with the swift got (minus the hostname) opener.addheaders.append( ('X-Original-URI', list(urlparse.urlsplit(reqorig.url))[2])) else: # ASSERT this code should never be hit since only thumbs # should call the 404 handler self.logger.warn( "non-thumb in 404 handler! encodedurl = %s" % encodedurl) resp = swob.HTTPNotFound('Unexpected error') return resp else: # log the result of the match here to test and make sure it's # sane before enabling the config match = re.match( r'^http://(?P<host>[^/]+)/(?P<proj>[^-/]+)/(?P<lang>[^/]+)/thumb/(?P<path>.+)', encodedurl) if match: proj = match.group('proj') lang = match.group('lang') self.logger.warn( "sitelang match has proj %s lang %s encodedurl %s" % (proj, lang, encodedurl)) else: self.logger.warn("no sitelang match on encodedurl: %s" % encodedurl) # To turn thumbor off and have thumbnail traffic served by image scalers, # replace the line below with this one: # upcopy = opener.open(encodedurl) upcopy = thumbor_opener.open(thumbor_encodedurl) except urllib2.HTTPError as error: # Wrap the urllib2 HTTPError into a swob HTTPException status = error.code if status not in swob.RESPONSE_REASONS: # Generic status description in case of unknown status reasons. status = "%s Error" % status return swob.HTTPException(status=status, body=error.msg, headers=error.hdrs.items()) except urllib2.URLError as error: msg = 'There was a problem while contacting the thumbnailing service: %s' % \ error.reason return swob.HTTPServiceUnavailable(msg) # get the Content-Type. uinfo = upcopy.info() c_t = uinfo.gettype() resp = swob.Response(app_iter=upcopy, content_type=c_t) headers_whitelist = [ 'Content-Length', 'Content-Disposition', 'Last-Modified', 'Accept-Ranges', 'XKey', 'Thumbor-Engine', 'Server', 'Nginx-Request-Date', 'Nginx-Response-Date', 'Thumbor-Processing-Time', 'Thumbor-Processing-Utime', 'Thumbor-Request-Id', 'Thumbor-Request-Date' ] # add in the headers if we've got them for header in headers_whitelist: if (uinfo.getheader(header) != ''): resp.headers[header] = uinfo.getheader(header) # also add CORS; see also our CORS middleware resp.headers['Access-Control-Allow-Origin'] = '*' return resp