def __call__(self, req):
try:
vrs, acc, con, obj = req.split_path(2, 4, rest_with_last=True)
except ValueError:
return self.app
# All requests
if req.method == 'GET':
account_info = get_account_info(req.environ, self.app)
if account_info:
recycled = account_info['meta'].get('recycled', '')
delete_date = account_info['meta'].get('earliest-delete-date',
'')
if recycled == 'yes' and delete_date != '':
return swob.HTTPNotFound(
headers={
'x-account-meta-recycled': 'yes',
'x-account-meta-earliest-delete-date': delete_date
},
body=
("Account is marked for deletion. "
"Send X-Remove-Account-Meta-Recycled header via POST to undelete."
))
# Account specific requests
if con is None:
if req.method == 'DELETE':
account_info = get_account_info(req.environ, self.app)
if account_info:
try:
recycled = account_info['meta'].get('recycled', '')
delete_date = int(account_info['meta'].get(
'earliest-delete-date', '0'))
if recycled != "yes":
return swob.HTTPMethodNotAllowed(
content_type="text/plain",
body=
("Account cannot be deleted directly. "
"Send 'X-Account-Meta-Recycled: yes' in POST request to mark for deletion.\n"
))
if time() < delete_date:
return swob.HTTPMethodNotAllowed(
content_type="text/plain",
headers={
'x-account-meta-recycled':
'yes',
'x-account-meta-earliest-delete-date':
delete_date
},
body=
("Account cannot be deleted yet, "
"X-Account-Meta-Earliest-Delete-Date not reached yet.\n"
))
return self.app
except ValueError:
return swob.HTTPInternalError(
content_type="text/plain",
body=(
"Internal error. Cannot read recycled state.\n"
))
if req.method == 'POST':
if 'x-account-meta-earliest-delete-date' in req.headers or 'x-remove-account-meta-earliest-delete-date' in req.headers:
return swob.HTTPMethodNotAllowed(
content_type="text/plain",
body=("Header X-Account-Meta-Earliest-Delete-Date "
"cannot be set manually.\n"))
if 'x-account-meta-recycled' in req.headers and req.headers[
'x-account-meta-recycled'] == "yes":
req.headers['x-account-meta-recycled'] = "yes"
req.headers['x-account-meta-earliest-delete-date'] = str(
int(time()) + self.account_recycled_seconds)
return self.app
if 'x-remove-account-meta-recycled' in req.headers:
req.headers['x-remove-account-meta-recycled'] = "x"
req.headers[
'x-remove-account-meta-earliest-delete-date'] = "x"
return self.app
return self.app
# Container specific requests
if obj is None:
return self.app
# Object specific requests
if req.method == 'GET':
object_info = get_object_info(req.environ, self.app)
if object_info:
recycled = object_info['meta'].get('recycled', '')
delete_date = object_info['meta'].get('delete-date', '')
if recycled == 'yes':
return swob.HTTPNotFound(
headers={
'x-object-meta-recycled': 'yes',
'x-object-meta-delete-date': delete_date
},
body=
("Object is marked for deletion. "
"Send X-Remove-Object-Meta-Recycled header via POST to undelete.\n"
))
if req.method == 'DELETE':
return swob.HTTPMethodNotAllowed(
content_type="text/plain",
body=(
"DELETE requests are not allowed. "
"Use POST with 'X-Object-Meta-Recycled: yes' instead.\n"))
if req.method == 'POST' or req.method == 'PUT':
if 'x-delete-at' in req.headers or 'x-delete-after' in req.headers or 'x-object-meta-delete-date' in req.headers:
return swob.HTTPMethodNotAllowed(
content_type="text/plain",
body=
("Setting X-Delete-At/X-Delete-After/X-Object-Meta-Delete-Date directly is not allowed. "
"Use POST with 'X-Object-Meta-Recycled: yes' instead.\n"))
if 'x-object-meta-recycled' in req.headers:
if req.headers['x-object-meta-recycled'] != "yes":
return swob.HTTPBadRequest(
content_type="text/plain",
body=("Invalid value for X-Object-Meta-Recycled. "
"Only 'yes' is allowed.\n"))
req.headers['x-object-meta-recycled'] = "yes"
req.headers['x-object-meta-delete-date'] = str(
int(time()) + self.object_recycle_keep_seconds)
req.headers['x-delete-after'] = str(
self.object_recycle_keep_seconds)
return self.app
if 'x-remove-object-meta-recycled' in req.headers:
req.headers['x-remove-object-meta-recycled'] = "x"
req.headers['x-remove-object-meta-delete-date'] = "x"
req.headers['x-remove-delete-at'] = "x"
req.headers['x-remove-delete-after'] = "x"
return self.app
return self.app
def _handle_request(self, env, start_response):
req = swob.Request(env)
# Double (or triple, etc.) slashes in the URL should be ignored;
# collapse them. fixes T34864
req.path_info = re.sub(r'/{2,}', '/', req.path_info)
# Keep a copy of the original request so we can ask the scalers for it
reqorig = swob.Request(req.environ.copy())
# Containers have 5 components: project, language, repo, zone, and shard.
# If there's no zone in the URL, the zone is assumed to be 'public' (for b/c).
# Shard is optional (and configurable), and is only used for large containers.
#
# Projects are wikipedia, wikinews, etc.
# Languages are en, de, fr, commons, etc.
# Repos are local, timeline, etc.
# Zones are public, thumb, temp, etc.
# Shard is extracted from "hash paths" in the URL and is 2 hex digits.
#
# These attributes are mapped to container names in the form of either:
# (a) proj-lang-repo-zone (if not sharded)
# (b) proj-lang-repo-zone.shard (if sharded)
# (c) global-data-repo-zone (if not sharded)
# (d) global-data-repo-zone.shard (if sharded)
#
# Rewrite wiki-global URLs of these forms:
# (a) http://upload.wikimedia.org/math/<relpath>
# => http://msfe/v1/AUTH_<hash>/global-data-math-render/<relpath>
# (b) http://upload.wikimedia.org/<proj>/<lang>/math/<relpath> (legacy)
# => http://msfe/v1/AUTH_<hash>/global-data-math-render/<relpath>
#
# Rewrite wiki-relative URLs of these forms:
# (a) http://upload.wikimedia.org/<proj>/<lang>/<relpath>
# => http://msfe/v1/AUTH_<hash>/<proj>-<lang>-local-public/<relpath>
# (b) http://upload.wikimedia.org/<proj>/<lang>/archive/<relpath>
# => http://msfe/v1/AUTH_<hash>/<proj>-<lang>-local-public/archive/<relpath>
# (c) http://upload.wikimedia.org/<proj>/<lang>/thumb/<relpath>
# => http://msfe/v1/AUTH_<hash>/<proj>-<lang>-local-thumb/<relpath>
# (d) http://upload.wikimedia.org/<proj>/<lang>/thumb/archive/<relpath>
# => http://msfe/v1/AUTH_<hash>/<proj>-<lang>-local-thumb/archive/<relpath>
# (e) http://upload.wikimedia.org/<proj>/<lang>/thumb/temp/<relpath>
# => http://msfe/v1/AUTH_<hash>/<proj>-<lang>-local-thumb/temp/<relpath>
# (f) http://upload.wikimedia.org/<proj>/<lang>/transcoded/<relpath>
# => http://msfe/v1/AUTH_<hash>/<proj>-<lang>-local-transcoded/<relpath>
# (g) http://upload.wikimedia.org/<proj>/<lang>/timeline/<relpath>
# => http://msfe/v1/AUTH_<hash>/<proj>-<lang>-timeline-render/<relpath>
# regular uploads
match = re.match(
(r'^/(?P<proj>[^/]+)/(?P<lang>[^/]+)/'
r'((?P<zone>transcoded|thumb)/)?'
r'(?P<path>((temp|archive)/)?[0-9a-f]/(?P<shard>[0-9a-f]{2})/.+)$'
), req.path)
if match:
proj = match.group('proj')
lang = match.group('lang')
repo = 'local' # the upload repo name is "local"
# Get the repo zone (if not provided that means "public")
zone = (match.group('zone') if match.group('zone') else 'public')
# Get the object path relative to the zone (and thus container)
obj = match.group('path') # e.g. "archive/a/ab/..."
shard = match.group('shard')
# timeline renderings
if match is None:
# /wikipedia/en/timeline/a876297c277d80dfd826e1f23dbfea3f.png
match = re.match(
r'^/(?P<proj>[^/]+)/(?P<lang>[^/]+)/(?P<repo>timeline)/(?P<path>.+)$',
req.path)
if match:
proj = match.group('proj') # wikipedia
lang = match.group('lang') # en
repo = match.group('repo') # timeline
zone = 'render'
obj = match.group(
'path') # a876297c277d80dfd826e1f23dbfea3f.png
shard = ''
# math renderings
if match is None:
# /math/c/9/f/c9f2055dadfb49853eff822a453d9ceb.png
# /wikipedia/en/math/c/9/f/c9f2055dadfb49853eff822a453d9ceb.png (legacy)
match = re.match(
(r'^(/(?P<proj>[^/]+)/(?P<lang>[^/]+))?/(?P<repo>math)/'
r'(?P<path>(?P<shard1>[0-9a-f])/(?P<shard2>[0-9a-f])/.+)$'),
req.path)
if match:
proj = 'global'
lang = 'data'
repo = match.group('repo') # math
zone = 'render'
obj = match.group(
'path') # c/9/f/c9f2055dadfb49853eff822a453d9ceb.png
shard = match.group('shard1') + match.group('shard2') # c9
# score renderings
if match is None:
# /score/j/q/jqn99bwy8777srpv45hxjoiu24f0636/jqn99bwy.png
# /score/override-midi/8/i/8i9pzt87wtpy45lpz1rox8wusjkt7ki.ogg
match = re.match(r'^/(?P<repo>score)/(?P<path>.+)$', req.path)
if match:
proj = 'global'
lang = 'data'
repo = match.group('repo') # score
zone = 'render'
obj = match.group(
'path') # j/q/jqn99bwy8777srpv45hxjoiu24f0636/jqn99bwy.png
shard = ''
if match is None:
match = re.match(r'^/monitoring/(?P<what>.+)$', req.path)
if match:
what = match.group('what')
if what == 'frontend':
headers = {'Content-Type': 'application/octet-stream'}
resp = swob.Response(headers=headers, body="OK\n")
elif what == 'backend':
req.host = '127.0.0.1:%s' % self.bind_port
req.path_info = "/v1/%s/monitoring/backend" % self.account
app_iter = self._app_call(env)
status = self._get_status_int()
headers = self._response_headers
resp = swob.Response(status=status,
headers=headers,
app_iter=app_iter)
else:
resp = swob.HTTPNotFound('Monitoring type not found "%s"' %
(req.path))
return resp(env, start_response)
if match is None:
match = re.match(r'^/(?P<path>[^/]+)?$', req.path)
# /index.html /favicon.ico /robots.txt etc.
# serve from a default "root" container
if match:
path = match.group('path')
if not path:
path = 'index.html'
req.host = '127.0.0.1:%s' % self.bind_port
req.path_info = "/v1/%s/root/%s" % (self.account, path)
app_iter = self._app_call(env)
status = self._get_status_int()
headers = self._response_headers
resp = swob.Response(status=status,
headers=headers,
app_iter=app_iter)
return resp(env, start_response)
# Internally rewrite the URL based on the regex it matched...
if match:
# Get the per-project "conceptual" container name, e.g. "<proj><lang><repo><zone>"
container = "%s-%s-%s-%s" % (proj, lang, repo, zone)
# Add 2-digit shard to the container if it is supposed to be sharded.
# We may thus have an "actual" container name like "<proj><lang><repo><zone>.<shard>"
if container in self.shard_container_list:
container += ".%s" % shard
# Save a url with just the account name in it.
req.path_info = "/v1/%s" % (self.account)
port = self.bind_port
req.host = '127.0.0.1:%s' % port
url = req.url[:]
# Create a path to our object's name.
req.path_info = "/v1/%s/%s/%s" % (self.account, container,
urllib2.unquote(obj))
# self.logger.warn("new path is %s" % req.path_info)
# do_start_response just remembers what it got called with,
# because our 404 handler will generate a different response.
app_iter = self._app_call(env)
status = self._get_status_int()
headers = self._response_headers
if status == 404:
# only send thumbs to the 404 handler; just return a 404 for everything else.
if repo == 'local' and zone == 'thumb':
resp = self.handle404(reqorig, url, container, obj)
return resp(env, start_response)
else:
resp = swob.HTTPNotFound('File not found: %s' % req.path)
return resp(env, start_response)
else:
if zone == 'thumb':
for key, value in headers:
if key == 'X-Delete-At' and self.thumbnail_update_expiry_headers:
# Update expiry header asynchronously
eventlet.spawn(self.update_expiry, env)
break
# Return the response verbatim
return swob.Response(status=status,
headers=headers,
app_iter=app_iter)(env, start_response)
else:
resp = swob.HTTPNotFound('Regexp failed to match URI: "%s"' %
(req.path))
return resp(env, start_response)
def handle404(self, reqorig, url, container, obj):
"""
Return a swob.Response which fetches the thumbnail from the thumb
host and returns it. Note also that the thumb host might write it out
to Swift so it won't 404 next time.
"""
# go to the thumb media store for unknown files
reqorig.host = self.thumbhost
# upload doesn't like our User-agent, otherwise we could call it
# using urllib2.url()
proxy_handler = urllib2.ProxyHandler({'http': self.thumbhost})
redirect_handler = DumbRedirectHandler()
opener = urllib2.build_opener(redirect_handler, proxy_handler)
# Thumbor doesn't need (and doesn't like) the proxy
thumbor_opener = urllib2.build_opener(redirect_handler)
# Pass on certain headers from the caller squid to the scalers
opener.addheaders = []
if reqorig.headers.get('User-Agent') is not None:
opener.addheaders.append(
('User-Agent', reqorig.headers.get('User-Agent')))
else:
opener.addheaders.append(('User-Agent', self.user_agent))
for header_to_pass in [
'X-Forwarded-For', 'X-Forwarded-Proto', 'Accept',
'Accept-Encoding', 'X-Original-URI'
]:
if reqorig.headers.get(header_to_pass) is not None:
opener.addheaders.append(
(header_to_pass, reqorig.headers.get(header_to_pass)))
thumbor_opener.addheaders = opener.addheaders
# At least in theory, we shouldn't be handing out links to originals
# that we don't have (or in the case of thumbs, can't generate).
# However, someone may have a formerly valid link to a file, so we
# should do them the favor of giving them a 404.
try:
# break apach the url, url-encode it, and put it back together
urlobj = list(urlparse.urlsplit(reqorig.url))
# encode the URL but don't encode %s and /s
urlobj[2] = urllib2.quote(urlobj[2], '%/')
encodedurl = urlparse.urlunsplit(urlobj)
# Thumbor never needs URL mangling and it needs a different host
if self.thumborhost:
thumbor_reqorig = swob.Request(reqorig.environ.copy())
thumbor_reqorig.host = self.thumborhost
thumbor_urlobj = list(urlparse.urlsplit(thumbor_reqorig.url))
thumbor_urlobj[2] = urllib2.quote(thumbor_urlobj[2], '%/')
thumbor_encodedurl = urlparse.urlunsplit(thumbor_urlobj)
# if sitelang, we're supposed to mangle the URL so that
# http://upload.wm.o/wikipedia/commons/thumb/a/a2/Foo_.jpg/330px-Foo_.jpg
# changes to
# http://commons.wp.o/w/thumb_handler.php/a/a2/Foo_.jpg/330px-Foo_.jpg
if self.backend_url_format == 'sitelang':
match = re.match(
r'^http://(?P<host>[^/]+)/(?P<proj>[^-/]+)/(?P<lang>[^/]+)/thumb/(?P<path>.+)',
encodedurl)
if match:
proj = match.group('proj')
lang = match.group('lang')
# and here are all the legacy special cases, imported from thumb_handler.php
if (proj == 'wikipedia'):
if (lang in ['meta', 'commons', 'internal', 'grants']):
proj = 'wikimedia'
if (lang in ['mediawiki']):
lang = 'www'
proj = 'mediawiki'
hostname = '%s.%s.%s' % (lang, proj, self.tld)
if (proj == 'wikipedia' and lang == 'sources'):
# yay special case
hostname = 'wikisource.%s' % self.tld
# ok, replace the URL with just the part starting with thumb/
# take off the first two parts of the path
# (eg /wikipedia/commons/); make sure the string starts
# with a /
encodedurl = 'http://%s/w/thumb_handler.php/%s' % (
hostname, match.group('path'))
# add in the X-Original-URI with the swift got (minus the hostname)
opener.addheaders.append(
('X-Original-URI',
list(urlparse.urlsplit(reqorig.url))[2]))
else:
# ASSERT this code should never be hit since only thumbs
# should call the 404 handler
self.logger.warn(
"non-thumb in 404 handler! encodedurl = %s" %
encodedurl)
resp = swob.HTTPNotFound('Unexpected error')
return resp
else:
# log the result of the match here to test and make sure it's
# sane before enabling the config
match = re.match(
r'^http://(?P<host>[^/]+)/(?P<proj>[^-/]+)/(?P<lang>[^/]+)/thumb/(?P<path>.+)',
encodedurl)
if match:
proj = match.group('proj')
lang = match.group('lang')
self.logger.warn(
"sitelang match has proj %s lang %s encodedurl %s" %
(proj, lang, encodedurl))
else:
self.logger.warn("no sitelang match on encodedurl: %s" %
encodedurl)
# To turn thumbor off and have thumbnail traffic served by image scalers,
# replace the line below with this one:
# upcopy = opener.open(encodedurl)
upcopy = thumbor_opener.open(thumbor_encodedurl)
except urllib2.HTTPError as error:
# Wrap the urllib2 HTTPError into a swob HTTPException
status = error.code
if status not in swob.RESPONSE_REASONS:
# Generic status description in case of unknown status reasons.
status = "%s Error" % status
return swob.HTTPException(status=status,
body=error.msg,
headers=error.hdrs.items())
except urllib2.URLError as error:
msg = 'There was a problem while contacting the thumbnailing service: %s' % \
error.reason
return swob.HTTPServiceUnavailable(msg)
# get the Content-Type.
uinfo = upcopy.info()
c_t = uinfo.gettype()
resp = swob.Response(app_iter=upcopy, content_type=c_t)
headers_whitelist = [
'Content-Length', 'Content-Disposition', 'Last-Modified',
'Accept-Ranges', 'XKey', 'Thumbor-Engine', 'Server',
'Nginx-Request-Date', 'Nginx-Response-Date',
'Thumbor-Processing-Time', 'Thumbor-Processing-Utime',
'Thumbor-Request-Id', 'Thumbor-Request-Date'
]
# add in the headers if we've got them
for header in headers_whitelist:
if (uinfo.getheader(header) != ''):
resp.headers[header] = uinfo.getheader(header)
# also add CORS; see also our CORS middleware
resp.headers['Access-Control-Allow-Origin'] = '*'
return resp