class AccountController(object):
"""WSGI controller for the account server."""
def __init__(self, conf):
self.logger = get_logger(conf, log_route='account-server')
self.root = conf.get('devices', '/srv/node')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
self.replicator_rpc = \
ReplicatorRpc(self.root, DATADIR, AccountBroker, self.mount_check)
def _get_account_broker(self, drive, part, account):
hsh = hash_path(account)
db_dir = storage_directory(DATADIR, part, hsh)
db_path = os.path.join(self.root, drive, db_dir, hsh + '.db')
return AccountBroker(db_path, account=account, logger=self.logger)
def DELETE(self, req):
"""Handle HTTP DELETE request."""
try:
drive, part, account = split_path(unquote(req.path), 3)
except ValueError, err:
return HTTPBadRequest(body=str(err), content_type='text/plain',
request=req)
if self.mount_check and not check_mount(self.root, drive):
return Response(status='507 %s is not mounted' % drive)
if 'x-timestamp' not in req.headers or \
not check_float(req.headers['x-timestamp']):
return HTTPBadRequest(body='Missing timestamp', request=req,
content_type='text/plain')
broker = self._get_account_broker(drive, part, account)
if broker.is_deleted():
return HTTPNotFound(request=req)
broker.delete_db(req.headers['x-timestamp'])
return HTTPNoContent(request=req)
def complete_rsync(self, drive, db_file, args):
old_filename = os.path.join(self.root, drive, 'tmp', args[0])
if os.path.exists(db_file):
return HTTPNotFound()
if not os.path.exists(old_filename):
return HTTPNotFound()
broker = self.broker_class(old_filename)
broker.newid(args[0])
renamer(old_filename, db_file)
return HTTPNoContent()
def __call__(self, env, start_response):
if env['REQUEST_METHOD'] == 'GET':
if self.status == 200:
start_response(Response().status)
start_response({'Content-Type': 'text/xml'})
json_pattern = [
'"name":%s', '"last_modified":%s', '"hash":%s',
'"bytes":%s'
]
json_pattern = '{' + ','.join(json_pattern) + '}'
json_out = []
for b in self.objects:
name = simplejson.dumps(b[0])
time = simplejson.dumps(b[1])
json_out.append(json_pattern % (name, time, b[2], b[3]))
account_list = '[' + ','.join(json_out) + ']'
return account_list
elif self.status == 401:
start_response(HTTPUnauthorized().status)
start_response({})
elif self.status == 404:
start_response(HTTPNotFound().status)
start_response({})
else:
start_response(HTTPBadRequest().status)
start_response({})
elif env['REQUEST_METHOD'] == 'PUT':
if self.status == 201:
start_response(HTTPCreated().status)
start_response({})
elif self.status == 401:
start_response(HTTPUnauthorized().status)
start_response({})
elif self.status == 202:
start_response(HTTPAccepted().status)
start_response({})
else:
start_response(HTTPBadRequest().status)
start_response({})
elif env['REQUEST_METHOD'] == 'DELETE':
if self.status == 204:
start_response(HTTPNoContent().status)
start_response({})
elif self.status == 401:
start_response(HTTPUnauthorized().status)
start_response({})
elif self.status == 404:
start_response(HTTPNotFound().status)
start_response({})
elif self.status == 409:
start_response(HTTPConflict().status)
start_response({})
else:
start_response(HTTPBadRequest().status)
start_response({})
def entry_update(slug):
bucket = riak.bucket("entries")
username = request.remote_user
data = json.loads(request.body)
# validate in someway
data['author'] = username
obj = bucket.new(slug, data=data)
obj.store()
return HTTPNoContent()
class AccountController(object):
"""WSGI controller for the account server."""
def __init__(self, conf):
self.logger = get_logger(conf, log_route='account-server')
self.root = conf.get('devices', '/srv/node')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
self.replicator_rpc = ReplicatorRpc(self.root,
DATADIR,
AccountBroker,
self.mount_check,
logger=self.logger)
self.auto_create_account_prefix = \
conf.get('auto_create_account_prefix') or '.'
swift.common.db.DB_PREALLOCATION = \
conf.get('db_preallocation', 'f').lower() in TRUE_VALUES
def _get_account_broker(self, drive, part, account):
hsh = hash_path(account)
db_dir = storage_directory(DATADIR, part, hsh)
db_path = os.path.join(self.root, drive, db_dir, hsh + '.db')
return AccountBroker(db_path, account=account, logger=self.logger)
@public
def DELETE(self, req):
"""Handle HTTP DELETE request."""
start_time = time.time()
try:
drive, part, account = split_path(unquote(req.path), 3)
validate_device_partition(drive, part)
except ValueError, err:
self.logger.increment('DELETE.errors')
return HTTPBadRequest(body=str(err),
content_type='text/plain',
request=req)
if self.mount_check and not check_mount(self.root, drive):
self.logger.increment('DELETE.errors')
return HTTPInsufficientStorage(drive=drive, request=req)
if 'x-timestamp' not in req.headers or \
not check_float(req.headers['x-timestamp']):
self.logger.increment('DELETE.errors')
return HTTPBadRequest(body='Missing timestamp',
request=req,
content_type='text/plain')
broker = self._get_account_broker(drive, part, account)
if broker.is_deleted():
self.logger.timing_since('DELETE.timing', start_time)
return HTTPNotFound(request=req)
broker.delete_db(req.headers['x-timestamp'])
self.logger.timing_since('DELETE.timing', start_time)
return HTTPNoContent(request=req)
def resource_options_view(obj, request, resource, smart=True):
methods = set()
q = Query('json').filter(model=type(obj))
for action, res in list(q(request.app)):
if 'request_method' in action.predicates:
methods.add(action.predicates.get('request_method'))
@request.after
def _after(response):
response.headers.add(
'Access-Control-Allow-Methods',
', '.join(sorted(methods))
)
return HTTPNoContent()
def handle_add_user(self, request):
"""
Handles Rest requests from developers to have a user added. If the
account specified doesn't exist, it will also be added. Currently,
updating a user's information (password, admin access) must be done by
directly updating the sqlite database.
Valid URL paths:
* PUT /account/<account-name>/<user-name> - create the account
Valid headers:
* X-Auth-User-Key: <password>
* X-Auth-User-Admin: <true|false>
* X-Auth-User-Reseller-Admin: <true|false>
If the HTTP request returns with a 204, then the user was added,
and the storage url will be available in the X-Storage-Url header.
:param request: webob.Request object
"""
try:
_junk, account_name, user_name = \
split_path(request.path, minsegs=3)
except ValueError:
return HTTPBadRequest()
create_reseller_admin = \
request.headers.get('x-auth-user-reseller-admin') == 'true'
if create_reseller_admin and (
request.headers.get('X-Auth-Admin-User') != '.super_admin'
or request.headers.get('X-Auth-Admin-Key') !=
self.super_admin_key):
return HTTPUnauthorized(request=request)
create_account_admin = \
request.headers.get('x-auth-user-admin') == 'true'
if create_account_admin and \
not self.is_account_admin(request, account_name):
return HTTPForbidden(request=request)
if 'X-Auth-User-Key' not in request.headers:
return HTTPBadRequest(body='X-Auth-User-Key is required')
password = request.headers['x-auth-user-key']
storage_url = self.create_user(account_name, user_name, password,
create_account_admin,
create_reseller_admin)
if storage_url == 'already exists':
return HTTPConflict(body=storage_url)
if not storage_url:
return HTTPServiceUnavailable()
return HTTPNoContent(headers={'x-storage-url': storage_url})
def rsync_then_merge(self, drive, db_file, args):
old_filename = os.path.join(self.root, drive, 'tmp', args[0])
if not os.path.exists(db_file) or not os.path.exists(old_filename):
return HTTPNotFound()
new_broker = self.broker_class(old_filename)
existing_broker = self.broker_class(db_file)
point = -1
objects = existing_broker.get_items_since(point, 1000)
while len(objects):
new_broker.merge_items(objects)
point = objects[-1]['ROWID']
objects = existing_broker.get_items_since(point, 1000)
sleep()
new_broker.newid(args[0])
renamer(old_filename, db_file)
return HTTPNoContent()
def __call__(self, env, start_response):
if env['REQUEST_METHOD'] == 'GET' or env['REQUEST_METHOD'] == 'HEAD':
if self.status == 200:
start_response(Response().status)
start_response(self.response_headers)
if env['REQUEST_METHOD'] == 'GET':
return self.object_body
elif self.status == 401:
start_response(HTTPUnauthorized().status)
start_response({})
elif self.status == 404:
start_response(HTTPNotFound().status)
start_response({})
else:
start_response(HTTPBadRequest().status)
start_response({})
elif env['REQUEST_METHOD'] == 'PUT':
if self.status == 201:
start_response(HTTPCreated().status)
start_response({'etag': self.response_headers['etag']})
elif self.status == 401:
start_response(HTTPUnauthorized().status)
start_response({})
elif self.status == 404:
start_response(HTTPNotFound().status)
start_response({})
else:
start_response(HTTPBadRequest().status)
start_response({})
elif env['REQUEST_METHOD'] == 'DELETE':
if self.status == 204:
start_response(HTTPNoContent().status)
start_response({})
elif self.status == 401:
start_response(HTTPUnauthorized().status)
start_response({})
elif self.status == 404:
start_response(HTTPNotFound().status)
start_response({})
else:
start_response(HTTPBadRequest().status)
start_response({})
def __call__(self, env, start_response):
if env['REQUEST_METHOD'] == 'GET' or env['REQUEST_METHOD'] == 'HEAD':
if self.status == 200:
if 'HTTP_RANGE' in env:
resp = Response(body=self.object_body,
conditional_response=True)
return resp(env, start_response)
start_response(Response().status,
self.response_headers.items())
if env['REQUEST_METHOD'] == 'GET':
return self.object_body
elif self.status == 401:
start_response(HTTPUnauthorized().status, [])
elif self.status == 404:
start_response(HTTPNotFound().status, [])
else:
start_response(HTTPBadRequest().status, [])
elif env['REQUEST_METHOD'] == 'PUT':
if self.status == 201:
start_response(HTTPCreated().status,
[('etag', self.response_headers['etag'])])
elif self.status == 401:
start_response(HTTPUnauthorized().status, [])
elif self.status == 404:
start_response(HTTPNotFound().status, [])
else:
start_response(HTTPBadRequest().status, [])
elif env['REQUEST_METHOD'] == 'DELETE':
if self.status == 204:
start_response(HTTPNoContent().status, [])
elif self.status == 401:
start_response(HTTPUnauthorized().status, [])
elif self.status == 404:
start_response(HTTPNotFound().status, [])
else:
start_response(HTTPBadRequest().status, [])
return []
def handle_token(self, request):
"""
Handles ReST requests from Swift to validate tokens
Valid URL paths:
* GET /token/<token>
If the HTTP request returns with a 204, then the token is valid, the
TTL of the token will be available in the X-Auth-Ttl header, and a
comma separated list of the "groups" the user belongs to will be in the
X-Auth-Groups header.
:param request: webob.Request object
"""
try:
_junk, token = split_path(request.path, minsegs=2)
except ValueError:
return HTTPBadRequest()
# Retrieves (TTL, account, user, cfaccount) if valid, False otherwise
headers = {}
if 'Authorization' in request.headers:
validation = self.validate_s3_sign(request, token)
if validation:
headers['X-Auth-Account-Suffix'] = validation[3]
else:
validation = self.validate_token(token)
if not validation:
return HTTPNotFound()
groups = ['%s:%s' % (validation[1], validation[2]), validation[1]]
if validation[3]:
# admin access to a cfaccount or ".reseller_admin" to access to all
# accounts, including creating new ones.
groups.append(validation[3])
headers['X-Auth-TTL'] = validation[0]
headers['X-Auth-Groups'] = ','.join(groups)
return HTTPNoContent(headers=headers)
raise HTTPForbidden()
elif req.context.owner is None:
raise HTTPUnauthorized(_("No authenticated user"))
try:
registry.replace_members(self.options, req.context, image_id, body)
except exception.NotFound, e:
msg = "%s" % e
logger.debug(msg)
raise HTTPNotFound(msg, request=req, content_type='text/plain')
except exception.NotAuthorized, e:
msg = "%s" % e
logger.debug(msg)
raise HTTPNotFound(msg, request=req, content_type='text/plain')
return HTTPNoContent()
def add_member(self, req, image_id, member, body=None):
"""
Adds a membership to the image, or updates an existing one.
If a body is present, it is a dict with the following format::
{"member": {
"can_share": [True|False]
}}
If "can_share" is provided, the member's ability to share is
set accordingly. If it is not provided, existing memberships
remain unchanged and new memberships default to False.
"""
if req.context.read_only:
if self.mount_check and not check_mount(self.root, drive):
return Response(status='507 %s is not mounted' % drive)
broker = self._get_account_broker(drive, part, account)
if container: # put account container
if 'x-cf-trans-id' in req.headers:
broker.pending_timeout = 3
if req.headers.get('x-account-override-deleted', 'no').lower() != \
'yes' and broker.is_deleted():
return HTTPNotFound(request=req)
broker.put_container(container, req.headers['x-put-timestamp'],
req.headers['x-delete-timestamp'],
req.headers['x-object-count'],
req.headers['x-bytes-used'])
if req.headers['x-delete-timestamp'] > \
req.headers['x-put-timestamp']:
return HTTPNoContent(request=req)
else:
return HTTPCreated(request=req)
else: # put account
timestamp = normalize_timestamp(req.headers['x-timestamp'])
if not os.path.exists(broker.db_file):
broker.initialize(timestamp)
created = True
elif broker.is_status_deleted():
return HTTPForbidden(request=req, body='Recently deleted')
else:
created = broker.is_deleted()
broker.update_put_timestamp(timestamp)
if broker.is_deleted():
return HTTPConflict(request=req)
metadata = {}
def test_204(self, *args, **kw):
from webob.exc import HTTPNoContent
raise HTTPNoContent()
def handle_auth(self, request):
"""
Handles ReST requests from end users for a Swift cluster url and auth
token. This can handle all the various headers and formats that
existing auth systems used, so it's a bit of a chameleon.
Valid URL paths:
* GET /v1/<account-name>/auth
* GET /auth
* GET /v1.0
Valid headers:
* X-Auth-User: <account-name>:<user-name>
* X-Auth-Key: <password>
* X-Storage-User: [<account-name>:]<user-name>
The [<account-name>:] is only optional here if the
/v1/<account-name>/auth path is used.
* X-Storage-Pass: <password>
The (currently) preferred method is to use /v1.0 path and the
X-Auth-User and X-Auth-Key headers.
:param request: A webob.Request instance.
"""
try:
pathsegs = split_path(request.path,
minsegs=1,
maxsegs=3,
rest_with_last=True)
except ValueError:
return HTTPBadRequest()
if pathsegs[0] == 'v1' and pathsegs[2] == 'auth':
account = pathsegs[1]
user = request.headers.get('x-storage-user')
if not user:
user = request.headers.get('x-auth-user')
if not user or ':' not in user:
return HTTPUnauthorized()
account2, user = user.split(':', 1)
if account != account2:
return HTTPUnauthorized()
password = request.headers.get('x-storage-pass')
if not password:
password = request.headers.get('x-auth-key')
elif pathsegs[0] in ('auth', 'v1.0'):
user = request.headers.get('x-auth-user')
if not user:
user = request.headers.get('x-storage-user')
if not user or ':' not in user:
return HTTPUnauthorized()
account, user = user.split(':', 1)
password = request.headers.get('x-auth-key')
if not password:
password = request.headers.get('x-storage-pass')
else:
return HTTPBadRequest()
if not all((account, user, password)):
return HTTPUnauthorized()
self.purge_old_tokens()
with self.get_conn() as conn:
row = conn.execute(
'''
SELECT cfaccount, url, admin, reseller_admin FROM account
WHERE account = ? AND user = ? AND password = ?''',
(account, user, password)).fetchone()
if row is None:
return HTTPUnauthorized()
cfaccount = row[0]
url = row[1]
admin = row[2] == 't'
reseller_admin = row[3] == 't'
row = conn.execute(
'''
SELECT token FROM token WHERE account = ? AND user = ?''',
(account, user)).fetchone()
if row:
token = row[0]
else:
token = '%stk%s' % (self.reseller_prefix, uuid4().hex)
token_cfaccount = ''
if admin:
token_cfaccount = cfaccount
if reseller_admin:
token_cfaccount = '.reseller_admin'
conn.execute(
'''
INSERT INTO token
(token, created, account, user, cfaccount)
VALUES (?, ?, ?, ?, ?)''',
(token, time(), account, user, token_cfaccount))
conn.commit()
return HTTPNoContent(
headers={
'x-auth-token': token,
'x-storage-token': token,
'x-storage-url': url
})
def entry_delete(slug):
bucket = riak.bucket("entries")
obj = bucket.new(slug)
obj.delete()
return HTTPNoContent()
class ContainerController(object):
"""WSGI Controller for the container server."""
# Ensure these are all lowercase
save_headers = [
'x-container-read', 'x-container-write', 'x-container-sync-key',
'x-container-sync-to'
]
def __init__(self, conf):
self.logger = get_logger(conf, log_route='container-server')
self.root = conf.get('devices', '/srv/node/')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
self.node_timeout = int(conf.get('node_timeout', 3))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.allowed_sync_hosts = [
h.strip()
for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',')
if h.strip()
]
self.replicator_rpc = ReplicatorRpc(self.root,
DATADIR,
ContainerBroker,
self.mount_check,
logger=self.logger)
self.auto_create_account_prefix = \
conf.get('auto_create_account_prefix') or '.'
self.fs_object = None
def _get_container_broker(self, drive, part, account, container):
"""
Get a DB broker for the container.
:param drive: drive that holds the container
:param part: partition the container is in
:param account: account name
:param container: container name
:returns: ContainerBroker object
"""
if self.fs_object:
return DiskDir(self.root,
drive,
part,
account,
container,
self.logger,
fs_object=self.fs_object)
hsh = hash_path(account, container)
db_dir = storage_directory(DATADIR, part, hsh)
db_path = os.path.join(self.root, drive, db_dir, hsh + '.db')
return ContainerBroker(db_path,
account=account,
container=container,
logger=self.logger)
def account_update(self, req, account, container, broker):
"""
Update the account server with latest container info.
:param req: webob.Request object
:param account: account name
:param container: container name
:param borker: container DB broker object
:returns: if the account request returns a 404 error code,
HTTPNotFound response object, otherwise None.
"""
account_host = req.headers.get('X-Account-Host')
account_partition = req.headers.get('X-Account-Partition')
account_device = req.headers.get('X-Account-Device')
if all([account_host, account_partition, account_device]):
account_ip, account_port = account_host.rsplit(':', 1)
new_path = '/' + '/'.join([account, container])
info = broker.get_info()
account_headers = {
'x-put-timestamp': info['put_timestamp'],
'x-delete-timestamp': info['delete_timestamp'],
'x-object-count': info['object_count'],
'x-bytes-used': info['bytes_used'],
'x-trans-id': req.headers.get('x-trans-id', '-')
}
if req.headers.get('x-account-override-deleted', 'no').lower() == \
'yes':
account_headers['x-account-override-deleted'] = 'yes'
try:
with ConnectionTimeout(self.conn_timeout):
conn = http_connect(account_ip, account_port,
account_device, account_partition,
'PUT', new_path, account_headers)
with Timeout(self.node_timeout):
account_response = conn.getresponse()
account_response.read()
if account_response.status == 404:
return HTTPNotFound(request=req)
elif account_response.status < 200 or \
account_response.status > 299:
self.logger.error(
_('ERROR Account update failed '
'with %(ip)s:%(port)s/%(device)s (will retry '
'later): Response %(status)s %(reason)s'), {
'ip': account_ip,
'port': account_port,
'device': account_device,
'status': account_response.status,
'reason': account_response.reason
})
except (Exception, Timeout):
self.logger.exception(
_('ERROR account update failed with '
'%(ip)s:%(port)s/%(device)s (will retry later)'), {
'ip': account_ip,
'port': account_port,
'device': account_device
})
return None
def DELETE(self, req):
"""Handle HTTP DELETE request."""
try:
drive, part, account, container, obj = split_path(
unquote(req.path), 4, 5, True)
except ValueError, err:
return HTTPBadRequest(body=str(err),
content_type='text/plain',
request=req)
if 'x-timestamp' not in req.headers or \
not check_float(req.headers['x-timestamp']):
return HTTPBadRequest(body='Missing timestamp',
request=req,
content_type='text/plain')
if self.mount_check and not check_mount(self.root, drive):
return Response(status='507 %s is not mounted' % drive)
broker = self._get_container_broker(drive, part, account, container)
if account.startswith(self.auto_create_account_prefix) and obj and \
not os.path.exists(broker.db_file):
broker.initialize(
normalize_timestamp(
req.headers.get('x-timestamp') or time.time()))
if not os.path.exists(broker.db_file):
return HTTPNotFound()
if obj: # delete object
broker.delete_object(obj, req.headers.get('x-timestamp'))
return HTTPNoContent(request=req)
else:
# delete container
if not broker.empty():
return HTTPConflict(request=req)
existed = float(broker.get_info()['put_timestamp']) and \
not broker.is_deleted()
broker.delete_db(req.headers['X-Timestamp'])
if not broker.is_deleted():
return HTTPConflict(request=req)
resp = self.account_update(req, account, container, broker)
if resp:
return resp
if existed:
return HTTPNoContent(request=req)
return HTTPNotFound()
def resource_head_view(obj, request, resource, smart=True):
return HTTPNoContent()
def resource_delete_view(obj, request, resource, smart=True):
if not smart:
return resource.delete()
resource.delete()
return HTTPNoContent()