def HEAD(self, request):
"""Handle HTTP HEAD requests for the Swift Object Server."""
device, partition, account, container, obj = \
split_and_validate_path(request, 5, 5, True)
try:
disk_file = self.get_diskfile(
device, partition, account, container, obj)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
metadata = disk_file.read_metadata()
except (DiskFileNotExist, DiskFileQuarantined):
return HTTPNotFound(request=request, conditional_response=True)
response = Response(request=request, conditional_response=True)
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in metadata.iteritems():
if is_user_meta('object', key) or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
ts = metadata['X-Timestamp']
response.last_modified = math.ceil(float(ts))
# Needed for container sync feature
response.headers['X-Timestamp'] = ts
response.content_length = int(metadata['Content-Length'])
try:
response.content_encoding = metadata['Content-Encoding']
except KeyError:
pass
return response
def HEAD(self, request):
"""Handle HTTP HEAD requests for the Swift Object Server."""
device, partition, account, container, obj, policy_idx = get_name_and_placement(request, 5, 5, True)
try:
disk_file = self.get_diskfile(device, partition, account, container, obj, policy_idx=policy_idx)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
metadata = disk_file.read_metadata()
except DiskFileXattrNotSupported:
return HTTPInsufficientStorage(drive=device, request=request)
except (DiskFileNotExist, DiskFileQuarantined) as e:
headers = {}
if hasattr(e, "timestamp"):
headers["X-Backend-Timestamp"] = e.timestamp.internal
return HTTPNotFound(request=request, headers=headers, conditional_response=True)
response = Response(request=request, conditional_response=True)
response.headers["Content-Type"] = metadata.get("Content-Type", "application/octet-stream")
for key, value in metadata.iteritems():
if is_sys_or_user_meta("object", key) or key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata["ETag"]
ts = Timestamp(metadata["X-Timestamp"])
response.last_modified = math.ceil(float(ts))
# Needed for container sync feature
response.headers["X-Timestamp"] = ts.normal
response.headers["X-Backend-Timestamp"] = ts.internal
response.content_length = int(metadata["Content-Length"])
try:
response.content_encoding = metadata["Content-Encoding"]
except KeyError:
pass
return response
def HEAD(self, request):
"""Handle HTTP HEAD requests for the Swift Object Server."""
device, partition, account, container, obj = \
split_and_validate_path(request, 5, 5, True)
try:
disk_file = self.get_diskfile(
device, partition, account, container, obj)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
metadata = disk_file.read_metadata()
except (DiskFileNotExist, DiskFileQuarantined):
return HTTPNotFound(request=request, conditional_response=True)
response = Response(request=request, conditional_response=True)
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in metadata.iteritems():
if is_user_meta('object', key) or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
ts = metadata['X-Timestamp']
response.last_modified = math.ceil(float(ts))
# Needed for container sync feature
response.headers['X-Timestamp'] = ts
response.content_length = int(metadata['Content-Length'])
try:
response.content_encoding = metadata['Content-Encoding']
except KeyError:
pass
return response
def HEAD(self, request):
"""Handle HTTP HEAD requests for the Swift Object Server."""
device, partition, account, container, obj = \
split_and_validate_path(request, 5, 5, True)
try:
disk_file = self._diskfile(device, partition, account, container,
obj)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
if disk_file.is_deleted() or disk_file.is_expired():
return HTTPNotFound(request=request)
try:
file_size = disk_file.get_data_file_size()
except (DiskFileError, DiskFileNotExist):
disk_file.quarantine()
return HTTPNotFound(request=request)
response = Response(request=request, conditional_response=True)
response.headers['Content-Type'] = disk_file.metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in disk_file.metadata.iteritems():
if key.lower().startswith('x-object-meta-') or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = disk_file.metadata['ETag']
response.last_modified = float(disk_file.metadata['X-Timestamp'])
# Needed for container sync feature
response.headers['X-Timestamp'] = disk_file.metadata['X-Timestamp']
response.content_length = file_size
if 'Content-Encoding' in disk_file.metadata:
response.content_encoding = disk_file.metadata['Content-Encoding']
return response
def make_object_response(self, req, metadata, stream=None):
conditional_etag = None
if 'X-Backend-Etag-Is-At' in req.headers:
conditional_etag = metadata.get(
req.headers['X-Backend-Etag-Is-At'])
resp = Response(request=req, conditional_response=True,
conditional_etag=conditional_etag)
resp.headers['Content-Type'] = metadata.get(
'mime-type', 'application/octet-stream')
for k, v in metadata.iteritems():
if k.startswith("user."):
meta = k[5:]
if is_sys_or_user_meta('object', meta) or \
meta.lower() in self.allowed_headers:
resp.headers[meta] = v
resp.etag = metadata['hash'].lower()
ts = Timestamp(metadata['ctime'])
resp.last_modified = math.ceil(float(ts))
if stream:
resp.app_iter = stream
resp.content_length = int(metadata['length'])
try:
resp.content_encoding = metadata['encoding']
except KeyError:
pass
return resp
def HEAD(self, request):
"""Handle HTTP HEAD requests for the Swift Object Server."""
device, partition, account, container, obj = \
split_and_validate_path(request, 5, 5, True)
try:
disk_file = self._diskfile(device, partition, account, container,
obj)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
with disk_file.open():
if disk_file.is_deleted() or disk_file.is_expired():
return HTTPNotFound(request=request)
try:
file_size = disk_file.get_data_file_size()
except (DiskFileError, DiskFileNotExist):
disk_file.quarantine()
return HTTPNotFound(request=request)
metadata = disk_file.get_metadata()
response = Response(request=request, conditional_response=True)
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in metadata.iteritems():
if key.lower().startswith('x-object-meta-') or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
response.last_modified = float(metadata['X-Timestamp'])
# Needed for container sync feature
response.headers['X-Timestamp'] = metadata['X-Timestamp']
response.content_length = file_size
if 'Content-Encoding' in metadata:
response.content_encoding = metadata['Content-Encoding']
return response
def HEAD(self, request):
"""Handle HTTP HEAD requests for the Swift Object Server."""
device, partition, account, container, obj = self._parse_path(request)
try:
disk_file = self._diskfile(device, partition, account, container, obj)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
if disk_file.is_deleted() or disk_file.is_expired():
return HTTPNotFound(request=request)
try:
file_size = disk_file.get_data_file_size()
except (DiskFileError, DiskFileNotExist):
disk_file.quarantine()
return HTTPNotFound(request=request)
response = Response(request=request, conditional_response=True)
response.headers["Content-Type"] = disk_file.metadata.get("Content-Type", "application/octet-stream")
for key, value in disk_file.metadata.iteritems():
if key.lower().startswith("x-object-meta-") or key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = disk_file.metadata["ETag"]
response.last_modified = float(disk_file.metadata["X-Timestamp"])
# Needed for container sync feature
response.headers["X-Timestamp"] = disk_file.metadata["X-Timestamp"]
response.content_length = file_size
if "Content-Encoding" in disk_file.metadata:
response.content_encoding = disk_file.metadata["Content-Encoding"]
return response
def HEAD(self, request):
"""Handle HTTP HEAD requests for the Swift on File object server"""
device, partition, account, container, obj, policy = \
get_name_and_placement(request, 5, 5, True)
# Get DiskFile
try:
disk_file = self.get_diskfile(device, partition, account, container,
obj, policy=policy)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
# Read DiskFile metadata
try:
disk_file.open()
metadata = disk_file.get_metadata()
except (DiskFileNotExist, DiskFileQuarantined) as e:
headers = {}
if hasattr(e, 'timestamp'):
headers['X-Backend-Timestamp'] = e.timestamp.internal
return HTTPNotFound(request=request, headers=headers,
conditional_respose=True)
# Create and populate our response
response = Response(request=request, conditional_response=True)
response.headers['Content-Type'] = \
metadata.get('Content-Type', 'application/octet-stream')
for key, value in metadata.iteritems():
if is_sys_or_user_meta('object', key) or key.lower() in \
self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
ts = Timestamp(metadata['X-Timestamp'])
response.last_modified = math.ceil(float(ts))
# Needed for container sync feature
response.headers['X-Timestamp'] = ts.normal
response.headers['X-Backend-Timestamp'] = ts.internal
response.content_length = int(metadata['Content-Length'])
try:
response.content_encoding = metadata['Content-Encoding']
except KeyError:
pass
# (HPSS) Inject HPSS xattr metadata into headers
want_hpss_metadata = request.headers.get('X-HPSS-Get-Metadata',
False)
if config_true_value(want_hpss_metadata):
try:
hpss_headers = disk_file.read_hpss_system_metadata()
response.headers.update(hpss_headers)
except SwiftOnFileSystemIOError:
return HTTPServiceUnavailable(request=request)
if 'X-Object-Sysmeta-Update-Container' in response.headers:
self._sof_container_update(request, response)
response.headers.pop('X-Object-Sysmeta-Update-Container')
return response
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
device, partition, account, container, obj, policy = \
get_name_and_placement(request, 5, 5, True)
keep_cache = self.keep_cache_private or (
'X-Auth-Token' not in request.headers
and 'X-Storage-Token' not in request.headers)
try:
disk_file = self.get_diskfile(device,
partition,
account,
container,
obj,
policy=policy)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
with disk_file.open():
metadata = disk_file.get_metadata()
obj_size = int(metadata['Content-Length'])
file_x_ts = Timestamp(metadata['X-Timestamp'])
keep_cache = (self.keep_cache_private or
('X-Auth-Token' not in request.headers
and 'X-Storage-Token' not in request.headers))
conditional_etag = None
if 'X-Backend-Etag-Is-At' in request.headers:
conditional_etag = metadata.get(
request.headers['X-Backend-Etag-Is-At'])
response = Response(
app_iter=disk_file.reader(keep_cache=keep_cache),
request=request,
conditional_response=True,
conditional_etag=conditional_etag)
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in metadata.iteritems():
if is_sys_or_user_meta('object', key) or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
response.last_modified = math.ceil(float(file_x_ts))
response.content_length = obj_size
try:
response.content_encoding = metadata['Content-Encoding']
except KeyError:
pass
response.headers['X-Timestamp'] = file_x_ts.normal
response.headers['X-Backend-Timestamp'] = file_x_ts.internal
resp = request.get_response(response)
except DiskFileXattrNotSupported:
return HTTPInsufficientStorage(drive=device, request=request)
except (DiskFileNotExist, DiskFileQuarantined) as e:
headers = {}
if hasattr(e, 'timestamp'):
headers['X-Backend-Timestamp'] = e.timestamp.internal
resp = HTTPNotFound(request=request,
headers=headers,
conditional_response=True)
return resp
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
device, partition, account, container, obj, policy = \
get_name_and_placement(request, 5, 5, True)
print 'request',request
print 'device, partition, account, container,obj, policy', device,partition,account,container,obj,policy
keep_cache = self.keep_cache_private or (
'X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers)
try:
disk_file = self.get_diskfile(
device, partition, account, container, obj,
policy=policy)
print 'disk_file',disk_file
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
with disk_file.open():
metadata = disk_file.get_metadata()
obj_size = int(metadata['Content-Length'])
file_x_ts = Timestamp(metadata['X-Timestamp'])
keep_cache = (self.keep_cache_private or
('X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers))
conditional_etag = None
if 'X-Backend-Etag-Is-At' in request.headers:
conditional_etag = metadata.get(
request.headers['X-Backend-Etag-Is-At'])
response = Response(
app_iter=disk_file.reader(keep_cache=keep_cache),
request=request, conditional_response=True,
conditional_etag=conditional_etag)
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in metadata.iteritems():
if is_sys_or_user_meta('object', key) or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
response.last_modified = math.ceil(float(file_x_ts))
response.content_length = obj_size
try:
response.content_encoding = metadata[
'Content-Encoding']
except KeyError:
pass
response.headers['X-Timestamp'] = file_x_ts.normal
response.headers['X-Backend-Timestamp'] = file_x_ts.internal
resp = request.get_response(response)
except DiskFileXattrNotSupported:
return HTTPInsufficientStorage(drive=device, request=request)
except (DiskFileNotExist, DiskFileQuarantined) as e:
headers = {}
if hasattr(e, 'timestamp'):
headers['X-Backend-Timestamp'] = e.timestamp.internal
resp = HTTPNotFound(request=request, headers=headers,
conditional_response=True)
print 'resp',resp
return resp
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
device, partition, account, container, obj = \
split_and_validate_path(request, 5, 5, True)
keep_cache = self.keep_cache_private or (
'X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers)
try:
disk_file = self.get_diskfile(
device, partition, account, container, obj)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
with disk_file.open():
metadata = disk_file.get_metadata()
obj_size = int(metadata['Content-Length'])
file_x_ts = metadata['X-Timestamp']
file_x_ts_flt = float(file_x_ts)
try:
if_unmodified_since = request.if_unmodified_since
except (OverflowError, ValueError):
# catches timestamps before the epoch
return HTTPPreconditionFailed(request=request)
file_x_ts_utc = datetime.fromtimestamp(file_x_ts_flt, UTC)
if if_unmodified_since and file_x_ts_utc > if_unmodified_since:
return HTTPPreconditionFailed(request=request)
try:
if_modified_since = request.if_modified_since
except (OverflowError, ValueError):
# catches timestamps before the epoch
return HTTPPreconditionFailed(request=request)
if if_modified_since and file_x_ts_utc <= if_modified_since:
return HTTPNotModified(request=request)
keep_cache = (self.keep_cache_private or
('X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers))
response = Response(
app_iter=disk_file.reader(keep_cache=keep_cache),
request=request, conditional_response=True)
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in metadata.iteritems():
if is_user_meta('object', key) or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
response.last_modified = math.ceil(file_x_ts_flt)
response.content_length = obj_size
try:
response.content_encoding = metadata[
'Content-Encoding']
except KeyError:
pass
response.headers['X-Timestamp'] = file_x_ts
resp = request.get_response(response)
except (DiskFileNotExist, DiskFileQuarantined):
resp = HTTPNotFound(request=request, conditional_response=True)
return resp
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
device, partition, account, container, obj = \
split_and_validate_path(request, 5, 5, True)
keep_cache = self.keep_cache_private or (
'X-Auth-Token' not in request.headers
and 'X-Storage-Token' not in request.headers)
try:
disk_file = self.get_diskfile(device, partition, account,
container, obj)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
with disk_file.open():
metadata = disk_file.get_metadata()
obj_size = int(metadata['Content-Length'])
file_x_ts = metadata['X-Timestamp']
file_x_ts_flt = float(file_x_ts)
try:
if_unmodified_since = request.if_unmodified_since
except (OverflowError, ValueError):
# catches timestamps before the epoch
return HTTPPreconditionFailed(request=request)
file_x_ts_utc = datetime.fromtimestamp(file_x_ts_flt, UTC)
if if_unmodified_since and file_x_ts_utc > if_unmodified_since:
return HTTPPreconditionFailed(request=request)
try:
if_modified_since = request.if_modified_since
except (OverflowError, ValueError):
# catches timestamps before the epoch
return HTTPPreconditionFailed(request=request)
if if_modified_since and file_x_ts_utc <= if_modified_since:
return HTTPNotModified(request=request)
keep_cache = (self.keep_cache_private or
('X-Auth-Token' not in request.headers
and 'X-Storage-Token' not in request.headers))
response = Response(
app_iter=disk_file.reader(keep_cache=keep_cache),
request=request,
conditional_response=True)
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in metadata.iteritems():
if is_user_meta('object', key) or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
response.last_modified = math.ceil(file_x_ts_flt)
response.content_length = obj_size
try:
response.content_encoding = metadata['Content-Encoding']
except KeyError:
pass
response.headers['X-Timestamp'] = file_x_ts
resp = request.get_response(response)
except (DiskFileNotExist, DiskFileQuarantined):
resp = HTTPNotFound(request=request, conditional_response=True)
return resp
def HEAD(self, request):
"""Handle HTTP HEAD requests for the Swift Object Server."""
device, partition, account, container, obj, policy = \
get_name_and_placement(request, 5, 5, True)
try:
disk_file = self.get_diskfile(device,
partition,
account,
container,
obj,
policy=policy)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
metadata = disk_file.read_metadata()
except DiskFileXattrNotSupported:
return HTTPInsufficientStorage(drive=device, request=request)
except (DiskFileNotExist, DiskFileQuarantined) as e:
headers = {}
if hasattr(e, 'timestamp'):
headers['X-Backend-Timestamp'] = e.timestamp.internal
return HTTPNotFound(request=request,
headers=headers,
conditional_response=True)
conditional_etag = None
if 'X-Backend-Etag-Is-At' in request.headers:
conditional_etag = metadata.get(
request.headers['X-Backend-Etag-Is-At'])
response = Response(request=request,
conditional_response=True,
conditional_etag=conditional_etag)
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in metadata.iteritems():
if is_sys_or_user_meta('object', key) or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
ts = Timestamp(metadata['X-Timestamp'])
response.last_modified = math.ceil(float(ts))
# Needed for container sync feature
response.headers['X-Timestamp'] = ts.normal
response.headers['X-Backend-Timestamp'] = ts.internal
response.content_length = int(metadata['Content-Length'])
try:
response.content_encoding = metadata['Content-Encoding']
except KeyError:
pass
return response
def GET(self, req):
"""
Load ring data from .gz files.
"""
self._reload()
ring_info = self._to_dict()
gz_file_to_ring_info = json.dumps(ring_info, sort_keys=True)
ring_info_json_md5 = self._ringinfo_md5(gz_file_to_ring_info)
resp = Response(request=req)
resp.etag = ring_info_json_md5
resp.content_length = len(gz_file_to_ring_info)
resp.body = gz_file_to_ring_info
resp.status_init = 200
resp.content_type = 'application/json; charset=UTF-8'
return resp
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
device, partition, account, container, obj, policy_idx = get_name_and_placement(request, 5, 5, True)
keep_cache = self.keep_cache_private or (
"X-Auth-Token" not in request.headers and "X-Storage-Token" not in request.headers
)
try:
disk_file = self.get_diskfile(device, partition, account, container, obj, policy_idx=policy_idx)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
with disk_file.open():
metadata = disk_file.get_metadata()
obj_size = int(metadata["Content-Length"])
file_x_ts = Timestamp(metadata["X-Timestamp"])
keep_cache = self.keep_cache_private or (
"X-Auth-Token" not in request.headers and "X-Storage-Token" not in request.headers
)
response = Response(
app_iter=disk_file.reader(keep_cache=keep_cache), request=request, conditional_response=True
)
response.headers["Content-Type"] = metadata.get("Content-Type", "application/octet-stream")
for key, value in metadata.iteritems():
if is_sys_or_user_meta("object", key) or key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata["ETag"]
response.last_modified = math.ceil(float(file_x_ts))
response.content_length = obj_size
try:
response.content_encoding = metadata["Content-Encoding"]
except KeyError:
pass
response.headers["X-Timestamp"] = file_x_ts.normal
response.headers["X-Backend-Timestamp"] = file_x_ts.internal
resp = request.get_response(response)
except DiskFileXattrNotSupported:
return HTTPInsufficientStorage(drive=device, request=request)
except (DiskFileNotExist, DiskFileQuarantined) as e:
headers = {}
if hasattr(e, "timestamp"):
headers["X-Backend-Timestamp"] = e.timestamp.internal
resp = HTTPNotFound(request=request, headers=headers, conditional_response=True)
return resp
def get_working_response(self, req):
source, node = self._get_source_and_node()
res = None
if source:
res = Response(request=req)
if req.method == "GET" and source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT):
res.app_iter = self._make_app_iter(node, source)
# See NOTE: swift_conn at top of file about this.
res.swift_conn = source.swift_conn
res.status = source.status
update_headers(res, source.getheaders())
if not res.environ:
res.environ = {}
res.environ["swift_x_timestamp"] = source.getheader("x-timestamp")
res.accept_ranges = "bytes"
res.content_length = source.getheader("Content-Length")
if source.getheader("Content-Type"):
res.charset = None
res.content_type = source.getheader("Content-Type")
return res
def make_object_response(self, req, metadata, stream=None):
conditional_etag = None
if 'X-Backend-Etag-Is-At' in req.headers:
conditional_etag = metadata.get(
req.headers['X-Backend-Etag-Is-At'])
resp = Response(request=req, conditional_response=True,
conditional_etag=conditional_etag)
if config_true_value(metadata['deleted']):
resp.headers['Content-Type'] = DELETE_MARKER_CONTENT_TYPE
else:
resp.headers['Content-Type'] = metadata.get(
'mime_type', 'application/octet-stream')
properties = metadata.get('properties')
if properties:
for k, v in properties.iteritems():
if is_sys_or_user_meta('object', k) or \
is_object_transient_sysmeta(k) or \
k.lower() in self.allowed_headers:
resp.headers[str(k)] = v
hash_ = metadata.get('hash')
if hash_ is not None:
hash_ = hash_.lower()
resp.headers['etag'] = hash_
resp.headers['x-object-sysmeta-version-id'] = metadata['version']
ts = Timestamp(metadata['ctime'])
resp.last_modified = math.ceil(float(ts))
if stream:
# Whether we are bothered with ranges or not, we wrap the
# stream in order to handle exceptions.
resp.app_iter = StreamRangeIterator(req, stream)
length_ = metadata.get('length')
if length_ is not None:
length_ = int(length_)
resp.content_length = length_
resp.content_encoding = metadata.get('encoding')
resp.accept_ranges = 'bytes'
return resp
def make_object_response(self, req, metadata, stream=None):
conditional_etag = None
if 'X-Backend-Etag-Is-At' in req.headers:
conditional_etag = metadata.get(
req.headers['X-Backend-Etag-Is-At'])
resp = Response(request=req,
conditional_response=True,
conditional_etag=conditional_etag)
if config_true_value(metadata['deleted']):
resp.headers['Content-Type'] = DELETE_MARKER_CONTENT_TYPE
else:
resp.headers['Content-Type'] = metadata.get(
'mime_type', 'application/octet-stream')
properties = metadata.get('properties')
if properties:
for k, v in properties.items():
if is_sys_or_user_meta('object', k) or \
is_object_transient_sysmeta(k) or \
k.lower() in self.allowed_headers:
resp.headers[str(k)] = v
hash_ = metadata.get('hash')
if hash_ is not None:
hash_ = hash_.lower()
resp.headers['etag'] = hash_
resp.headers['x-object-sysmeta-version-id'] = metadata['version']
resp.last_modified = int(metadata['mtime'])
if stream:
# Whether we are bothered with ranges or not, we wrap the
# stream in order to handle exceptions.
resp.app_iter = StreamRangeIterator(req, stream)
length_ = metadata.get('length')
if length_ is not None:
length_ = int(length_)
resp.content_length = length_
resp.content_encoding = metadata.get('encoding')
resp.accept_ranges = 'bytes'
return resp
def get_working_response(self, req):
source, node = self._get_source_and_node()
res = None
if source:
res = Response(request=req)
if req.method == 'GET' and \
source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT):
res.app_iter = self._make_app_iter(node, source)
# See NOTE: swift_conn at top of file about this.
res.swift_conn = source.swift_conn
res.status = source.status
update_headers(res, source.getheaders())
if not res.environ:
res.environ = {}
res.environ['swift_x_timestamp'] = \
source.getheader('x-timestamp')
res.accept_ranges = 'bytes'
res.content_length = source.getheader('Content-Length')
if source.getheader('Content-Type'):
res.charset = None
res.content_type = source.getheader('Content-Type')
return res
def make_object_response(self, req, metadata, stream=None, ranges=None):
conditional_etag = None
if 'X-Backend-Etag-Is-At' in req.headers:
conditional_etag = metadata.get(
req.headers['X-Backend-Etag-Is-At'])
resp = Response(request=req,
conditional_response=True,
conditional_etag=conditional_etag)
if config_true_value(metadata['deleted']):
resp.headers['Content-Type'] = DELETE_MARKER_CONTENT_TYPE
else:
resp.headers['Content-Type'] = metadata.get(
'mime_type', 'application/octet-stream')
properties = metadata.get('properties')
if properties:
for k, v in properties.iteritems():
if is_sys_or_user_meta('object', k) or \
is_object_transient_sysmeta(k) or \
k.lower() in self.allowed_headers:
resp.headers[str(k)] = v
resp.headers['etag'] = metadata['hash'].lower()
resp.headers['x-object-sysmeta-version-id'] = metadata['version']
ts = Timestamp(metadata['ctime'])
resp.last_modified = math.ceil(float(ts))
if stream:
if ranges:
resp.app_iter = StreamRangeIterator(stream)
else:
resp.app_iter = stream
resp.content_length = int(metadata['length'])
try:
resp.content_encoding = metadata['encoding']
except KeyError:
pass
resp.accept_ranges = 'bytes'
return resp
def make_object_response(self, req, metadata, stream=None, ranges=None):
conditional_etag = None
if 'X-Backend-Etag-Is-At' in req.headers:
conditional_etag = metadata.get(
req.headers['X-Backend-Etag-Is-At'])
resp = Response(request=req, conditional_response=True,
conditional_etag=conditional_etag)
if config_true_value(metadata['deleted']):
resp.headers['Content-Type'] = DELETE_MARKER_CONTENT_TYPE
else:
resp.headers['Content-Type'] = metadata.get(
'mime_type', 'application/octet-stream')
properties = metadata.get('properties')
if properties:
for k, v in properties.iteritems():
if is_sys_or_user_meta('object', k) or \
is_object_transient_sysmeta(k) or \
k.lower() in self.allowed_headers:
resp.headers[str(k)] = v
resp.headers['etag'] = metadata['hash'].lower()
resp.headers['x-object-sysmeta-version-id'] = metadata['version']
ts = Timestamp(metadata['ctime'])
resp.last_modified = math.ceil(float(ts))
if stream:
if ranges:
resp.app_iter = StreamRangeIterator(stream)
else:
resp.app_iter = stream
resp.content_length = int(metadata['length'])
try:
resp.content_encoding = metadata['encoding']
except KeyError:
pass
resp.accept_ranges = 'bytes'
return resp
etag = md5(
''.join(o['hash'] for o in listing)).hexdigest()
except KeyError:
return HTTPServerError('Invalid Manifest File',
request=req)
else:
content_length = 0
last_modified = resp.last_modified
etag = md5().hexdigest()
resp = Response(headers=resp.headers, request=req,
conditional_response=True)
resp.app_iter = SegmentedIterable(
self, lcontainer, listing, resp,
is_slo=(large_object == 'SLO'))
resp.content_length = content_length
resp.last_modified = last_modified
resp.etag = etag
resp.headers['accept-ranges'] = 'bytes'
# In case of a manifest file of nonzero length, the
# backend may have sent back a Content-Range header for
# the manifest. It's wrong for the client, though.
resp.content_range = None
return resp
@public
@cors_validation
@delay_denial
def GET(self, req):
"""Handler for HTTP GET requests."""
def GETorHEAD_base(self, req, server_type, partition, nodes, path, attempts):
"""
Base handler for HTTP GET or HEAD requests.
:param req: swob.Request object
:param server_type: server type
:param partition: partition
:param nodes: nodes
:param path: path for the request
:param attempts: number of attempts to try
:returns: swob.Response object
"""
statuses = []
reasons = []
bodies = []
sources = []
newest = config_true_value(req.headers.get("x-newest", "f"))
nodes = iter(nodes)
while len(statuses) < attempts:
try:
node = nodes.next()
except StopIteration:
break
if self.error_limited(node):
continue
try:
with ConnectionTimeout(self.app.conn_timeout):
headers = dict(req.headers)
headers["Connection"] = "close"
conn = http_connect(
node["ip"],
node["port"],
node["device"],
partition,
req.method,
path,
headers=headers,
query_string=req.query_string,
)
with Timeout(self.app.node_timeout):
possible_source = conn.getresponse()
# See NOTE: swift_conn at top of file about this.
possible_source.swift_conn = conn
except (Exception, Timeout):
self.exception_occurred(
node, server_type, _("Trying to %(method)s %(path)s") % {"method": req.method, "path": req.path}
)
continue
if self.is_good_source(possible_source):
# 404 if we know we don't have a synced copy
if not float(possible_source.getheader("X-PUT-Timestamp", 1)):
statuses.append(HTTP_NOT_FOUND)
reasons.append("")
bodies.append("")
self.close_swift_conn(possible_source)
else:
statuses.append(possible_source.status)
reasons.append(possible_source.reason)
bodies.append("")
sources.append(possible_source)
if not newest: # one good source is enough
break
else:
statuses.append(possible_source.status)
reasons.append(possible_source.reason)
bodies.append(possible_source.read())
if possible_source.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node)
elif is_server_error(possible_source.status):
self.error_occurred(
node,
_("ERROR %(status)d %(body)s " "From %(type)s Server")
% {"status": possible_source.status, "body": bodies[-1][:1024], "type": server_type},
)
if sources:
sources.sort(key=source_key)
source = sources.pop()
for src in sources:
self.close_swift_conn(src)
res = Response(request=req, conditional_response=True)
if req.method == "GET" and source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT):
res.app_iter = self._make_app_iter(node, source)
# See NOTE: swift_conn at top of file about this.
res.swift_conn = source.swift_conn
res.status = source.status
update_headers(res, source.getheaders())
if not res.environ:
res.environ = {}
res.environ["swift_x_timestamp"] = source.getheader("x-timestamp")
res.accept_ranges = "bytes"
res.content_length = source.getheader("Content-Length")
if source.getheader("Content-Type"):
res.charset = None
res.content_type = source.getheader("Content-Type")
return res
return self.best_response(req, statuses, reasons, bodies, "%s %s" % (server_type, req.method))
def GETorHEAD_base(self, req, server_type, ring, partition, path):
"""
Base handler for HTTP GET or HEAD requests.
:param req: swob.Request object
:param server_type: server type
:param ring: the ring to obtain nodes from
:param partition: partition
:param path: path for the request
:returns: swob.Response object
"""
statuses = []
reasons = []
bodies = []
source_headers = []
sources = []
newest = config_true_value(req.headers.get('x-newest', 'f'))
headers = self.generate_request_headers(req, additional=req.headers)
for node in self.iter_nodes(ring, partition):
start_node_timing = time.time()
try:
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(
node['ip'], node['port'], node['device'], partition,
req.method, path, headers=headers,
query_string=req.query_string)
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(self.app.node_timeout):
possible_source = conn.getresponse()
# See NOTE: swift_conn at top of file about this.
possible_source.swift_conn = conn
except (Exception, Timeout):
self.exception_occurred(
node, server_type, _('Trying to %(method)s %(path)s') %
{'method': req.method, 'path': req.path})
continue
if self.is_good_source(possible_source):
# 404 if we know we don't have a synced copy
if not float(possible_source.getheader('X-PUT-Timestamp', 1)):
statuses.append(HTTP_NOT_FOUND)
reasons.append('')
bodies.append('')
source_headers.append('')
self.close_swift_conn(possible_source)
else:
statuses.append(possible_source.status)
reasons.append(possible_source.reason)
bodies.append('')
source_headers.append('')
sources.append((possible_source, node))
if not newest: # one good source is enough
break
else:
statuses.append(possible_source.status)
reasons.append(possible_source.reason)
bodies.append(possible_source.read())
source_headers.append(possible_source.getheaders())
if possible_source.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node, _('ERROR Insufficient Storage'))
elif is_server_error(possible_source.status):
self.error_occurred(node, _('ERROR %(status)d %(body)s '
'From %(type)s Server') %
{'status': possible_source.status,
'body': bodies[-1][:1024],
'type': server_type})
res = None
if sources:
sources.sort(key=lambda s: source_key(s[0]))
source, node = sources.pop()
for src, _junk in sources:
self.close_swift_conn(src)
res = Response(request=req)
if req.method == 'GET' and \
source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT):
res.app_iter = self._make_app_iter(node, source)
# See NOTE: swift_conn at top of file about this.
res.swift_conn = source.swift_conn
res.status = source.status
update_headers(res, source.getheaders())
if not res.environ:
res.environ = {}
res.environ['swift_x_timestamp'] = \
source.getheader('x-timestamp')
res.accept_ranges = 'bytes'
res.content_length = source.getheader('Content-Length')
if source.getheader('Content-Type'):
res.charset = None
res.content_type = source.getheader('Content-Type')
if not res:
res = self.best_response(req, statuses, reasons, bodies,
'%s %s' % (server_type, req.method),
headers=source_headers)
try:
(account, container) = split_path(req.path_info, 1, 2)
_set_info_cache(self.app, req.environ, account, container, res)
except ValueError:
pass
try:
(account, container, obj) = split_path(req.path_info, 3, 3, True)
_set_object_info_cache(self.app, req.environ, account,
container, obj, res)
except ValueError:
pass
return res
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
print("-------------------- IN OBJ SERVER GET -------------------")
device, partition, account, container, obj, policy = \
get_name_and_placement(request, 5, 5, True)
keep_cache = self.keep_cache_private or (
'X-Auth-Token' not in request.headers
and 'X-Storage-Token' not in request.headers)
try:
disk_file = self.get_diskfile(device,
partition,
account,
container,
obj,
policy=policy)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
with disk_file.open():
metadata = disk_file.get_metadata()
auth_token = request.headers['X-Auth-Token']
#rebac_metadata = metadata['X-Object-Meta-Rebac']
#acl_metadata = metadata['X-Object-Meta-Oacl']
line1 = re.sub(r"^'", "", metadata['name'])
line2 = re.sub(r"'.*", "", line1)
line3 = re.sub(r"^/", "", line2)
line4 = re.sub(r"/", ":", line3)
authorize_file_info = re.sub(r"^", "cloud1:", line4)
authorize_user_info = "cloud1:" + request.headers[
'X-Project-Id'] + ":" + request.headers['X-User-Id']
print("--- The user info is -> ", authorize_user_info)
print("--- The File info -> ", authorize_file_info)
auth_resp = rebac.authorize(auth_token, authorize_user_info,
authorize_file_info)
if auth_resp == False:
raise AttributeError('User Not Allowed to access File')
else:
print(
"**** User [authorize_user_info] is allowed to access File [authorize_file_info] ****"
)
obj_size = int(metadata['Content-Length'])
file_x_ts = Timestamp(metadata['X-Timestamp'])
keep_cache = (self.keep_cache_private or
('X-Auth-Token' not in request.headers
and 'X-Storage-Token' not in request.headers))
conditional_etag = None
if 'X-Backend-Etag-Is-At' in request.headers:
conditional_etag = metadata.get(
request.headers['X-Backend-Etag-Is-At'])
response = Response(
app_iter=disk_file.reader(keep_cache=keep_cache),
request=request,
conditional_response=True,
conditional_etag=conditional_etag)
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in metadata.iteritems():
if is_sys_or_user_meta('object', key) or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
response.last_modified = math.ceil(float(file_x_ts))
response.content_length = obj_size
try:
response.content_encoding = metadata['Content-Encoding']
except KeyError:
pass
response.headers['X-Timestamp'] = file_x_ts.normal
response.headers['X-Backend-Timestamp'] = file_x_ts.internal
resp = request.get_response(response)
except DiskFileXattrNotSupported:
return HTTPInsufficientStorage(drive=device, request=request)
except (DiskFileNotExist, DiskFileQuarantined) as e:
headers = {}
if hasattr(e, 'timestamp'):
headers['X-Backend-Timestamp'] = e.timestamp.internal
resp = HTTPNotFound(request=request,
headers=headers,
conditional_response=True)
return resp
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
device, partition, account, container, obj = \
split_and_validate_path(request, 5, 5, True)
try:
disk_file = self._diskfile(device,
partition,
account,
container,
obj,
keep_data_fp=True,
iter_hook=sleep)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
if disk_file.is_deleted() or disk_file.is_expired():
if request.headers.get('if-match') == '*':
return HTTPPreconditionFailed(request=request)
else:
return HTTPNotFound(request=request)
try:
file_size = disk_file.get_data_file_size()
except (DiskFileError, DiskFileNotExist):
disk_file.quarantine()
return HTTPNotFound(request=request)
if request.headers.get('if-match') not in (None, '*') and \
disk_file.metadata['ETag'] not in request.if_match:
disk_file.close()
return HTTPPreconditionFailed(request=request)
if request.headers.get('if-none-match') is not None:
if disk_file.metadata['ETag'] in request.if_none_match:
resp = HTTPNotModified(request=request)
resp.etag = disk_file.metadata['ETag']
disk_file.close()
return resp
try:
if_unmodified_since = request.if_unmodified_since
except (OverflowError, ValueError):
# catches timestamps before the epoch
return HTTPPreconditionFailed(request=request)
if if_unmodified_since and \
datetime.fromtimestamp(
float(disk_file.metadata['X-Timestamp']), UTC) > \
if_unmodified_since:
disk_file.close()
return HTTPPreconditionFailed(request=request)
try:
if_modified_since = request.if_modified_since
except (OverflowError, ValueError):
# catches timestamps before the epoch
return HTTPPreconditionFailed(request=request)
if if_modified_since and \
datetime.fromtimestamp(
float(disk_file.metadata['X-Timestamp']), UTC) < \
if_modified_since:
disk_file.close()
return HTTPNotModified(request=request)
response = Response(app_iter=disk_file,
request=request,
conditional_response=True)
response.headers['Content-Type'] = disk_file.metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in disk_file.metadata.iteritems():
if key.lower().startswith('x-object-meta-') or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = disk_file.metadata['ETag']
response.last_modified = float(disk_file.metadata['X-Timestamp'])
response.content_length = file_size
if response.content_length < self.keep_cache_size and \
(self.keep_cache_private or
('X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers)):
disk_file.keep_cache = True
if 'Content-Encoding' in disk_file.metadata:
response.content_encoding = disk_file.metadata['Content-Encoding']
response.headers['X-Timestamp'] = disk_file.metadata['X-Timestamp']
return request.get_response(response)
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
device, partition, account, container, obj = split_and_validate_path(request, 5, 5, True)
try:
disk_file = self._diskfile(device, partition, account, container, obj, keep_data_fp=True, iter_hook=sleep)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
if disk_file.is_deleted() or disk_file.is_expired():
if request.headers.get("if-match") == "*":
return HTTPPreconditionFailed(request=request)
else:
return HTTPNotFound(request=request)
try:
file_size = disk_file.get_data_file_size()
except (DiskFileError, DiskFileNotExist):
disk_file.quarantine()
return HTTPNotFound(request=request)
if request.headers.get("if-match") not in (None, "*") and disk_file.metadata["ETag"] not in request.if_match:
disk_file.close()
return HTTPPreconditionFailed(request=request)
if request.headers.get("if-none-match") is not None:
if disk_file.metadata["ETag"] in request.if_none_match:
resp = HTTPNotModified(request=request)
resp.etag = disk_file.metadata["ETag"]
disk_file.close()
return resp
try:
if_unmodified_since = request.if_unmodified_since
except (OverflowError, ValueError):
# catches timestamps before the epoch
return HTTPPreconditionFailed(request=request)
if (
if_unmodified_since
and datetime.fromtimestamp(float(disk_file.metadata["X-Timestamp"]), UTC) > if_unmodified_since
):
disk_file.close()
return HTTPPreconditionFailed(request=request)
try:
if_modified_since = request.if_modified_since
except (OverflowError, ValueError):
# catches timestamps before the epoch
return HTTPPreconditionFailed(request=request)
if (
if_modified_since
and datetime.fromtimestamp(float(disk_file.metadata["X-Timestamp"]), UTC) < if_modified_since
):
disk_file.close()
return HTTPNotModified(request=request)
response = Response(app_iter=disk_file, request=request, conditional_response=True)
response.headers["Content-Type"] = disk_file.metadata.get("Content-Type", "application/octet-stream")
for key, value in disk_file.metadata.iteritems():
if key.lower().startswith("x-object-meta-") or key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = disk_file.metadata["ETag"]
response.last_modified = float(disk_file.metadata["X-Timestamp"])
response.content_length = file_size
if response.content_length < self.keep_cache_size and (
self.keep_cache_private
or ("X-Auth-Token" not in request.headers and "X-Storage-Token" not in request.headers)
):
disk_file.keep_cache = True
if "Content-Encoding" in disk_file.metadata:
response.content_encoding = disk_file.metadata["Content-Encoding"]
response.headers["X-Timestamp"] = disk_file.metadata["X-Timestamp"]
return request.get_response(response)
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
device, partition, account, container, obj = \
split_and_validate_path(request, 5, 5, True)
try:
disk_file = self._diskfile(device, partition, account, container,
obj, iter_hook=sleep)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
disk_file.open()
if disk_file.is_deleted() or disk_file.is_expired():
if request.headers.get('if-match') == '*':
return HTTPPreconditionFailed(request=request)
else:
return HTTPNotFound(request=request)
try:
file_size = disk_file.get_data_file_size()
except (DiskFileError, DiskFileNotExist):
disk_file.quarantine()
return HTTPNotFound(request=request)
metadata = disk_file.get_metadata()
if request.headers.get('if-match') not in (None, '*') and \
metadata['ETag'] not in request.if_match:
disk_file.close()
return HTTPPreconditionFailed(request=request)
if request.headers.get('if-none-match') is not None:
if metadata['ETag'] in request.if_none_match:
resp = HTTPNotModified(request=request)
resp.etag = metadata['ETag']
disk_file.close()
return resp
try:
if_unmodified_since = request.if_unmodified_since
except (OverflowError, ValueError):
# catches timestamps before the epoch
return HTTPPreconditionFailed(request=request)
if if_unmodified_since and \
datetime.fromtimestamp(
float(metadata['X-Timestamp']), UTC) > \
if_unmodified_since:
disk_file.close()
return HTTPPreconditionFailed(request=request)
try:
if_modified_since = request.if_modified_since
except (OverflowError, ValueError):
# catches timestamps before the epoch
return HTTPPreconditionFailed(request=request)
if if_modified_since and \
datetime.fromtimestamp(
float(metadata['X-Timestamp']), UTC) < \
if_modified_since:
disk_file.close()
return HTTPNotModified(request=request)
response = Response(app_iter=disk_file,
request=request, conditional_response=True)
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in metadata.iteritems():
if key.lower().startswith('x-object-meta-') or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
response.last_modified = float(metadata['X-Timestamp'])
response.content_length = file_size
if response.content_length < self.keep_cache_size and \
(self.keep_cache_private or
('X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers)):
disk_file.keep_cache = True
if 'Content-Encoding' in metadata:
response.content_encoding = metadata['Content-Encoding']
response.headers['X-Timestamp'] = metadata['X-Timestamp']
return request.get_response(response)
def GET(self, request):
__CBAC__ = True
"""Handle HTTP GET requests for the Swift Object Server."""
my_debug('testing', '@GET()')
my_debug('#request headers', request.headers)
#print my_debug
device, partition, account, container, obj = \
split_and_validate_path(request, 5, 5, True)
keep_cache = self.keep_cache_private or (
'X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers)
try:
disk_file = self.get_diskfile(
device, partition, account, container, obj)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
my_debug("locals()",locals())
my_debug("original Diskfile", disk_file)
try:
with disk_file.open():
metadata = disk_file.get_metadata()
my_debug("metadata", metadata)
obj_size = int(metadata['Content-Length'])
file_x_ts = metadata['X-Timestamp']
#json_policy = metadata['JSON-Policy']
file_x_ts_flt = float(file_x_ts)
keep_cache = (self.keep_cache_private or
('X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers))
''' if policy file is present & user_label is present apply JSONAC
Additional thing to have:
1. Check if policy from metadata is valid json
2. Check if user_clearance is present is headers. Read keystone info.
3. Pass JSONPath as headers
4.
'''
if __CBAC__:
my_debug("----CBAC Starts here-----", "***********")
disk_file_iter = disk_file.reader(keep_cache=keep_cache)
original_data = ""
for chunk in iter(disk_file_iter):
original_data += chunk
userRoles = request.headers['X-Roles']
json_policy = metadata['X-Object-Meta-Jsonpolicy'] \
if metadata.has_key('X-Object-Meta-Jsonpolicy') else None
user_labels = metadata['X-Object-Meta-Userlabels'] \
if metadata.has_key('X-Object-Meta-Userlabels') else None
object_labels = metadata['X-Object-Meta-Objectlabels'] \
if metadata.has_key('X-Object-Meta-Objectlabels') else None
object_labelling = metadata['X-Object-Meta-Jsonlabelling'] \
if metadata.has_key('X-Object-Meta-Jsonlabelling') else None
if json_policy and user_labels and object_labels :
cbac_policy = {}
cbac_policy['user_labels'] = json.loads(user_labels)
cbac_policy['object_labels'] = json.loads(object_labels)
cbac_policy['policy'] = json.loads(json_policy)
#user_clearance = ['manager']
user_clearance = userRoles
jsonpath = "/"
filtered_content = ""
my_debug("json_policy is", json_policy)
my_debug("original data is ", original_data)
#try:
if json_policy :
if json_policy and user_clearance and jsonpath and object_labelling:
'''filtered_content = ContentFilter(content_str=original_data,
labeling_policy_str=object_labelling,
user_clearance=user_clearance, query=jsonpath,
cbac_policy=cbac_policy).apply()'''
filtered_content = "testing"
my_debug("#filtered content is ", filtered_content)
#else:
#my_debug("#content_filter not working", True)
#except Exception as e:
#else:
# my_debug("Exception with content filtering {}".format("e"), True)
'''end of content -filter '''
tmp_file = self.tmp_disk_file( request=request, device=device, partition=partition,
container=container, obj="tmp", data=filtered_content, account=account)
tmp_file = self.get_diskfile(
device, partition, account, container, "tmp")
my_debug("tmp_file is", tmp_file)
try:
with tmp_file.open():
tmp_metadata = tmp_file.get_metadata()
my_debug("with tmp_file.open()", tmp_file.reader(keep_cache=keep_cache))
response = Response(
app_iter=tmp_file.reader(keep_cache=keep_cache),
request=request, conditional_response=True)
response.headers['Content-Type'] = tmp_metadata.get(
'Content-Type', 'application/octet-stream')
response.content_length = len(filtered_content)
for key, value in tmp_metadata.iteritems():
if is_user_meta('object', key) or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = tmp_metadata['ETag']
#response.last_modified = math.ceil(file_x_ts_flt)
#response.content_length = obj_size
my_debug("get_response", "test")
resp = request.get_response(response)
my_debug( "response", resp )
my_debug("rsponse.__dict__", resp.__dict__)
except (DiskFileNotExist, DiskFileQuarantined):
my_debug("tmp file is not found", "test")
resp = HTTPNotFound(request=request, conditional_response=True)
my_debug("final resp object", resp)
return resp
disk_file.open()
response = Response(
app_iter=disk_file.reader(keep_cache=keep_cache),
# instead of app_iter which reads from the file, content is given in body
#body="Yes, you are smart!\n",
request=request, conditional_response=True)
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in metadata.iteritems():
if is_user_meta('object', key) or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
response.last_modified = math.ceil(file_x_ts_flt)
response.content_length = obj_size
#response.content_length = response.headers['Content-Length']
try:
response.content_encoding = metadata[
'Content-Encoding']
except KeyError:
pass
response.headers['X-Timestamp'] = file_x_ts
resp = request.get_response(response)
except (DiskFileNotExist, DiskFileQuarantined):
resp = HTTPNotFound(request=request, conditional_response=True)
my_debug("resp object without cbac", resp.__dict__)
return resp
def GETorHEAD_base(self, req, server_type, ring, partition, path):
"""
Base handler for HTTP GET or HEAD requests.
:param req: swob.Request object
:param server_type: server type
:param ring: the ring to obtain nodes from
:param partition: partition
:param path: path for the request
:returns: swob.Response object
"""
statuses = []
reasons = []
bodies = []
sources = []
newest = config_true_value(req.headers.get('x-newest', 'f'))
for node in self.iter_nodes(ring, partition):
start_node_timing = time.time()
try:
with ConnectionTimeout(self.app.conn_timeout):
headers = dict(req.headers)
headers['Connection'] = 'close'
conn = http_connect(node['ip'],
node['port'],
node['device'],
partition,
req.method,
path,
headers=headers,
query_string=req.query_string)
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(self.app.node_timeout):
possible_source = conn.getresponse()
# See NOTE: swift_conn at top of file about this.
possible_source.swift_conn = conn
except (Exception, Timeout):
self.exception_occurred(
node, server_type,
_('Trying to %(method)s %(path)s') % {
'method': req.method,
'path': req.path
})
continue
if self.is_good_source(possible_source):
# 404 if we know we don't have a synced copy
if not float(possible_source.getheader('X-PUT-Timestamp', 1)):
statuses.append(HTTP_NOT_FOUND)
reasons.append('')
bodies.append('')
self.close_swift_conn(possible_source)
else:
statuses.append(possible_source.status)
reasons.append(possible_source.reason)
bodies.append('')
sources.append((possible_source, node))
if not newest: # one good source is enough
break
else:
statuses.append(possible_source.status)
reasons.append(possible_source.reason)
bodies.append(possible_source.read())
if possible_source.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node, _('ERROR Insufficient Storage'))
elif is_server_error(possible_source.status):
self.error_occurred(
node,
_('ERROR %(status)d %(body)s '
'From %(type)s Server') % {
'status': possible_source.status,
'body': bodies[-1][:1024],
'type': server_type
})
if sources:
sources.sort(key=lambda s: source_key(s[0]))
source, node = sources.pop()
for src, _junk in sources:
self.close_swift_conn(src)
res = Response(request=req, conditional_response=True)
if req.method == 'GET' and \
source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT):
res.app_iter = self._make_app_iter(node, source)
# See NOTE: swift_conn at top of file about this.
res.swift_conn = source.swift_conn
res.status = source.status
update_headers(res, source.getheaders())
if not res.environ:
res.environ = {}
res.environ['swift_x_timestamp'] = \
source.getheader('x-timestamp')
res.accept_ranges = 'bytes'
res.content_length = source.getheader('Content-Length')
if source.getheader('Content-Type'):
res.charset = None
res.content_type = source.getheader('Content-Type')
return res
return self.best_response(req, statuses, reasons, bodies,
'%s %s' % (server_type, req.method))
def GETorHEAD_base(self, req, server_type, partition, nodes, path,
attempts):
"""
Base handler for HTTP GET or HEAD requests.
:param req: swob.Request object
:param server_type: server type
:param partition: partition
:param nodes: nodes
:param path: path for the request
:param attempts: number of attempts to try
:returns: swob.Response object
"""
statuses = []
reasons = []
bodies = []
sources = []
newest = config_true_value(req.headers.get('x-newest', 'f'))
nodes = iter(nodes)
while len(statuses) < attempts:
try:
node = nodes.next()
except StopIteration:
break
if self.error_limited(node):
continue
start_node_timing = time.time()
try:
with ConnectionTimeout(self.app.conn_timeout):
headers = dict(req.headers)
headers['Connection'] = 'close'
conn = http_connect(
node['ip'], node['port'], node['device'], partition,
req.method, path, headers=headers,
query_string=req.query_string)
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(self.app.node_timeout):
possible_source = conn.getresponse()
# See NOTE: swift_conn at top of file about this.
possible_source.swift_conn = conn
except (Exception, Timeout):
self.exception_occurred(
node, server_type, _('Trying to %(method)s %(path)s') %
{'method': req.method, 'path': req.path})
continue
if self.is_good_source(possible_source):
# 404 if we know we don't have a synced copy
if not float(possible_source.getheader('X-PUT-Timestamp', 1)):
statuses.append(HTTP_NOT_FOUND)
reasons.append('')
bodies.append('')
self.close_swift_conn(possible_source)
else:
statuses.append(possible_source.status)
reasons.append(possible_source.reason)
bodies.append('')
sources.append(possible_source)
if not newest: # one good source is enough
break
else:
statuses.append(possible_source.status)
reasons.append(possible_source.reason)
bodies.append(possible_source.read())
if possible_source.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node)
elif is_server_error(possible_source.status):
self.error_occurred(node, _('ERROR %(status)d %(body)s '
'From %(type)s Server') %
{'status': possible_source.status,
'body': bodies[-1][:1024],
'type': server_type})
if sources:
sources.sort(key=source_key)
source = sources.pop()
for src in sources:
self.close_swift_conn(src)
res = Response(request=req, conditional_response=True)
if req.method == 'GET' and \
source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT):
res.app_iter = self._make_app_iter(node, source)
# See NOTE: swift_conn at top of file about this.
res.swift_conn = source.swift_conn
res.status = source.status
update_headers(res, source.getheaders())
if not res.environ:
res.environ = {}
res.environ['swift_x_timestamp'] = \
source.getheader('x-timestamp')
res.accept_ranges = 'bytes'
res.content_length = source.getheader('Content-Length')
if source.getheader('Content-Type'):
res.charset = None
res.content_type = source.getheader('Content-Type')
return res
return self.best_response(req, statuses, reasons, bodies,
'%s %s' % (server_type, req.method))
float(file.metadata['X-Timestamp']), UTC) < \
if_modified_since:
file.close()
return HTTPNotModified(request=request)
response = Response(app_iter=file,
request=request,
conditional_response=True)
response.headers['Content-Type'] = file.metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in file.metadata.iteritems():
if key.lower().startswith('x-object-meta-') or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = file.metadata['ETag']
response.last_modified = float(file.metadata['X-Timestamp'])
response.content_length = file_size
if response.content_length < self.keep_cache_size and \
(self.keep_cache_private or
('X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers)):
file.keep_cache = True
if 'Content-Encoding' in file.metadata:
response.content_encoding = file.metadata['Content-Encoding']
response.headers['X-Timestamp'] = file.metadata['X-Timestamp']
return request.get_response(response)
@public
@timing_stats(sample_rate=0.8)
def HEAD(self, request):
"""Handle HTTP HEAD requests for the Swift Object Server."""
try:
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
device, partition, account, container, obj = split_and_validate_path(request, 5, 5, True)
keep_cache = self.keep_cache_private or (
"X-Auth-Token" not in request.headers and "X-Storage-Token" not in request.headers
)
try:
disk_file = self.get_diskfile(device, partition, account, container, obj)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
with disk_file.open():
metadata = disk_file.get_metadata()
obj_size = int(metadata["Content-Length"])
if request.headers.get("if-match") not in (None, "*") and metadata["ETag"] not in request.if_match:
return HTTPPreconditionFailed(request=request)
if request.headers.get("if-none-match") is not None:
if metadata["ETag"] in request.if_none_match:
resp = HTTPNotModified(request=request)
resp.etag = metadata["ETag"]
return resp
file_x_ts = metadata["X-Timestamp"]
file_x_ts_flt = float(file_x_ts)
try:
if_unmodified_since = request.if_unmodified_since
except (OverflowError, ValueError):
# catches timestamps before the epoch
return HTTPPreconditionFailed(request=request)
file_x_ts_utc = datetime.fromtimestamp(file_x_ts_flt, UTC)
if if_unmodified_since and file_x_ts_utc > if_unmodified_since:
return HTTPPreconditionFailed(request=request)
try:
if_modified_since = request.if_modified_since
except (OverflowError, ValueError):
# catches timestamps before the epoch
return HTTPPreconditionFailed(request=request)
if if_modified_since and file_x_ts_utc <= if_modified_since:
return HTTPNotModified(request=request)
keep_cache = self.keep_cache_private or (
"X-Auth-Token" not in request.headers and "X-Storage-Token" not in request.headers
)
response = Response(
app_iter=disk_file.reader(keep_cache=keep_cache), request=request, conditional_response=True
)
response.headers["Content-Type"] = metadata.get("Content-Type", "application/octet-stream")
for key, value in metadata.iteritems():
if is_user_meta("object", key) or key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata["ETag"]
response.last_modified = math.ceil(file_x_ts_flt)
response.content_length = obj_size
try:
response.content_encoding = metadata["Content-Encoding"]
except KeyError:
pass
response.headers["X-Timestamp"] = file_x_ts
resp = request.get_response(response)
except DiskFileNotExist:
if request.headers.get("if-match") == "*":
resp = HTTPPreconditionFailed(request=request)
else:
resp = HTTPNotFound(request=request)
except DiskFileQuarantined:
resp = HTTPNotFound(request=request)
return resp
etag = md5(
''.join(o['hash'] for o in listing)).hexdigest()
except KeyError:
return HTTPServerError('Invalid Manifest File',
request=req)
else:
content_length = 0
last_modified = resp.last_modified
etag = md5().hexdigest()
resp = Response(headers=resp.headers, request=req,
conditional_response=True)
resp.app_iter = SegmentedIterable(
self, lcontainer, listing, resp,
is_slo=(large_object == 'SLO'))
resp.content_length = content_length
resp.last_modified = last_modified
resp.etag = etag
resp.headers['accept-ranges'] = 'bytes'
# In case of a manifest file of nonzero length, the
# backend may have sent back a Content-Range header for
# the manifest. It's wrong for the client, though.
resp.content_range = None
return resp
@public
@cors_validation
@delay_denial
def GET(self, req):
"""Handler for HTTP GET requests."""
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
#with open("/home/ubuntu/spawn-read.txt", "a") as tran_file:
# tran_file.write("At Read Start Datetime = "+str(datetime.now())+"\n")
a=time.time()
device, partition, account, container, obj, policy_idx = \
get_name_and_placement(request, 5, 5, True)
keep_cache = self.keep_cache_private or (
'X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers)
try:
disk_file = self.get_diskfile(
device, partition, account, container, obj,
policy_idx=policy_idx)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
with disk_file.open():
metadata = disk_file.get_metadata()
obj_size = int(metadata['Content-Length'])
file_x_ts = Timestamp(metadata['X-Timestamp'])
keep_cache = (self.keep_cache_private or
('X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers))
response = Response(
app_iter=disk_file.reader(keep_cache=keep_cache),
request=request, conditional_response=True)
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in metadata.iteritems():
if is_sys_or_user_meta('object', key) or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
response.last_modified = math.ceil(float(file_x_ts))
response.content_length = obj_size
try:
response.content_encoding = metadata[
'Content-Encoding']
except KeyError:
pass
response.headers['X-Timestamp'] = file_x_ts.normal
response.headers['X-Backend-Timestamp'] = file_x_ts.internal
resp = request.get_response(response)
except DiskFileXattrNotSupported:
return HTTPInsufficientStorage(drive=device, request=request)
except (DiskFileNotExist, DiskFileQuarantined) as e:
headers = {}
if hasattr(e, 'timestamp'):
headers['X-Backend-Timestamp'] = e.timestamp.internal
resp = HTTPNotFound(request=request, headers=headers,
conditional_response=True)
#with open("/home/ubuntu/spawn-read.txt", "a") as tran_file:
# tran_file.write("At Read Delay start Datetime = "+str(datetime.now())+"\n")
#a=2
#num=22227727
#for a in range(a, num):
# if a % num == 0:
# print('not prime')
# break
# else: # loop not exited via break
# print('prime')
#time.sleep(2)
#with open("/home/ubuntu/spawn-read.txt", "a") as tran_file:
# tran_file.write("At Read End Datetime = "+str(datetime.now())+"\n")
b=time.time()-a
#with open("/home/ubuntu/RSTORAGE.txt", "a") as tran_file:
# tran_file.write("Total GET duration of obj = "+str(request.path)+"="+str(b)+"="+str(datetime.now())+"\n")
return resp
float(file.metadata['X-Timestamp']), UTC) < \
if_modified_since:
file.close()
self.logger.timing_since('GET.timing', start_time)
return HTTPNotModified(request=request)
response = Response(app_iter=file,
request=request, conditional_response=True)
response.headers['Content-Type'] = file.metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in file.metadata.iteritems():
if key.lower().startswith('x-object-meta-') or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = file.metadata['ETag']
response.last_modified = float(file.metadata['X-Timestamp'])
response.content_length = file_size
if response.content_length < self.keep_cache_size and \
(self.keep_cache_private or
('X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers)):
file.keep_cache = True
if 'Content-Encoding' in file.metadata:
response.content_encoding = file.metadata['Content-Encoding']
response.headers['X-Timestamp'] = file.metadata['X-Timestamp']
self.logger.timing_since('GET.timing', start_time)
return request.get_response(response)
@public
def HEAD(self, request):
"""Handle HTTP HEAD requests for the Swift Object Server."""
start_time = time.time()
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
print("-------------------- IN OBJ SERVER GET -------------------")
device, partition, account, container, obj, policy = \
get_name_and_placement(request, 5, 5, True)
keep_cache = self.keep_cache_private or (
'X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers)
try:
disk_file = self.get_diskfile(
device, partition, account, container, obj,
policy=policy)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
with disk_file.open():
metadata = disk_file.get_metadata()
auth_token = request.headers['X-Auth-Token']
#rebac_metadata = metadata['X-Object-Meta-Rebac']
#acl_metadata = metadata['X-Object-Meta-Oacl']
line1=re.sub(r"^'", "", metadata['name'])
line2=re.sub(r"'.*", "", line1)
line3=re.sub(r"^/", "", line2)
line4=re.sub(r"/", ":", line3)
authorize_file_info = re.sub(r"^", "cloud1:", line4)
authorize_user_info = "cloud1:"+request.headers['X-Project-Id']+":"+request.headers['X-User-Id']
print("--- The user info is -> ", authorize_user_info)
print("--- The File info -> ", authorize_file_info)
auth_resp = rebac.authorize(auth_token, authorize_user_info, authorize_file_info)
if auth_resp == False:
raise AttributeError('User Not Allowed to access File')
else:
print("**** User [authorize_user_info] is allowed to access File [authorize_file_info] ****")
obj_size = int(metadata['Content-Length'])
file_x_ts = Timestamp(metadata['X-Timestamp'])
keep_cache = (self.keep_cache_private or
('X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers))
conditional_etag = None
if 'X-Backend-Etag-Is-At' in request.headers:
conditional_etag = metadata.get(
request.headers['X-Backend-Etag-Is-At'])
response = Response(
app_iter=disk_file.reader(keep_cache=keep_cache),
request=request, conditional_response=True,
conditional_etag=conditional_etag)
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in metadata.iteritems():
if is_sys_or_user_meta('object', key) or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
response.last_modified = math.ceil(float(file_x_ts))
response.content_length = obj_size
try:
response.content_encoding = metadata[
'Content-Encoding']
except KeyError:
pass
response.headers['X-Timestamp'] = file_x_ts.normal
response.headers['X-Backend-Timestamp'] = file_x_ts.internal
resp = request.get_response(response)
except DiskFileXattrNotSupported:
return HTTPInsufficientStorage(drive=device, request=request)
except (DiskFileNotExist, DiskFileQuarantined) as e:
headers = {}
if hasattr(e, 'timestamp'):
headers['X-Backend-Timestamp'] = e.timestamp.internal
resp = HTTPNotFound(request=request, headers=headers,
conditional_response=True)
return resp
def GET(self, request):
"""Handle HTTP GET requests for the Swift on File object server"""
device, partition, account, container, obj, policy = \
get_name_and_placement(request, 5, 5, True)
keep_cache = self.keep_cache_private or (
'X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers
)
# Get Diskfile
try:
disk_file = self.get_diskfile(device, partition, account, container,
obj, policy)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
# Get metadata and append it to response
try:
with disk_file.open():
metadata = disk_file.get_metadata()
obj_size = int(metadata['Content-Length'])
file_x_ts = Timestamp(metadata['X-Timestamp'])
try:
# (HPSS) Our file could end up being on an offline
# tape, so we need to check for it and return an
# HTTP 'accepted, but still processing' response.
if disk_file.is_offline():
return HTTPAccepted(request=request)
except (SwiftOnFileSystemIOError, SwiftOnFileFsException):
return HTTPServiceUnavailable(request=request)
response = Response(
app_iter=disk_file.reader(keep_cache=keep_cache),
request=request, conditional_response=True
)
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream'
)
for key, value in metadata.iteritems():
if is_sys_or_user_meta('object', key) or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
response.last_modified = math.ceil(float(file_x_ts))
response.content_length = obj_size
try:
response.content_encoding = metadata['Content-Encoding']
except KeyError:
pass
response.headers['X-Timestamp'] = file_x_ts.normal
response.headers['X-Backend-Timestamp'] = file_x_ts.internal
# (HPSS) Inject HPSS xattr metadata into headers
want_hpss_metadata = request.headers.get('X-HPSS-Get-Metadata',
False)
if config_true_value(want_hpss_metadata):
try:
hpss_headers = disk_file.read_hpss_system_metadata()
response.headers.update(hpss_headers)
except SwiftOnFileSystemIOError:
return HTTPServiceUnavailable(request=request)
return request.get_response(response)
except (DiskFileNotExist, DiskFileQuarantined) as e:
headers = {}
if hasattr(e, 'timestamp'):
headers['X-Backend-Timestamp'] = e.timestamp.internal
return HTTPNotFound(request=request, headers=headers,
conditional_response=True)
