def __call__(self, env, start_response):
request = Request(env)
try:
(version, account, container, objname) = split_path(request.path_info, 1, 4, True)
except ValueError:
response = request.get_response(self.app)
return response(env, start_response)
if not objname:
response = request.get_response(self.app)
if container:
if not request.params.has_key('compress'):
response.body = response.body.replace(self.compress_suffix, '')
return response(env, start_response)
original_path_info = request.path_info
request.path_info += self.compress_suffix
if request.method == 'GET':
if not request.params.has_key('compress'):
# we need to decompress
response = request.get_response(self.app)
if response.status_int == 404:
# it may not be compressed, if admin added the compress filter after
# some files have been uploaded
request.path_info = original_path_info
response = request.get_response(self.app)
return response(env, start_response)
uncompressed_data = create_uncompress(response.body)
response.body = uncompressed_data
return response(env, start_response)
if request.method == 'PUT':
if hasattr(request, 'body_file'):
data = ""
while True:
chunk = request.body_file.read()
if not chunk:
break
data += chunk
request.body = data
compress_data = create_compress(data)
else:
compress_data = create_compress(request.body)
if compress_data:
request.body = compress_data
response = request.get_response(self.app)
return response(env, start_response)
def __call__(self, env, start_response):
request = Request(env)
resp = request.get_response(self.app)
try:
(version, account, container,
objname) = split_path(request.path_info, 4, 4, True)
except ValueError:
return self.app
if 'HTTP_X_NO_PREFETCH' not in request.environ:
if request.method == 'GET':
(site, objid, size, ext) = objname.split('_')
if site not in self.chain:
self.chain[site] = Chain(self.logger,
self.conf['chainsave'],
self.conf['totalseconds'],
self.conf['probthreshold'],
self.conf['twolevels'])
oid = (hashlib.md5(request.path_info).hexdigest())
self.add_object_to_chain(site, oid, container, objname)
if PREFETCH:
data = self.get_prefetched(site, oid, objname)
self.prefetch_objects(site, oid, account, request)
if data:
resp.headers['X-object-prefetched'] = 'True'
resp.body = data
return resp(env, start_response)
def transition(self, env):
# GET Object body
req = Request(copy(env))
req.method = 'GET'
resp = req.get_response(self.app)
obj_body = resp.body
# Glacier로 업로드
tmpfile = self.save_to_tempfile(obj_body)
try:
glacier = self._init_glacier()
archive_id = glacier.upload_archive(tmpfile)
glacier_obj = make_glacier_hidden_object_name(self.obj, archive_id)
except Exception as e:
return Response(status=HTTP_INTERNAL_SERVER_ERROR, body=e.message)
finally:
self.delete_tempfile(tmpfile)
# Object를 0KB로 만들기
req = Request(copy(env))
req.headers[GLACIER_FLAG_META] = True
resp = req.get_response(self.app)
# Glacier Hidden account에 기록
glacier_account = self.glacier_account_prefix + self.account
part, nodes = self.container_ring.get_nodes(glacier_account,
self.container)
hidden_path = '/%s/%s/%s' % (glacier_account, self.container,
glacier_obj)
for node in nodes:
ip = node['ip']
port = node['port']
dev = node['device']
headers = dict()
headers['user-agent'] = 'transition-middleware'
headers['X-Timestamp'] = normalize_timestamp(time.time())
headers['referer'] = req.as_referer()
headers['x-size'] = '0'
headers['x-content-type'] = 'text/plain'
headers['x-etag'] = 'd41d8cd98f00b204e9800998ecf8427e'
conn = http_connect(ip, port, dev, part, 'PUT', hidden_path,
headers)
conn.getresponse().read()
return Response(status=HTTP_NO_CONTENT)
def __call__(self, env, start_response):
req = Request(env)
resp = req.get_response(self.app)
self.logger.info("Serverless: available headers: {}".format(
str(dict(req.headers))))
try:
if "X-Function-URL" in req.headers:
version, account, container, obj = split_path(
req.path_info, 4, 4, True)
self.logger.info(
"Serverless: version {}, account {}, container {}, object {}"
.format(version, account, container, obj))
if obj and is_success(resp.status_int) and req.method == 'PUT':
webhook = req.headers.get("X-Function-URL")
data = json.dumps({
"x-auth-token":
req.headers.get("X-Auth-Token"),
"version":
version,
"account":
account,
"container":
container,
"object":
obj,
"project_id":
req.headers.get("X-Project-Id"),
})
self.logger.info(
"Serverless: data to send to a function {}".format(
str(data)))
data_as_bytes = data.encode('utf-8')
webhook_req = urllib2.Request(webhook, data=data_as_bytes)
webhook_req.add_header('Content-Type', 'application/json')
webhook_req.add_header('Content-Length',
len(data_as_bytes))
self.logger.info(
"Serverless: data to send as bytes {}".format(
data_as_bytes))
with Timeout(60):
try:
result = urllib2.urlopen(webhook_req).read()
self.logger.info(
"Serverless: function worked fine. Result {}".
format(str(result)))
except (Exception, Timeout) as ex:
self.logger.error(
'Serverless: failed POST to webhook {}, '
'error {}'.format(webhook, str(ex)))
else:
self.logger.info("Serverless: skipping functions middleware "
"due to absence of function URL")
except ValueError:
# not an object request
pass
return self.app(env, start_response)
def __call__(self, env, start_response):
req = Request(env)
resp = req.get_response(self.app)
try:
(version, account, container,
objname) = split_path(req.path_info, 1, 4, True)
except ValueError:
return resp(env, start_response)
is_grok_request = req.params.has_key(
'grok') or 'grok-pattern' in req.headers
# grok request has to be explicit, and only expected for GET operations
if not req.method == 'GET' or not is_grok_request:
return resp(env, start_response)
self.logger.debug('Calling grok middleware')
# make sure we have an object to work on
if not objname or not resp.status_int == 200:
return resp(env, start_response)
# the grok pattern is expected to be in the request headers
# if the pattern is missing, we ignore the grok request
pattern = req.headers.get('grok-pattern')
if not pattern:
self.logger.debug(
'Object found, but no pattern requested, aborting')
return self.get_err_response('Grok pattern is missing')(
env, start_response)
self.logger.debug('Starting grok operation')
# we are going to assume the retrieved object is string object
# and iterate through lines of resp.body and execute grok_match
grokked_content = ''
try:
strbuf = StringIO.StringIO(resp.body)
for line in strbuf:
parsed_line = pygrok.grok_match(line, pattern)
grokked_content += json.dumps(parsed_line) + '\n'
except Exception as e:
return self.get_err_response(str(e))(env, start_response)
resp.body = grokked_content
return resp(env, start_response)
def __call__(self, env, start_response):
req = Request(env)
resp = req.get_response(self.app)
try:
(version, account, container, objname) = split_path(req.path_info, 1, 4, True)
except ValueError:
return resp(env, start_response)
is_grok_request = req.params.has_key("grok") or "grok-pattern" in req.headers
# grok request has to be explicit, and only expected for GET operations
if not req.method == "GET" or not is_grok_request:
return resp(env, start_response)
self.logger.debug("Calling grok middleware")
# make sure we have an object to work on
if not objname or not resp.status_int == 200:
return resp(env, start_response)
# the grok pattern is expected to be in the request headers
# if the pattern is missing, we ignore the grok request
pattern = req.headers.get("grok-pattern")
if not pattern:
self.logger.debug("Object found, but no pattern requested, aborting")
return self.get_err_response("Grok pattern is missing")(env, start_response)
self.logger.debug("Starting grok operation")
# we are going to assume the retrieved object is string object
# and iterate through lines of resp.body and execute grok_match
grokked_content = ""
try:
strbuf = StringIO.StringIO(resp.body)
for line in strbuf:
parsed_line = pygrok.grok_match(line, pattern)
grokked_content += json.dumps(parsed_line) + "\n"
except Exception as e:
return self.get_err_response(str(e))(env, start_response)
resp.body = grokked_content
return resp(env, start_response)
def _initialize(self):
resp = None
if self.swift_client:
resp = self.swift_client.make_request('HEAD', self.path, {},
(2, 4))
elif self.env:
req = Request(self.env)
req.method = 'HEAD'
req.path_info = self.path
req.headers['Content-Length'] = '0'
resp = req.get_response(self.app)
if resp is None:
return
self.status = resp.status_int
if is_success(self.status):
self.headers = resp.headers
def __call__(self, env, start_response):
req = Request(env)
username = env.get('HTTP_X_USER_NAME',None)
userid = env.get('HTTP_X_USER_ID',None)
tenant = env.get('HTTP_X_PROJECT_NAME',None)
version, account, container, obj = req.split_path(1,4,True)
#COMMENT: Control the author of the request.
if req.method == "PUT" and req.headers.get('x-container-read',None) is not None and container is not None and obj is None:
#Associate owner to container
req.headers['x-container-sysmeta-owner'] = userid
req.headers['x-container-meta-owner'] = userid
if req.method =="POST" and req.headers.get('x-container-read',None) is not None:
new_req = Request.blank(req.path_info,None,req.headers,None)
new_req.method = "HEAD"
new_req.path_info = "/".join(["",version,account,container])
new_resp = new_req.get_response(self.app)
if new_resp.headers.get('x-container-meta-bel-id',None) is None:
#Container public -> private. Associate owner
req.headers['x-container-sysmeta-owner'] = userid
req.headers['x-container-meta-owner'] = userid
elif new_resp.headers.get('x-container-sysmeta-owner',None) != userid:
#Container already private and user is not the owner
return HTTPUnauthorized(body="Unauthorized")(env, start_response)
if req.method == "GET" and username != "ceilometer" and username != None:
if obj != None:
#Request a container
new_req = Request.blank(req.path_info,None,req.headers,None)
new_req.method = "HEAD"
new_req.path_info = "/".join(["",version,account,container])
response = new_req.get_response(self.app)
cont_header = response.headers
container_sel_id = cont_header.get('x-container-meta-sel-id',None)
cont_secret_ref = cont_header.get('x-container-meta-container-ref',None)
env['swift_crypto_fetch_cont_id'] = container_sel_id
resp_obj = req.get_response(self.app)
object_sel_id = resp_obj.headers.get('x-object-meta-sel-id',None)
if object_sel_id != container_sel_id:# and onResource=="False":
#The object has been uploaded before the last policy change
if object_sel_id is not None:
old_dek = get_secret(self.userID,cont_secret_ref,object_sel_id,tenant).get('KEK',None)
if old_dek is not None:
env['swift_crypto_old_fetch_key'] = old_dek
else:
env['swift_crypto_old_fetch_key'] = "NotAuthorized"
if container_sel_id is not None:
dek = get_secret(self.userID,cont_secret_ref,container_sel_id,tenant).get('KEK',None)
if dek is not None:
env['swift_crypto_fetch_key'] = dek
else:
env['swift_crypto_fetch_key'] = "NotAuthorized"
"""elif obj == None and container == None:
resp_account = req.get_response(self.app)
list_containers = resp_account.body
list_containers = json.loads(list_containers)
for cont in list_containers:
new_req = Request.blank(req.path_info,None,None,None)
new_req.method = "GET"
cont_name = cont.get('name','')
new_req.path_info = "/".join(["",version,account,cont_name])
response = new_req.get_response(self.app)
print response.headers
print cont
print "-----------------------"
list_acl = self.extractACL(response.headers)
if list_acl != [] and userid not in list_acl:
print "list_acl\n"
print list_acl
list_containers.remove(cont)
resp_account.body = json.dumps(list_containers)"""
return self.app(env, start_response)
