def __call__(self, env, start_response): request = Request(env) try: (version, account, container, objname) = split_path(request.path_info, 1, 4, True) except ValueError: response = request.get_response(self.app) return response(env, start_response) if not objname: response = request.get_response(self.app) if container: if not request.params.has_key('compress'): response.body = response.body.replace(self.compress_suffix, '') return response(env, start_response) original_path_info = request.path_info request.path_info += self.compress_suffix if request.method == 'GET': if not request.params.has_key('compress'): # we need to decompress response = request.get_response(self.app) if response.status_int == 404: # it may not be compressed, if admin added the compress filter after # some files have been uploaded request.path_info = original_path_info response = request.get_response(self.app) return response(env, start_response) uncompressed_data = create_uncompress(response.body) response.body = uncompressed_data return response(env, start_response) if request.method == 'PUT': if hasattr(request, 'body_file'): data = "" while True: chunk = request.body_file.read() if not chunk: break data += chunk request.body = data compress_data = create_compress(data) else: compress_data = create_compress(request.body) if compress_data: request.body = compress_data response = request.get_response(self.app) return response(env, start_response)
def __call__(self, env, start_response): request = Request(env) resp = request.get_response(self.app) try: (version, account, container, objname) = split_path(request.path_info, 4, 4, True) except ValueError: return self.app if 'HTTP_X_NO_PREFETCH' not in request.environ: if request.method == 'GET': (site, objid, size, ext) = objname.split('_') if site not in self.chain: self.chain[site] = Chain(self.logger, self.conf['chainsave'], self.conf['totalseconds'], self.conf['probthreshold'], self.conf['twolevels']) oid = (hashlib.md5(request.path_info).hexdigest()) self.add_object_to_chain(site, oid, container, objname) if PREFETCH: data = self.get_prefetched(site, oid, objname) self.prefetch_objects(site, oid, account, request) if data: resp.headers['X-object-prefetched'] = 'True' resp.body = data return resp(env, start_response)
def transition(self, env): # GET Object body req = Request(copy(env)) req.method = 'GET' resp = req.get_response(self.app) obj_body = resp.body # Glacier로 업로드 tmpfile = self.save_to_tempfile(obj_body) try: glacier = self._init_glacier() archive_id = glacier.upload_archive(tmpfile) glacier_obj = make_glacier_hidden_object_name(self.obj, archive_id) except Exception as e: return Response(status=HTTP_INTERNAL_SERVER_ERROR, body=e.message) finally: self.delete_tempfile(tmpfile) # Object를 0KB로 만들기 req = Request(copy(env)) req.headers[GLACIER_FLAG_META] = True resp = req.get_response(self.app) # Glacier Hidden account에 기록 glacier_account = self.glacier_account_prefix + self.account part, nodes = self.container_ring.get_nodes(glacier_account, self.container) hidden_path = '/%s/%s/%s' % (glacier_account, self.container, glacier_obj) for node in nodes: ip = node['ip'] port = node['port'] dev = node['device'] headers = dict() headers['user-agent'] = 'transition-middleware' headers['X-Timestamp'] = normalize_timestamp(time.time()) headers['referer'] = req.as_referer() headers['x-size'] = '0' headers['x-content-type'] = 'text/plain' headers['x-etag'] = 'd41d8cd98f00b204e9800998ecf8427e' conn = http_connect(ip, port, dev, part, 'PUT', hidden_path, headers) conn.getresponse().read() return Response(status=HTTP_NO_CONTENT)
def __call__(self, env, start_response): req = Request(env) resp = req.get_response(self.app) self.logger.info("Serverless: available headers: {}".format( str(dict(req.headers)))) try: if "X-Function-URL" in req.headers: version, account, container, obj = split_path( req.path_info, 4, 4, True) self.logger.info( "Serverless: version {}, account {}, container {}, object {}" .format(version, account, container, obj)) if obj and is_success(resp.status_int) and req.method == 'PUT': webhook = req.headers.get("X-Function-URL") data = json.dumps({ "x-auth-token": req.headers.get("X-Auth-Token"), "version": version, "account": account, "container": container, "object": obj, "project_id": req.headers.get("X-Project-Id"), }) self.logger.info( "Serverless: data to send to a function {}".format( str(data))) data_as_bytes = data.encode('utf-8') webhook_req = urllib2.Request(webhook, data=data_as_bytes) webhook_req.add_header('Content-Type', 'application/json') webhook_req.add_header('Content-Length', len(data_as_bytes)) self.logger.info( "Serverless: data to send as bytes {}".format( data_as_bytes)) with Timeout(60): try: result = urllib2.urlopen(webhook_req).read() self.logger.info( "Serverless: function worked fine. Result {}". format(str(result))) except (Exception, Timeout) as ex: self.logger.error( 'Serverless: failed POST to webhook {}, ' 'error {}'.format(webhook, str(ex))) else: self.logger.info("Serverless: skipping functions middleware " "due to absence of function URL") except ValueError: # not an object request pass return self.app(env, start_response)
def __call__(self, env, start_response): req = Request(env) resp = req.get_response(self.app) try: (version, account, container, objname) = split_path(req.path_info, 1, 4, True) except ValueError: return resp(env, start_response) is_grok_request = req.params.has_key( 'grok') or 'grok-pattern' in req.headers # grok request has to be explicit, and only expected for GET operations if not req.method == 'GET' or not is_grok_request: return resp(env, start_response) self.logger.debug('Calling grok middleware') # make sure we have an object to work on if not objname or not resp.status_int == 200: return resp(env, start_response) # the grok pattern is expected to be in the request headers # if the pattern is missing, we ignore the grok request pattern = req.headers.get('grok-pattern') if not pattern: self.logger.debug( 'Object found, but no pattern requested, aborting') return self.get_err_response('Grok pattern is missing')( env, start_response) self.logger.debug('Starting grok operation') # we are going to assume the retrieved object is string object # and iterate through lines of resp.body and execute grok_match grokked_content = '' try: strbuf = StringIO.StringIO(resp.body) for line in strbuf: parsed_line = pygrok.grok_match(line, pattern) grokked_content += json.dumps(parsed_line) + '\n' except Exception as e: return self.get_err_response(str(e))(env, start_response) resp.body = grokked_content return resp(env, start_response)
def __call__(self, env, start_response): req = Request(env) resp = req.get_response(self.app) try: (version, account, container, objname) = split_path(req.path_info, 1, 4, True) except ValueError: return resp(env, start_response) is_grok_request = req.params.has_key("grok") or "grok-pattern" in req.headers # grok request has to be explicit, and only expected for GET operations if not req.method == "GET" or not is_grok_request: return resp(env, start_response) self.logger.debug("Calling grok middleware") # make sure we have an object to work on if not objname or not resp.status_int == 200: return resp(env, start_response) # the grok pattern is expected to be in the request headers # if the pattern is missing, we ignore the grok request pattern = req.headers.get("grok-pattern") if not pattern: self.logger.debug("Object found, but no pattern requested, aborting") return self.get_err_response("Grok pattern is missing")(env, start_response) self.logger.debug("Starting grok operation") # we are going to assume the retrieved object is string object # and iterate through lines of resp.body and execute grok_match grokked_content = "" try: strbuf = StringIO.StringIO(resp.body) for line in strbuf: parsed_line = pygrok.grok_match(line, pattern) grokked_content += json.dumps(parsed_line) + "\n" except Exception as e: return self.get_err_response(str(e))(env, start_response) resp.body = grokked_content return resp(env, start_response)
def _initialize(self): resp = None if self.swift_client: resp = self.swift_client.make_request('HEAD', self.path, {}, (2, 4)) elif self.env: req = Request(self.env) req.method = 'HEAD' req.path_info = self.path req.headers['Content-Length'] = '0' resp = req.get_response(self.app) if resp is None: return self.status = resp.status_int if is_success(self.status): self.headers = resp.headers
def __call__(self, env, start_response): req = Request(env) username = env.get('HTTP_X_USER_NAME',None) userid = env.get('HTTP_X_USER_ID',None) tenant = env.get('HTTP_X_PROJECT_NAME',None) version, account, container, obj = req.split_path(1,4,True) #COMMENT: Control the author of the request. if req.method == "PUT" and req.headers.get('x-container-read',None) is not None and container is not None and obj is None: #Associate owner to container req.headers['x-container-sysmeta-owner'] = userid req.headers['x-container-meta-owner'] = userid if req.method =="POST" and req.headers.get('x-container-read',None) is not None: new_req = Request.blank(req.path_info,None,req.headers,None) new_req.method = "HEAD" new_req.path_info = "/".join(["",version,account,container]) new_resp = new_req.get_response(self.app) if new_resp.headers.get('x-container-meta-bel-id',None) is None: #Container public -> private. Associate owner req.headers['x-container-sysmeta-owner'] = userid req.headers['x-container-meta-owner'] = userid elif new_resp.headers.get('x-container-sysmeta-owner',None) != userid: #Container already private and user is not the owner return HTTPUnauthorized(body="Unauthorized")(env, start_response) if req.method == "GET" and username != "ceilometer" and username != None: if obj != None: #Request a container new_req = Request.blank(req.path_info,None,req.headers,None) new_req.method = "HEAD" new_req.path_info = "/".join(["",version,account,container]) response = new_req.get_response(self.app) cont_header = response.headers container_sel_id = cont_header.get('x-container-meta-sel-id',None) cont_secret_ref = cont_header.get('x-container-meta-container-ref',None) env['swift_crypto_fetch_cont_id'] = container_sel_id resp_obj = req.get_response(self.app) object_sel_id = resp_obj.headers.get('x-object-meta-sel-id',None) if object_sel_id != container_sel_id:# and onResource=="False": #The object has been uploaded before the last policy change if object_sel_id is not None: old_dek = get_secret(self.userID,cont_secret_ref,object_sel_id,tenant).get('KEK',None) if old_dek is not None: env['swift_crypto_old_fetch_key'] = old_dek else: env['swift_crypto_old_fetch_key'] = "NotAuthorized" if container_sel_id is not None: dek = get_secret(self.userID,cont_secret_ref,container_sel_id,tenant).get('KEK',None) if dek is not None: env['swift_crypto_fetch_key'] = dek else: env['swift_crypto_fetch_key'] = "NotAuthorized" """elif obj == None and container == None: resp_account = req.get_response(self.app) list_containers = resp_account.body list_containers = json.loads(list_containers) for cont in list_containers: new_req = Request.blank(req.path_info,None,None,None) new_req.method = "GET" cont_name = cont.get('name','') new_req.path_info = "/".join(["",version,account,cont_name]) response = new_req.get_response(self.app) print response.headers print cont print "-----------------------" list_acl = self.extractACL(response.headers) if list_acl != [] and userid not in list_acl: print "list_acl\n" print list_acl list_containers.remove(cont) resp_account.body = json.dumps(list_containers)""" return self.app(env, start_response)