def work(self):
request = self.request
exists = self.user.exists(self.bucketName)
if self.acl:
if not exists:
raise cbException('NoSuchBucket')
(perms, data_key) = self.user.get_perms(self.bucketName)
ndx = perms.find("W")
if ndx < 0:
raise cbException('AccessDenied')
rc = self.grant_public_permissions(self.bucketName,
self.objectName)
if not rc:
xml = self.request.content.read()
pycb.log(logging.INFO, "xml %s" % (xml))
grants = parse_acl_request(xml)
for g in grants:
pycb.log(logging.INFO, "granting %s to %s" % (g[2], g[0]))
self.user.grant(g[0], self.bucketName, perms=g[2])
else:
if exists:
raise cbException('BucketAlreadyExists')
self.user.put_bucket(self.bucketName)
self.grant_public_permissions(self.bucketName, self.objectName)
self.set_common_headers()
self.setHeader(request, 'Content-Length', 0)
self.setHeader(request, 'Connection', 'close')
self.setHeader(request, 'Location', "/" + self.bucketName)
self.setResponseCode(request, 200, 'OK')
self.finish(request)
def work(self):
request = self.request
exists = self.user.exists(self.bucketName)
if self.acl:
if not exists:
raise cbException('NoSuchBucket')
(perms, data_key) = self.user.get_perms(self.bucketName)
ndx = perms.find("W")
if ndx < 0:
raise cbException('AccessDenied')
rc = self.grant_public_permissions(self.bucketName, self.objectName)
if not rc:
xml = self.request.content.read()
pycb.log(logging.INFO, "xml %s" % (xml))
grants = parse_acl_request(xml)
for g in grants:
pycb.log(logging.INFO, "granting %s to %s" % (g[2], g[0]))
self.user.grant(g[0], self.bucketName, perms=g[2])
else:
if exists:
raise cbException('BucketAlreadyExists')
self.user.put_bucket(self.bucketName)
self.grant_public_permissions(self.bucketName, self.objectName)
self.set_common_headers()
self.setHeader(request, 'Content-Length', 0)
self.setHeader(request, 'Connection', 'close')
self.setHeader(request, 'Location', "/" + self.bucketName)
self.setResponseCode(request, 200, 'OK')
self.finish(request)
def work(self):
exists = self.user.exists(self.bucketName)
if not exists:
raise cbException('NoSuchBucket')
(perms, data_key) = self.user.get_perms(self.bucketName)
ndx = perms.find("w")
if ndx < 0:
raise cbException('AccessDenied')
self.deleteIt(data_key)
def work(self):
exists = self.user.exists(self.bucketName)
if not exists:
raise cbException('NoSuchBucket')
(perms, data_key) = self.user.get_perms(self.bucketName)
ndx = perms.find("w")
if ndx < 0:
raise cbException('AccessDenied')
self.deleteIt(data_key)
def request_object_factory(self, request, user, path, requestId):
pycb.log(logging.INFO, "path %s" % (path))
# handle the one service operation
if path == "/":
if request.method == 'GET':
cbR = cbGetService(request, user, requestId,
pycb.config.bucket)
return cbR
raise cbException('InvalidArgument')
(bucketName, objectName) = path_to_bucket_object(path)
init_redirector(request, bucketName, objectName)
pycb.log(
logging.INFO, "path %s bucket %s object %s" %
(path, bucketName, str(objectName)))
if request.method == 'GET':
if objectName == None:
cbR = cbGetBucket(request, user, bucketName, requestId,
pycb.config.bucket)
else:
cbR = cbGetObject(request, user, bucketName, objectName,
requestId, pycb.config.bucket)
return cbR
elif request.method == 'PUT':
if objectName == None:
cbR = cbPutBucket(request, user, bucketName, requestId,
pycb.config.bucket)
else:
args = request.getAllHeaders()
if 'x-amz-copy-source' in args:
(srcBucketName, srcObjectName) = path_to_bucket_object(
args['x-amz-copy-source'])
cbR = cbCopyObject(request, user, requestId,
pycb.config.bucket, srcBucketName,
srcObjectName, bucketName, objectName)
else:
cbR = cbPutObject(request, user, bucketName, objectName,
requestId, pycb.config.bucket)
return cbR
elif request.method == 'POST':
pycb.log(logging.ERROR, "Nothing to handle POST")
elif request.method == 'DELETE':
if objectName == None:
cbR = cbDeleteBucket(request, user, bucketName, requestId,
pycb.config.bucket)
else:
cbR = cbDeleteObject(request, user, bucketName, objectName,
requestId, pycb.config.bucket)
return cbR
elif request.method == 'HEAD' and objectName != None:
cbR = cbHeadObject(request, user, bucketName, objectName,
requestId, pycb.config.bucket)
return cbR
raise cbException('InvalidArgument')
def work(self):
pycb.log(logging.INFO, "===== def work of cbRequest.py")
exists = self.user.exists(self.bucketName)
if not exists:
raise cbException('NoSuchBucket')
(perms, data_key) = self.user.get_perms(self.bucketName)
ndx = perms.find("w")
if ndx < 0:
raise cbException('AccessDenied')
self.deleteIt(data_key)
def check_permissions(self):
srcExists = self.user.exists(self.srcBucketName, self.srcObjectName)
if not srcExists:
raise cbException('NoSuchKey')
(perms, src_data_key) = self.user.get_perms(self.srcBucketName,
self.srcObjectName)
# make sure that we can read the source
ndx = perms.find("r")
if ndx < 0:
raise cbException('AccessDenied')
# make sure we can write to the destination
dstExists = self.user.exists(self.dstBucketName, self.dstObjectName)
(bperms, bdata_key) = self.user.get_perms(self.dstBucketName)
ndx = bperms.find("w")
if ndx < 0:
raise cbException('AccessDenied')
dst_data_key = None
dst_size = 0
if dstExists:
(perms,
dst_data_key) = self.user.get_perms(self.dstBucketName,
self.dstObjectName)
ndx = perms.find("w")
if ndx < 0:
raise cbException('AccessDenied')
(dst_size, ctm, md5) = self.user.get_info(self.dstBucketName,
self.dstObjectName)
(src_size, self.src_ctm,
self.src_md5) = self.user.get_info(self.srcBucketName,
self.srcObjectName)
# check the quota
remaining_quota = self.user.get_remaining_quota()
if remaining_quota != User.UNLIMITED:
if remaining_quota < src_size - dst_size:
pycb.log(
logging.INFO,
"user %s did not pass quota. file size %d quota %d" %
(self.user, src_size, remaining_quota))
raise cbException('AccountProblem')
# if we get to here we are allowed to do the copy
if dst_data_key == None:
self.dst_file = self.bucketIface.put_object(
self.dstBucketName, self.dstObjectName)
else:
self.dst_file = self.bucketIface.get_object(dst_data_key)
self.dst_file.set_delete_on_close(True)
self.src_file = self.bucketIface.get_object(src_data_key)
def parse_acl_request(xml):
dom = parseString(xml)
grant_a = dom.getElementsByTagName("Grant")
grants = []
users = {}
for g in grant_a:
email = getText(g.getElementsByTagName("DisplayName")[0].childNodes)
id = getText(g.getElementsByTagName("ID")[0].childNodes)
perm_set = g.getElementsByTagName("Permission")
perms = ""
for p in perm_set:
requested_perm = getText(p.childNodes)
if requested_perm not in perms_strings:
raise cbException('InvalidArgument')
pv = perms_strings[requested_perm]
ndx = perms.find(pv)
if ndx < 0:
perms = perms + pv
if id in users.keys():
(i, e, p) = users[id]
perms = perms + p
users[id] = (id, email, perms)
return users.values()
def __init__(self, data_key, access="r", openIt=True):
# file like stuff
#self.closed = True
#self.encoding =
#self.errors
#self.mode
#self.name
#self.newlines
#self.softspace
self.fname = data_key
self.metafname = data_key + ".meta"
self.data_key = data_key
self.blockSize = 1024*512
self.hashValue = None
self.delete_on_close = False
self.md5er = hashlib.md5()
if not openIt:
return
try:
mFile = open(self.metafname, 'r')
self.hashValue = mFile.readline()
mFile.close()
except:
pass
try:
self.file = open(self.fname, access)
except OSError, (OsEx):
if OsEx.errno == errno.ENOENT:
raise cbException('NoSuchKey')
def seek(self, offset, whence=None):
pycb.log(logging.INFO, "===== def seek of cbPosixBackend.py")
self.seek_count = self.seek_count + 1
pycb.log(logging.WARNING, "Someone is seeking %s %d :: %d" % (self.fname, offset, self.seek_count), tb=traceback)
if self.seek_count > 1:
raise cbException('InternalError')
return self.file.seek(offset, whence)
def __init__(self, data_key, access="r", openIt=True):
self.fname = data_key
self.metafname = data_key + ".meta"
self.data_key = data_key
self.blockSize = pycb.config.block_size
self.hashValue = None
self.delete_on_close = False
self.md5er = hashlib.md5()
self.access = access
self.seek_count = 0
if not openIt:
return
try:
# this allows head to be very fast
if access == "r":
mFile = open(self.metafname, 'r')
self.hashValue = mFile.readline()
mFile.close()
except:
pass
try:
self.file = open(self.fname, access)
except OSError, (OsEx):
if OsEx.errno == errno.ENOENT:
raise cbException('NoSuchKey')
def __init__(self, data_key, access="r", openIt=True):
self.fname = data_key
self.metafname = data_key + ".meta"
self.data_key = data_key
self.blockSize = pycb.config.block_size
self.hashValue = None
self.delete_on_close = False
self.md5er = hashlib.md5()
self.access = access
self.seek_count = 0
if not openIt:
return
try:
# this allows head to be very fast
if access == "r":
mFile = open(self.metafname, 'r')
self.hashValue = mFile.readline()
mFile.close()
except:
pass
try:
self.file = open(self.fname, access)
except OSError, (OsEx):
if OsEx.errno == errno.ENOENT:
raise cbException('NoSuchKey')
def get_uf(self, bucketName, objectName=None):
file = self.get_file_obj(bucketName, objectName)
if file == None:
pycb.log(logging.INFO, "b:o not found %s:%s" % (bucketName, str(objectName)))
raise cbException('NoSuchKey')
uf = UserFile(file, self.user)
return uf
def work(self):
exists = self.user.exists(self.bucketName)
if not exists:
raise cbException('NoSuchBucket')
(perms, data_key) = self.user.get_perms(self.bucketName)
if self.acl:
ndx = perms.find("R")
if ndx < 0:
raise cbException('AccessDenied')
else:
self.get_acl()
else:
ndx = perms.find("r")
if ndx < 0:
raise cbException('AccessDenied')
else:
self.list_bucket()
def get_uf(self, bucketName, objectName=None):
file = self.get_file_obj(bucketName, objectName)
if file == None:
pycb.log(logging.INFO,
"b:o not found %s:%s" % (bucketName, str(objectName)))
raise cbException('NoSuchKey')
uf = UserFile(file, self.user)
return uf
def work(self):
exists = self.user.exists(self.bucketName)
if not exists:
raise cbException('NoSuchBucket')
(perms, data_key) = self.user.get_perms(self.bucketName)
if self.acl:
ndx = perms.find("R")
if ndx < 0:
raise cbException('AccessDenied')
else:
self.get_acl()
else:
ndx = perms.find("r")
if ndx < 0:
raise cbException('AccessDenied')
else:
self.list_bucket()
def work(self):
exists = self.user.exists(self.bucketName, self.objectName)
if not exists:
raise cbException('NoSuchKey')
(perms, data_key) = self.user.get_perms(self.bucketName, self.objectName)
if self.acl:
ndx = perms.find("R")
if ndx < 0:
raise cbException('AccessDenied')
else:
self.get_acl()
else:
ndx = perms.find("r")
if ndx < 0:
raise cbException('AccessDenied')
else:
obj = self.bucketIface.get_object(data_key)
self.sendObject(obj)
def seek(self, offset, whence=None):
self.seek_count = self.seek_count + 1
pycb.log(logging.WARNING,
"Someone is seeking %s %d :: %d" %
(self.fname, offset, self.seek_count),
tb=traceback)
if self.seek_count > 1:
raise cbException('InternalError')
return self.file.seek(offset, whence)
def work(self):
pycb.log(logging.INFO, "===== def work of cbRequest.py")
exists = self.user.exists(self.bucketName)
if not exists:
raise cbException('NoSuchBucket')
(perms, data_key) = self.user.get_perms(self.bucketName)
if self.acl:
ndx = perms.find("R")
if ndx < 0:
raise cbException('AccessDenied')
else:
self.get_acl()
else:
ndx = perms.find("r")
if ndx < 0:
raise cbException('AccessDenied')
else:
self.list_bucket()
def delete_bucket(self, bucketName):
try:
file = self.get_file_obj(bucketName)
kids = file.get_all_children()
if len(list(kids)) != 0:
raise cbException('BucketNotEmpty')
file.delete()
finally:
self.db_obj.commit()
def __init__(self, alias_name, con_str):
self.db_obj = DB(con_str=con_str)
alias = User.find_alias(self.db_obj, alias_name, pynimbusauthz.alias_type_s3)
a_list = list(alias)
if len(a_list) < 1:
raise cbException('AccessDenied')
# pick the first one, hmmm XXX
self.alias = a_list[0]
self.user = self.alias.get_canonical_user()
def work(self):
exists = self.user.exists(self.bucketName, self.objectName)
if not exists:
raise cbException('NoSuchKey')
(perms, data_key) = self.user.get_perms(self.bucketName, self.objectName)
if self.acl:
ndx = perms.find("R")
if ndx < 0:
raise cbException('AccessDenied')
else:
self.get_acl()
else:
ndx = perms.find("r")
if ndx < 0:
raise cbException('AccessDenied')
else:
obj = self.bucketIface.get_object(data_key)
self.sendObject(obj)
def delete_bucket(self, bucketName):
try:
file = self.get_file_obj(bucketName)
kids = file.get_all_children()
if len(list(kids)) != 0:
raise cbException('BucketNotEmpty')
file.delete()
finally:
self.db_obj.commit()
def request_object_factory(self, request, user, path, requestId):
pycb.log(logging.INFO, "===== def request_object_factory of cumulus.py")
pycb.log(logging.INFO, "path %s" % (path))
# handle the one service operation
if path == "/":
if request.method == 'GET':
cbR = cbGetService(request, user, requestId, pycb.config.bucket)
return cbR
raise cbException('InvalidArgument')
(bucketName, objectName) = path_to_bucket_object(path)
init_redirector(request, bucketName, objectName)
pycb.log(logging.INFO, "path %s bucket %s object %s" % (path, bucketName, str(objectName)))
if request.method == 'GET':
if objectName == None:
cbR = cbGetBucket(request, user, bucketName, requestId, pycb.config.bucket)
else:
cbR = cbGetObject(request, user, bucketName, objectName, requestId, pycb.config.bucket)
return cbR
elif request.method == 'PUT':
if objectName == None:
cbR = cbPutBucket(request, user, bucketName, requestId, pycb.config.bucket)
else:
args = request.getAllHeaders()
if 'x-amz-copy-source' in args:
(srcBucketName, srcObjectName) = path_to_bucket_object(args['x-amz-copy-source'])
cbR = cbCopyObject(request, user, requestId, pycb.config.bucket, srcBucketName, srcObjectName, bucketName, objectName)
else:
cbR = cbPutObject(request, user, bucketName, objectName, requestId, pycb.config.bucket)
return cbR
elif request.method == 'POST':
pycb.log(logging.ERROR, "Nothing to handle POST")
elif request.method == 'DELETE':
if objectName == None:
cbR = cbDeleteBucket(request, user, bucketName, requestId, pycb.config.bucket)
else:
cbR = cbDeleteObject(request, user, bucketName, objectName, requestId, pycb.config.bucket)
return cbR
elif request.method == 'HEAD' and objectName != None:
cbR = cbHeadObject(request, user, bucketName, objectName, requestId, pycb.config.bucket)
return cbR
raise cbException('InvalidArgument')
def check_permissions(self):
pycb.log(logging.INFO, "===== def check_permissions of cbRequest.py")
srcExists = self.user.exists(self.srcBucketName, self.srcObjectName)
if not srcExists:
raise cbException('NoSuchKey')
(perms, src_data_key) = self.user.get_perms(self.srcBucketName, self.srcObjectName)
# make sure that we can read the source
ndx = perms.find("r")
if ndx < 0:
raise cbException('AccessDenied')
# make sure we can write to the destination
dstExists = self.user.exists(self.dstBucketName, self.dstObjectName)
(bperms, bdata_key) = self.user.get_perms(self.dstBucketName)
ndx = bperms.find("w")
if ndx < 0:
raise cbException('AccessDenied')
dst_data_key = None
dst_size = 0
if dstExists:
(perms, dst_data_key) = self.user.get_perms(self.dstBucketName, self.dstObjectName)
ndx = perms.find("w")
if ndx < 0:
raise cbException('AccessDenied')
(dst_size, ctm, md5) = self.user.get_info(self.dstBucketName, self.dstObjectName)
(src_size, self.src_ctm, self.src_md5) = self.user.get_info(self.srcBucketName, self.srcObjectName)
# check the quota
remaining_quota = self.user.get_remaining_quota()
if remaining_quota != User.UNLIMITED:
if remaining_quota < src_size - dst_size:
pycb.log(logging.INFO, "user %s did not pass quota. file size %d quota %d" % (self.user, src_size, remaining_quota))
raise cbException('AccountProblem')
# if we get to here we are allowed to do the copy
if dst_data_key == None:
self.dst_file = self.bucketIface.put_object(self.dstBucketName, self.dstObjectName)
else:
self.dst_file = self.bucketIface.get_object(dst_data_key)
self.dst_file.set_delete_on_close(True)
self.src_file = self.bucketIface.get_object(src_data_key)
def __init__(self, alias_name, con_str):
self.db_obj = DB(con_str=con_str)
alias = User.find_alias(self.db_obj, alias_name,
pynimbusauthz.alias_type_s3)
a_list = list(alias)
if len(a_list) < 1:
raise cbException('AccessDenied')
# pick the first one, hmmm XXX
self.alias = a_list[0]
self.user = self.alias.get_canonical_user()
def __init__(self, request, user, bucketName, objName, requestId, bucketIface):
cbRequest.__init__(self, request, user, requestId, bucketIface)
ndx = objName.find("cumulus:/")
if ndx >= 0:
pycb.log(logging.ERROR, "someone tried to make a key named cumulus://... why would someone do that? %d" % (ndx))
raise cbException('InvalidURI')
self.checkMD5 = None
self.bucketName = bucketName
self.objectName = objName
def get_owner(self, bucketName, objectName=None):
try:
uf = self.get_uf(bucketName, objectName)
o = uf.get_owner()
uas = list(o.get_alias_by_type(pynimbusauthz.alias_type_s3))
if len(uas) < 1:
raise cbException('InternalError')
return (uas[0].get_name(), uas[0].get_friendly_name())
finally:
self.db_obj.commit()
def __init__(self, request, user, bucketName, objName, requestId, bucketIface):
cbRequest.__init__(self, request, user, requestId, bucketIface)
ndx = objName.find("cumulus:/")
if ndx >= 0:
pycb.log(logging.ERROR, "someone tried to make a key named cumulus://... why would someone do that? %d" % (ndx))
raise cbException('InvalidURI')
self.checkMD5 = None
self.bucketName = bucketName
self.objectName = objName
def parse_acl_request(xml):
pycb.log(logging.INFO, "===== def parse_acl_request of cbRequest.py")
dom = parseString(xml)
grant_a = dom.getElementsByTagName("Grant")
grants = []
users = {}
for g in grant_a:
el = g.getElementsByTagName("URI")
if el != None and len(el) >= 1:
u = getText(el[0].childNodes)
if u == "http://acs.amazonaws.com/groups/global/AllUsers":
id = "CumulusPublicUser"
email = ""
else:
raise cbException('InvalidArgument')
else:
el = g.getElementsByTagName("DisplayName")
if el == None or len(el) < 1:
email = None
else:
email = getText(el[0].childNodes)
id = getText(g.getElementsByTagName("ID")[0].childNodes)
perm_set = g.getElementsByTagName("Permission")
perms = ""
for p in perm_set:
requested_perm = getText(p.childNodes)
if requested_perm not in perms_strings:
raise cbException('InvalidArgument')
pv = perms_strings[requested_perm]
ndx = perms.find(pv)
if ndx < 0:
perms = perms + pv
if id in users.keys():
(i, e, p) = users[id]
perms = perms + p
users[id] = (id, email, perms)
return users.values()
def send_access_error(self, req):
ex = cbException('AccessDenied')
m_msg = "HTTP/1.1 %s %s\r\n" % (ex.httpCode, ex.httpDesc)
self.transport.write(m_msg)
m_msg = "%s: %s\r\n" % (('x-amz-request-id', str(uuid.uuid1())))
self.transport.write(m_msg)
self.transport.write('content-type: text/html')
e_msg = ex.make_xml_string(self._path, str(uuid.uuid1()))
self.transport.write(e_msg)
self.transport.loseConnection()
def delete_bucket(self, bucketName):
pycb.log(logging.INFO, "===== def delete_bucket of cbAuthzSecurity.py")
try:
file = self.get_file_obj(bucketName)
kids = file.get_all_children()
if len(list(kids)) != 0:
raise cbException('BucketNotEmpty')
file.delete()
finally:
self.db_obj.commit()
def get_owner(self, bucketName, objectName=None):
try:
uf = self.get_uf(bucketName, objectName)
o = uf.get_owner()
uas = list(o.get_alias_by_type(pynimbusauthz.alias_type_s3))
if len(uas) < 1:
raise cbException('InternalError')
return (uas[0].get_name(), uas[0].get_friendly_name())
finally:
self.db_obj.commit()
def parse_acl_request(xml):
dom = parseString(xml)
grant_a = dom.getElementsByTagName("Grant")
grants = []
users = {}
for g in grant_a:
el = g.getElementsByTagName("URI")
if el != None and len(el) >= 1:
u = getText(el[0].childNodes)
if u == "http://acs.amazonaws.com/groups/global/AllUsers":
id = "CumulusPublicUser"
email = ""
else:
raise cbException('InvalidArgument')
else:
el = g.getElementsByTagName("DisplayName")
if el == None or len(el) < 1:
email = None
else:
email = getText(el[0].childNodes)
id = getText(g.getElementsByTagName("ID")[0].childNodes)
perm_set = g.getElementsByTagName("Permission")
perms = ""
for p in perm_set:
requested_perm = getText(p.childNodes)
if requested_perm not in perms_strings:
raise cbException('InvalidArgument')
pv = perms_strings[requested_perm]
ndx = perms.find(pv)
if ndx < 0:
perms = perms + pv
if id in users.keys():
(i, e, p) = users[id]
perms = perms + p
users[id] = (id, email, perms)
return users.values()
def grant(self, user_id, bucketName, objectName=None, perms="Rr"):
(o_perms, key) = self.getPerms(bucketName, objectName)
ndx = o_perms.find("W")
if ndx < 0:
raise cbException('AccessDenied')
if objectName == None:
f = self.getBucketPermFile(bucketName)
else:
f = self.getObjectFile(bucketName, objectName)
self.add_perms(f, user_id, perms)
pycb.log(logging.INFO, "granted: %s to %s" % (perms, user_id))
def grant(self, user_id, bucketName, objectName=None, perms="Rr"):
(o_perms, key) = self.getPerms(bucketName, objectName)
ndx = o_perms.find("W")
if ndx < 0:
raise cbException('AccessDenied')
if objectName == None:
f = self.getBucketPermFile(bucketName)
else:
f = self.getObjectFile(bucketName, objectName)
self.add_perms(f, user_id, perms)
pycb.log(logging.INFO, "granted: %s to %s" % (perms, user_id))
def init_redirector(req, bucketName, objectName):
redir_host = pycb.config.redirector.new_connection(req)
req.notifyFinish().addBoth(end_redirector, req)
if redir_host:
pycb.log(logging.INFO, "REDIRECT %s" % (redir_host))
ex = cbException('TemporaryRedirect')
req.setHeader('location', "http://%s%s" % (redir_host, req.uri))
ex.add_custom_xml("Bucket", bucketName)
ex.add_custom_xml("Endpoint", redir_host)
raise ex
def __init__(self, request, user, requestId, bucketIface, srcBucket, srcObject, dstBucket, dstObject):
cbRequest.__init__(self, request, user, requestId, bucketIface)
ndx = dstObject.find("cumulus:/")
if ndx >= 0:
pycb.log(logging.ERROR, "someone tried to make a key named cumulus://... why would someone do that? %d" % (ndx))
raise cbException('InvalidURI')
self.dstBucketName = dstBucket
self.dstObjectName = dstObject
self.srcBucketName = srcBucket
self.srcObjectName = srcObject
def __init__(self, request, user, requestId, bucketIface, srcBucket, srcObject, dstBucket, dstObject):
cbRequest.__init__(self, request, user, requestId, bucketIface)
ndx = dstObject.find("cumulus:/")
if ndx >= 0:
pycb.log(logging.ERROR, "someone tried to make a key named cumulus://... why would someone do that? %d" % (ndx))
raise cbException('InvalidURI')
self.dstBucketName = dstBucket
self.dstObjectName = dstObject
self.srcBucketName = srcBucket
self.srcObjectName = srcObject
def work(self):
pycb.log(logging.INFO, "===== def work of cbRequest.py")
dataObj = self.request.content
exists = self.user.exists(self.bucketName, self.objectName)
if self.acl:
if not exists:
raise cbException('NoSuchKey')
(perms, data_key) = self.user.get_perms(self.bucketName, self.objectName)
ndx = perms.find("W")
if ndx < 0:
raise cbException('AccessDenied')
rc = self.grant_public_permissions(self.bucketName, self.objectName)
if not rc:
xml = self.request.content.read()
pycb.log(logging.ERROR, "acl xml %s" % (xml))
grants = parse_acl_request(xml)
for g in grants:
pycb.log(logging.INFO, "granting %s to %s" % (g[2], g[0]))
self.user.grant(g[0], self.bucketName, self.objectName, perms=g[2])
self.set_common_headers()
self.setHeader(self.request, 'Content-Length', 0)
self.setHeader(self.request, 'Connection', 'close')
self.setHeader(self.request, 'Location', "/" + self.bucketName)
self.setResponseCode(self.request, 200, 'OK')
self.finish(self.request)
else:
(bperms, bdata_key) = self.user.get_perms(self.bucketName)
ndx = bperms.find("w")
if ndx < 0:
raise cbException('AccessDenied')
file_size = 0
if exists:
(perms, data_key) = self.user.get_perms(self.bucketName, self.objectName)
ndx = perms.find("w")
if ndx < 0:
raise cbException('AccessDenied')
# make sure they can write to the bucket
(perms, data_key) = self.user.get_perms(self.bucketName)
ndx = perms.find("w")
if ndx < 0:
raise cbException('AccessDenied')
(file_size, ctm, md5) = self.user.get_info(self.bucketName, self.objectName)
# gotta decide quota, if existed should get credit for the
# existing size
remaining_quota = self.user.get_remaining_quota()
if remaining_quota != User.UNLIMITED:
new_file_len = int(self.request.getHeader('content-length'))
if remaining_quota + file_size < new_file_len:
pycb.log(logging.INFO, "user %s did not pass quota. file size %d quota %d" % (self.user, new_file_len, remaining_quota))
raise cbException('AccountProblem')
obj = self.request.content
self.recvObject(self.request, obj)
def request_object_factory(self, request, user, path, requestId):
pycb.log(logging.INFO, "path %s" % (path))
# handle the one service operation
if path == "/":
if request.method == 'GET':
cbR = cbGetService(request, user, requestId, pycb.config.bucket)
return cbR
raise cbException('InvalidArgument')
(bucketName, objectName) = path_to_bucket_object(path)
pycb.log(logging.INFO, "path %s bucket %s object %s" % (path, bucketName, str(objectName)))
if request.method == 'GET':
if objectName == None:
cbR = cbGetBucket(request, user, bucketName, requestId, pycb.config.bucket)
else:
cbR = cbGetObject(request, user, bucketName, objectName, requestId, pycb.config.bucket)
return cbR
elif request.method == 'PUT':
if objectName == None:
cbR = cbPutBucket(request, user, bucketName, requestId, pycb.config.bucket)
else:
cbR = cbPutObject(request, user, bucketName, objectName, requestId, pycb.config.bucket)
return cbR
elif request.method == 'POST':
pycb.log(logging.ERROR, "Nothing to handle POST")
elif request.method == 'DELETE':
if objectName == None:
cbR = cbDeleteBucket(request, user, bucketName, requestId, pycb.config.bucket)
else:
cbR = cbDeleteObject(request, user, bucketName, objectName, requestId, pycb.config.bucket)
return cbR
elif request.method == 'HEAD' and objectName != None:
cbR = cbHeadObject(request, user, bucketName, objectName, requestId, pycb.config.bucket)
return cbR
raise cbException('InvalidArgument')
def get_user(self, id):
usr_ath_filename = getPosixAuthDir() + id
if os.path.exists(usr_ath_filename) == 0:
ex = cbException('InvalidAccessKeyId')
pycb.log(logging.ERROR, "could not open: %s" % (usr_ath_filename))
raise ex
auth_file = open(usr_ath_filename, 'r')
key = auth_file.readline()
key = key.strip()
display_name = auth_file.readline()
display_name = display_name.strip()
auth_file.close()
return cbPosixUserObject(id, display_name, key)
def get_user(self, id):
usr_ath_filename = getPosixAuthDir() + id
if os.path.exists(usr_ath_filename) == 0:
ex = cbException('InvalidAccessKeyId')
pycb.log(logging.ERROR, "could not open: %s" % (usr_ath_filename))
raise ex
auth_file = open(usr_ath_filename, 'r')
key = auth_file.readline()
key = key.strip()
display_name = auth_file.readline()
display_name = display_name.strip()
auth_file.close()
return cbPosixUserObject(id, display_name, key)
def work(self):
dataObj = self.request.content
exists = self.user.exists(self.bucketName, self.objectName)
if self.acl:
if not exists:
raise cbException('NoSuchKey')
(perms, data_key) = self.user.get_perms(self.bucketName, self.objectName)
ndx = perms.find("W")
if ndx < 0:
raise cbException('AccessDenied')
rc = self.grant_public_permissions(self.bucketName, self.objectName)
if not rc:
xml = self.request.content.read()
pycb.log(logging.ERROR, "acl xml %s" % (xml))
grants = parse_acl_request(xml)
for g in grants:
pycb.log(logging.INFO, "granting %s to %s" % (g[2], g[0]))
self.user.grant(g[0], self.bucketName, self.objectName, perms=g[2])
self.set_common_headers()
self.setHeader(self.request, 'Content-Length', 0)
self.setHeader(self.request, 'Connection', 'close')
self.setHeader(self.request, 'Location', "/" + self.bucketName)
self.setResponseCode(self.request, 200, 'OK')
self.finish(self.request)
else:
(bperms, bdata_key) = self.user.get_perms(self.bucketName)
ndx = bperms.find("w")
if ndx < 0:
raise cbException('AccessDenied')
file_size = 0
if exists:
(perms, data_key) = self.user.get_perms(self.bucketName, self.objectName)
ndx = perms.find("w")
if ndx < 0:
raise cbException('AccessDenied')
# make sure they can write to the bucket
(perms, data_key) = self.user.get_perms(self.bucketName)
ndx = perms.find("w")
if ndx < 0:
raise cbException('AccessDenied')
(file_size, ctm, md5) = self.user.get_info(self.bucketName, self.objectName)
# gotta decide quota, if existed should get credit for the
# existing size
remaining_quota = self.user.get_remaining_quota()
if remaining_quota != User.UNLIMITED:
new_file_len = int(self.request.getHeader('content-length'))
if remaining_quota + file_size < new_file_len:
pycb.log(logging.INFO, "user %s did not pass quota. file size %d quota %d" % (self.user, new_file_len, remaining_quota))
raise cbException('AccountProblem')
obj = self.request.content
self.recvObject(self.request, obj)
def authorize(headers, message_type, path, uri):
sent_auth = headers['authorization']
auth_A = sent_auth.split(':')
auth_hash = auth_A[1]
id = auth_A[0].split()[1].strip()
user = pycb.config.auth.get_user(id)
key = user.get_password()
pycb.log(logging.INFO, "AUTHORIZING %s %s %s" % (message_type, path, headers))
b64_hmac = pycb.get_auth_hash(key, message_type, path, headers, uri)
if auth_hash == b64_hmac:
return user
pycb.log(logging.ERROR, "%s %s %s" % (key, b64_hmac, auth_hash))
ec = 'AccessDenied'
ex = cbException(ec)
raise ex
def list_bucket(self, bucketIface, bucketName, args):
pycb.log(logging.INFO, "===== def list_bucket of cbPosixSecurity.py")
d = self.getBucketDir(bucketName)
if not os.path.isdir(d):
pycb.log(logging.ERROR, "did not find %s" %(d))
raise cbException('NoSuchBucket')
if 'prefix' in args:
p = args['prefix'][0]
p = self.hashFile(p)
prefix = d + "/" + p + "*"
else:
prefix = d + "/" + "*"
dir_list = glob.glob(prefix)
if 'max-keys' in args:
max_a = args['max-keys']
max = int(max_a[0])
else:
max = len(dir_list) + 1
results = []
for f in dir_list:
if len(results) >= max:
break
file = f.replace(d, "")
st = os.stat(f)
if stat.S_ISREG(st.st_mode):
# XXX just pick the size of the first one
dir_list = os.listdir(d)
if len(dir_list) > 0:
data_key = self.getDataKey(d+"/"+dir_list[0])
size = bucketIface.getSize(data_key)
tm = bucketIface.getModTime(data_key)
key = self.unhashFile(file)
display_name = key
obj = cbObject(tm, size, key, display_name, self)
results.append(obj)
return results
def get_acl(self, bucketName, objectName=None):
(perms, key) = self.getPerms(bucketName, objectName)
ndx = perms.find("R")
if ndx < 0:
raise cbException('AccessDenied')
if objectName == None:
f = self.getBucketPermFile(bucketName)
else:
f = self.getObjectFile(bucketName, objectName)
auth_file = open(f, 'r')
key = auth_file.readline()
owner_a = self.read_owner_line(auth_file)
line = self.read_auth_line(auth_file)
acls = {}
while line != None:
(id, display_name, perms) = line
acls[id] = line
line = self.read_auth_line(auth_file)
auth_file.close()
return acls.itervalues()
def get_acl(self, bucketName, objectName=None):
(perms, key) = self.getPerms(bucketName, objectName)
ndx = perms.find("R")
if ndx < 0:
raise cbException('AccessDenied')
if objectName == None:
f = self.getBucketPermFile(bucketName)
else:
f = self.getObjectFile(bucketName, objectName)
auth_file = open(f, 'r')
key = auth_file.readline()
owner_a = self.read_owner_line(auth_file)
line = self.read_auth_line(auth_file)
acls = {}
while line != None:
(id, display_name, perms) = line
acls[id] = line
line = self.read_auth_line(auth_file)
auth_file.close()
return acls.itervalues()
def list_bucket(self, bucketIface, bucketName, args):
d = self.getBucketDir(bucketName)
if not os.path.isdir(d):
pycb.log(logging.ERROR, "did not find %s" % (d))
raise cbException('NoSuchBucket')
if 'prefix' in args:
p = args['prefix'][0]
p = self.hashFile(p)
prefix = d + "/" + p + "*"
else:
prefix = d + "/" + "*"
dir_list = glob.glob(prefix)
if 'max-keys' in args:
max_a = args['max-keys']
max = int(max_a[0])
else:
max = len(dir_list) + 1
results = []
for f in dir_list:
if len(results) >= max:
break
file = f.replace(d, "")
st = os.stat(f)
if stat.S_ISREG(st.st_mode):
# XXX just pick the size of the first one
dir_list = os.listdir(d)
if len(dir_list) > 0:
data_key = self.getDataKey(d + "/" + dir_list[0])
size = bucketIface.getSize(data_key)
tm = bucketIface.getModTime(data_key)
key = self.unhashFile(file)
display_name = key
obj = cbObject(tm, size, key, display_name, self)
results.append(obj)
return results
def work(self):
(perms, data_key) = self.user.get_perms(self.bucketName, self.objectName)
ndx = perms.find("r")
if ndx < 0:
raise cbException('AccessDenied')
(sz, tm, md5) = self.user.get_info(self.bucketName, self.objectName)
d_str = "%04d-%02d-%02dT%02d:%02d:%02d.000Z" % (tm.tm_year, tm.tm_mon, tm.tm_wday, tm.tm_hour, tm.tm_min, tm.tm_sec)
self.set_common_headers()
self.setHeader(self.request, 'Content-Type', 'binary/octet-stream')
self.setHeader(self.request, 'Last-Modified', d_str)
self.setHeader(self.request, 'ETag', '"%s"' % (str(md5)))
self.setHeader(self.request, 'Content-Length', str(sz))
#Content-Type: text/plain
self.setResponseCode(self.request, 200, 'OK')
self.finish(self.request)
def work(self):
(perms, data_key) = self.user.get_perms(self.bucketName, self.objectName)
ndx = perms.find("r")
if ndx < 0:
raise cbException('AccessDenied')
(sz, tm, md5) = self.user.get_info(self.bucketName, self.objectName)
d_str = "%04d-%02d-%02dT%02d:%02d:%02d.000Z" % (tm.tm_year, tm.tm_mon, tm.tm_wday, tm.tm_hour, tm.tm_min, tm.tm_sec)
self.set_common_headers()
self.setHeader(self.request, 'Content-Type', 'binary/octet-stream')
self.setHeader(self.request, 'Last-Modified', d_str)
self.setHeader(self.request, 'ETag', '"%s"' % (str(md5)))
self.setHeader(self.request, 'Content-Length', str(sz))
#Content-Type: text/plain
self.setResponseCode(self.request, 200, 'OK')
self.finish(self.request)
# now that we have the file set delete on close to false
# it will now be safe to deal with dropped connections
# without having large files left around
dataObj.set_delete_on_close(False)
# dataObj.close() do no need to close for now. twisted will
# do this for us
self.user.put_object(dataObj, self.bucketName, self.objectName)
self.grant_public_permissions(self.bucketName, self.objectName)
self.finish(self.request)
except cbException, (ex):
ex.sendErrorResponse(self.request, self.requestId)
except:
traceback.print_exc(file=sys.stdout)
gdEx = cbException('InvalidArgument')
gdEx.sendErrorResponse(self.request, self.requestId)
# receive object looks strange because twisted has already received
# the entire file and put it in a temp location. we now just have
# to recognize that we have it all
def recvObject(self, request, dataObj):
self.set_common_headers()
self.setHeader(request, 'Connection', 'close')
self.setHeader(request, 'Content-Length', 0)
self.setResponseCode(request, 200, 'OK')
self.dataObj = dataObj
self.block_size = 1024 * 256
headers = request.getAllHeaders()
for (k, v) in headers.iteritems():
class cbGetObject(cbRequest):
def __init__(self, request, user, bucketName, objName, requestId,
bucketIface):
cbRequest.__init__(self, request, user, requestId, bucketIface)
self.bucketName = bucketName
self.objectName = objName
def work(self):
exists = self.user.exists(self.bucketName, self.objectName)
if not exists:
raise cbException('NoSuchKey')
(perms, data_key) = self.user.get_perms(self.bucketName,
self.objectName)
if self.acl:
ndx = perms.find("R")
if ndx < 0:
raise cbException('AccessDenied')
else:
self.get_acl()
else:
ndx = perms.find("r")
if ndx < 0:
raise cbException('AccessDenied')
else:
obj = self.bucketIface.get_object(data_key)
self.sendObject(obj)
def get_acl(self):
payload = self.get_acl_xml()
pycb.log(logging.INFO, "GET BUCKET ACL XML %s" % (payload))
self.send_xml(payload)
self.finish(self.request)
def calcMd5Sum(self, dataObj):
md5obj = self.bucketIface.get_object(dataObj.getDataKey())
md5er = hashlib.md5()
done = False
while not done:
b = md5obj.read()
if len(b) > 0:
md5er.update(b)
else:
done = True
md5obj.close()
etag = str(md5er.hexdigest()).strip()
return etag
def sendFile(self, dataObj):
try:
etag = dataObj.get_md5()
if etag == None:
etag = self.etag
if etag == None:
etag = self.calcMd5Sum(dataObj)
dataObj.set_md5(etag)
self.setHeader(self.request, 'ETag', '"%s"' % (etag))
self.setResponseCode(self.request, 200, 'OK')
fp = dataObj
d = FileSender().beginFileTransfer(fp, self.request)
def cbFinished(ignored):
fp.close()
self.request.finish()
d.addErrback(err).addCallback(cbFinished)
except cbException, (ex):
ex.sendErrorResponse(self.request, self.requestId)
traceback.print_exc(file=sys.stdout)
pycb.log(logging.ERROR, "Error sending file %s" % (str(ex)),
traceback)
except Exception, ex2:
traceback.print_exc(file=sys.stdout)
gdEx = cbException('InvalidArgument')
gdEx.sendErrorResponse(self.request, self.requestId)
pycb.log(logging.ERROR, "Error sending file %s" % (str(ex2)),
traceback)
def getBucketLocation(self, bucketName):
ex = cbException('NotImplemented')
raise ex
def post_object():
ex = cbException('NotImplemented')
raise ex
def copy_object(self, srcObjectName, dstObjectName, moveOrCopy,
httpHeaders):
ex = cbException('NotImplemented')
raise ex
class CBService(resource.Resource):
isLeaf = True
def __init__(self):
pass
def get_port(self):
return pycb.config.port
# amazon uses some smaller num generator, might have to match theres
# for clients that make assumptions
def next_request_id(self):
return str(uuid.uuid1()).replace("-", "")
# figure out if the operation is targeted at a service, bucket, or
# object
def request_object_factory(self, request, user, path, requestId):
pycb.log(logging.INFO, "path %s" % (path))
# handle the one service operation
if path == "/":
if request.method == 'GET':
cbR = cbGetService(request, user, requestId, pycb.config.bucket)
return cbR
raise cbException('InvalidArgument')
(bucketName, objectName) = path_to_bucket_object(path)
init_redirector(request, bucketName, objectName)
pycb.log(logging.INFO, "path %s bucket %s object %s" % (path, bucketName, str(objectName)))
if request.method == 'GET':
if objectName == None:
cbR = cbGetBucket(request, user, bucketName, requestId, pycb.config.bucket)
else:
cbR = cbGetObject(request, user, bucketName, objectName, requestId, pycb.config.bucket)
return cbR
elif request.method == 'PUT':
if objectName == None:
cbR = cbPutBucket(request, user, bucketName, requestId, pycb.config.bucket)
else:
args = request.getAllHeaders()
if 'x-amz-copy-source' in args:
(srcBucketName, srcObjectName) = path_to_bucket_object(args['x-amz-copy-source'])
cbR = cbCopyObject(request, user, requestId, pycb.config.bucket, srcBucketName, srcObjectName, bucketName, objectName)
else:
cbR = cbPutObject(request, user, bucketName, objectName, requestId, pycb.config.bucket)
return cbR
elif request.method == 'POST':
pycb.log(logging.ERROR, "Nothing to handle POST")
elif request.method == 'DELETE':
if objectName == None:
cbR = cbDeleteBucket(request, user, bucketName, requestId, pycb.config.bucket)
else:
cbR = cbDeleteObject(request, user, bucketName, objectName, requestId, pycb.config.bucket)
return cbR
elif request.method == 'HEAD' and objectName != None:
cbR = cbHeadObject(request, user, bucketName, objectName, requestId, pycb.config.bucket)
return cbR
raise cbException('InvalidArgument')
# everything does through here to localize access control
def process_event(self, request):
try:
rPath = createPath(request.getAllHeaders(), request.path)
requestId = self.next_request_id()
pycb.log(logging.INFO, "%s %s Incoming" % (requestId, str(datetime.now())))
pycb.log(logging.INFO, "%s %s" % (requestId, str(request)))
pycb.log(logging.INFO, "%s %s" % (requestId, rPath))
pycb.log(logging.INFO, "%s %s" %(requestId, request.getAllHeaders()))
pycb.log(logging.INFO, "request URI %s method %s %s" %(request.uri, request.method, str(request.args)))
user = authorize(request.getAllHeaders(), request.method, rPath, request.uri)
self.allowed_event(request, user, requestId, rPath)
except cbException, ex:
eMsg = ex.sendErrorResponse(request, requestId)
pycb.log(logging.ERROR, eMsg, traceback)
except Exception, ex2:
traceback.print_exc(file=sys.stdout)
gdEx = cbException('InternalError')
eMsg = gdEx.sendErrorResponse(request, requestId)
pycb.log(logging.ERROR, eMsg, traceback)
