def finish(self, request):
pycb.log(
logging.INFO,
"%s %s Reply sent %d %s" % (self.requestId, str(
datetime.now()), self.responseCode, self.responseMsg))
pycb.log(logging.INFO, str(self.outGoingHeaders))
request.finish()
def main(argv=sys.argv[0:]):
pycb.config.parse_cmdline(argv)
cumulus = CumulusRunner()
pycb.log(logging.INFO, "listening at %s" % (str(cumulus.getListener())))
cumulus.run()
return 0
def sendFile(self, dataObj):
try:
etag = dataObj.get_md5()
if etag == None:
etag = self.etag
if etag == None:
etag = self.calcMd5Sum(dataObj)
dataObj.set_md5(etag)
self.setHeader(self.request, 'ETag', '"%s"' % (etag))
self.setResponseCode(self.request, 200, 'OK')
fp = dataObj
d = FileSender().beginFileTransfer(fp, self.request)
def cbFinished(ignored):
fp.close()
self.request.finish()
d.addErrback(err).addCallback(cbFinished)
except cbException, (ex):
ex.sendErrorResponse(self.request, self.requestId)
traceback.print_exc(file=sys.stdout)
pycb.log(logging.ERROR, "Error sending file %s" % (str(ex)),
traceback)
def work(self):
request = self.request
exists = self.user.exists(self.bucketName)
if self.acl:
if not exists:
raise cbException('NoSuchBucket')
(perms, data_key) = self.user.get_perms(self.bucketName)
ndx = perms.find("W")
if ndx < 0:
raise cbException('AccessDenied')
rc = self.grant_public_permissions(self.bucketName,
self.objectName)
if not rc:
xml = self.request.content.read()
pycb.log(logging.INFO, "xml %s" % (xml))
grants = parse_acl_request(xml)
for g in grants:
pycb.log(logging.INFO, "granting %s to %s" % (g[2], g[0]))
self.user.grant(g[0], self.bucketName, perms=g[2])
else:
if exists:
raise cbException('BucketAlreadyExists')
self.user.put_bucket(self.bucketName)
self.grant_public_permissions(self.bucketName, self.objectName)
self.set_common_headers()
self.setHeader(request, 'Content-Length', 0)
self.setHeader(request, 'Connection', 'close')
self.setHeader(request, 'Location', "/" + self.bucketName)
self.setResponseCode(request, 200, 'OK')
self.finish(request)
def work(self):
request = self.request
exists = self.user.exists(self.bucketName)
if self.acl:
if not exists:
raise cbException('NoSuchBucket')
(perms, data_key) = self.user.get_perms(self.bucketName)
ndx = perms.find("W")
if ndx < 0:
raise cbException('AccessDenied')
rc = self.grant_public_permissions(self.bucketName, self.objectName)
if not rc:
xml = self.request.content.read()
pycb.log(logging.INFO, "xml %s" % (xml))
grants = parse_acl_request(xml)
for g in grants:
pycb.log(logging.INFO, "granting %s to %s" % (g[2], g[0]))
self.user.grant(g[0], self.bucketName, perms=g[2])
else:
if exists:
raise cbException('BucketAlreadyExists')
self.user.put_bucket(self.bucketName)
self.grant_public_permissions(self.bucketName, self.objectName)
self.set_common_headers()
self.setHeader(request, 'Content-Length', 0)
self.setHeader(request, 'Connection', 'close')
self.setHeader(request, 'Location', "/" + self.bucketName)
self.setResponseCode(request, 200, 'OK')
self.finish(request)
def get_md5(self):
pycb.log(logging.INFO, "===== def get_md5 of cbPosixBackend.py")
if self.hashValue == None:
v = str(self.md5er.hexdigest()).strip()
return v
pycb.log(logging.INFO, "=====## self.hashValue is %s"%self.hashValue)
return self.hashValue
def create_user(self, display_name, id, pw, opts):
pycb.log(logging.INFO, "===== def create_user of cbAuthzSecurity.py")
db_obj = DB(con_str=self.con_str)
user = User(db_obj, friendly=display_name)
user_alias = user.create_alias(id, "s3", display_name, alias_data=pw)
db_obj.commit()
db_obj.close()
def list_bucket(self, bucketName, args):
pycb.log(logging.INFO, "===== def list_bucket of cbAuthzSecurity.py")
clause = " ORDER BY name"
prefix = None
if 'prefix' in args:
prefix = args['prefix'][0]
prefix = "%s%%" % (prefix)
limit = None
if 'max-keys' in args:
max_a = args['max-keys']
limit = int(max_a[0])
if 'delimiter' in args:
pass
if 'key-marker' in args:
km = args['key-marker'][0]
clause = " and name > '%s'" % (km)
try:
bucket = File.get_file(self.db_obj, bucketName, pynimbusauthz.alias_type_s3)
iter = bucket.get_all_children(limit=limit, match_str=prefix, clause=clause)
new_it = itertools.imap(lambda r: _convert_File_to_cbObject(self, r), iter)
return list(new_it)
finally:
self.db_obj.commit()
def delete(self):
try:
os.unlink(self.metafname)
except Exception, ex:
pycb.log(
logging.WARNING, "error deleting %s %s %s" %
(self.metafname, str(sys.exc_info()[0]), str(ex)))
def allowed_event(self, request, user, requestId, path):
pycb.log(
logging.INFO, "Access granted to ID=%s requestId=%s uri=%s" %
(user.get_id(), requestId, request.uri))
cbR = self.request_object_factory(request, user, path, requestId)
cbR.work()
def exists(self, bucketName, objectName=None):
pycb.log(logging.INFO, "===== def exists of cbAuthzSecurity.py")
try:
file = self.get_file_obj(bucketName, objectName)
return file != None
finally:
self.db_obj.commit()
def get_uf(self, bucketName, objectName=None):
file = self.get_file_obj(bucketName, objectName)
if file == None:
pycb.log(logging.INFO, "b:o not found %s:%s" % (bucketName, str(objectName)))
raise cbException('NoSuchKey')
uf = UserFile(file, self.user)
return uf
def perm2string(p):
pycb.log(logging.INFO, "===== def perm2string of cbRequest.py")
global perms_strings
for (k, v) in perms_strings.iteritems():
if p == v:
return k
return None
def new_connection(self, request):
pycb.log(logging.INFO, "===== def new_connection of cbRedirector.py")
h = None
self.connection_count = self.connection_count + 1
if self.connection_count > self.max:
h = self.get_next_host()
return h
def get_perms(self, bucketName, objectName=None):
pycb.log(logging.INFO, "===== def get_perms of cbPosixSecurity.py")
if objectName == None:
perms = self.getBucketPerms(bucketName)
else:
perms = self.getObjectPerms(bucketName, objectName)
return perms
def exists(self, bucketName, objectName=None):
pycb.log(logging.INFO, "===== def exists of cbPosixSecurity.py")
if objectName == None:
f = self.getBucketPermFile(bucketName)
else:
f = self.getObjectFile(bucketName, objectName)
return os.path.exists(f)
def end_copy(self):
pycb.log(logging.INFO, "===== def end_copy of cbRequest.py")
try:
self.user.put_object(self.dst_file, self.dstBucketName, self.dstObjectName)
self.grant_public_permissions(self.dstBucketName, self.dstObjectName)
doc = Document()
cor = doc.createElement("CopyObjectResult")
doc.appendChild(cor)
lm = doc.createElement("LastModified")
cor.appendChild(lm)
lmText = doc.createTextNode(datetime(*self.src_ctm[:6]).isoformat())
lm.appendChild(lmText)
lm = doc.createElement("ETag")
cor.appendChild(lm)
lmText = doc.createTextNode(str(self.src_md5))
lm.appendChild(lmText)
x = doc.toxml();
self.setHeader(self.request, 'x-amz-copy-source-version-id', "1")
self.setHeader(self.request, 'x-amz-version-id', "1")
self.send_xml(x)
self.request.finish()
except cbException, (ex):
ex.sendErrorResponse(self.request, self.requestId)
traceback.print_exc(file=sys.stdout)
def read_owner_line(self, auth_file):
pycb.log(logging.INFO, "===== def read_owner_line of cbPosixSecurity.py")
line = auth_file.readline()
linea = line.split("::", 1)
id = linea[0].strip()
display_name = linea[1].strip()
return (id, display_name)
def getText(nodelist):
pycb.log(logging.INFO, "===== def getText of cbRequest.py")
rc = ""
for node in nodelist:
if node.nodeType == node.TEXT_NODE:
rc = rc + node.data
return rc
def deleteIt(self, data_key):
pycb.log(logging.INFO, "===== def deleteIt of cbRequest.py")
request = self.request
self.set_no_content_header()
self.bucketIface.delete_object(data_key)
self.user.delete_object(self.bucketName, self.objectName)
self.finish(request)
def list_bucket(self):
pycb.log(logging.INFO, "===== def list_bucket of cbRequest.py")
dirL = self.user.list_bucket(self.bucketName, self.request.args)
doc = Document()
# Create the <wml> base element
xList = doc.createElement("ListBucketResult")
xList.setAttribute("xmlns", "http://doc.s3.amazonaws.com/2006-03-01")
doc.appendChild(xList)
# Create the main <card> element
xName = doc.createElement("Name")
xList.appendChild(xName)
xNameText = doc.createTextNode(str(self.bucketName))
xName.appendChild(xNameText)
xIsTruncated = doc.createElement("IsTruncated")
xList.appendChild(xIsTruncated)
xIsTText = doc.createTextNode('false')
xIsTruncated.appendChild(xIsTText)
for obj in dirL:
xObj = obj.create_xml_element(doc)
xList.appendChild(xObj)
x = doc.toxml();
self.send_xml(x)
self.finish(self.request)
def seek(self, offset, whence=None):
pycb.log(logging.INFO, "===== def seek of cbPosixBackend.py")
self.seek_count = self.seek_count + 1
pycb.log(logging.WARNING, "Someone is seeking %s %d :: %d" % (self.fname, offset, self.seek_count), tb=traceback)
if self.seek_count > 1:
raise cbException('InternalError')
return self.file.seek(offset, whence)
def get_info(self, bucketName, objectName=None):
pycb.log(logging.INFO, "===== def get_info of cbAuthzSecurity.py")
try:
file = self.get_file_obj(bucketName, objectName)
return (file.get_size(), file.get_creation_time(), file.get_md5sum())
finally:
self.db_obj.commit()
def delete_object(self, bucketName, objectName):
pycb.log(logging.INFO, "===== def delete_object of cbAuthzSecurity.py")
try:
file = self.get_file_obj(bucketName, objectName)
file.delete()
finally:
self.db_obj.commit()
def read(self, size=None):
pycb.log(logging.INFO, "===== def read of cbPosixBackend.py")
if size == None:
st = self.file.read(self.blockSize)
else:
st = self.file.read(size)
self.md5er.update(st)
return st
def get_my_buckets(self):
pycb.log(logging.INFO, "===== def get_my_buckets of cbAuthzSecurity.py")
try:
file_iterater = File.get_user_files(self.db_obj, self.user, root=True)
new_it = itertools.imap(lambda r: _convert_bucket_to_cbObject(self, r), file_iterater)
return list(new_it)
finally:
self.db_obj.commit()
def get_uf(self, bucketName, objectName=None):
file = self.get_file_obj(bucketName, objectName)
if file == None:
pycb.log(logging.INFO,
"b:o not found %s:%s" % (bucketName, str(objectName)))
raise cbException('NoSuchKey')
uf = UserFile(file, self.user)
return uf
def get_remaining_quota(self):
pycb.log(logging.INFO, "===== def get_remaining_quota of cbAuthzSecurity.py")
quota = self.user.get_quota()
if quota == User.UNLIMITED:
return User.UNLIMITED
u = self.user.get_quota_usage()
return quota - u
def main(argv=sys.argv[0:]):
pycb.config.parse_cmdline(argv)
try:
cumulus = CumulusRunner()
except Exception, ex:
pycb.log(logging.ERROR, "error starting the server, check that the port is not already taken: %s" % (str(ex)), tb=traceback)
raise ex
def _convert_bucket_to_cbObject(user, file):
pycb.log(logging.INFO, "===== def _convert_bucket_to_cbObject of cbAuthzSecurity.py")
tm = file.get_creation_time()
size = -1
key = file.get_name()
display_name = file.get_name()
obj = cbObject(tm, size, key, display_name, user)
return obj
def merge_permissions(p1, p2):
pycb.log(logging.INFO, "===== def merge_permissions of cbAuthzSecurity.py")
perms = p2
for i in range(0, len(p1)):
ndx = p2.find(p1[i])
if ndx < 0:
perms = perms + p1[i]
return perms
def send_xml(self, x):
xLen = len(x)
self.set_common_headers()
self.setHeader(self.request, 'Content-Length', str(xLen))
self.setHeader(self.request, 'Connection', 'close')
self.setResponseCode(self.request, 200, 'OK')
self.request.write(x)
pycb.log(logging.INFO, "Sent %s" % (x))
def send_xml(self, x):
xLen = len(x)
self.set_common_headers()
self.setHeader(self.request, 'Content-Length', str(xLen))
self.setHeader(self.request, 'Connection', 'close')
self.setResponseCode(self.request, 200, 'OK')
self.request.write(x)
pycb.log(logging.INFO, "Sent %s" % (x))
def get_user_id_by_display(self, display_name):
pycb.log(logging.INFO, "===== def get_user_id_by_display of cbAuthzSecurity.py")
db_obj = DB(con_str=self.con_str)
a_it = UserAlias.find_alias_by_friendly(db_obj, display_name)
a_list = list(a_it)
if len(a_list) < 1:
return None
alias = a_list[0]
return alias.get_name()
def seek(self, offset, whence=None):
self.seek_count = self.seek_count + 1
pycb.log(logging.WARNING,
"Someone is seeking %s %d :: %d" %
(self.fname, offset, self.seek_count),
tb=traceback)
if self.seek_count > 1:
raise cbException('InternalError')
return self.file.seek(offset, whence)
def set_common_headers(self):
pycb.log(logging.INFO, "===== def set_common_headers of cbRequest.py")
amzid2 = str(uuid.uuid1()).replace("-", "")
self.setHeader(self.request, 'x-amz-id-2', amzid2)
self.setHeader(self.request,'x-amz-request-id', self.requestId)
self.setHeader(self.request, 'Server', "cumulus")
tm = datetime.utcnow()
tmstr = tm.strftime("%a, %d %b %Y %H:%M:%S GMT")
self.setHeader(self.request, 'date', tmstr)
def work(self):
dataObj = self.request.content
exists = self.user.exists(self.bucketName, self.objectName)
if self.acl:
if not exists:
raise cbException('NoSuchKey')
(perms, data_key) = self.user.get_perms(self.bucketName, self.objectName)
ndx = perms.find("W")
if ndx < 0:
raise cbException('AccessDenied')
rc = self.grant_public_permissions(self.bucketName, self.objectName)
if not rc:
xml = self.request.content.read()
grants = parse_acl_request(xml)
for g in grants:
pycb.log(logging.INFO, "granting %s to %s" % (g[2], g[0]))
self.user.grant(g[0], self.bucketName, self.objectName, perms=g[2])
self.set_common_headers()
self.setHeader(self.request, 'Content-Length', 0)
self.setHeader(self.request, 'Connection', 'close')
self.setHeader(self.request, 'Location', "/" + self.bucketName)
self.setResponseCode(self.request, 200, 'OK')
self.finish(self.request)
else:
(bperms, bdata_key) = self.user.get_perms(self.bucketName)
ndx = bperms.find("w")
if ndx < 0:
raise cbException('AccessDenied')
file_size = 0
if exists:
(perms, data_key) = self.user.get_perms(self.bucketName, self.objectName)
ndx = perms.find("w")
if ndx < 0:
raise cbException('AccessDenied')
# make sure they can write to the bucket
(perms, data_key) = self.user.get_perms(self.bucketName)
ndx = perms.find("w")
if ndx < 0:
raise cbException('AccessDenied')
(file_size, ctm, md5) = self.user.get_info(self.bucketName, self.objectName)
# gotta decide quota, if existed should get credit for the
# existing size
remaining_quota = self.user.get_remaining_quota()
if remaining_quota != User.UNLIMITED:
new_file_len = int(self.request.getHeader('content-length'))
if remaining_quota + file_size < new_file_len:
pycb.log(logging.INFO, "user %s did not pass quota. file size %d quota %d" % (self.user, new_file_len, remaining_quota))
raise cbException('AccountProblem')
obj = self.request.content
self.recvObject(self.request, obj)
def __init__(self, request, user, bucketName, objName, requestId, bucketIface):
cbRequest.__init__(self, request, user, requestId, bucketIface)
ndx = objName.find("cumulus:/")
if ndx >= 0:
pycb.log(logging.ERROR, "someone tried to make a key named cumulus://... why would someone do that? %d" % (ndx))
raise cbException('InvalidURI')
self.checkMD5 = None
self.bucketName = bucketName
self.objectName = objName
def __init__(self, request, user, bucketName, objName, requestId, bucketIface):
cbRequest.__init__(self, request, user, requestId, bucketIface)
ndx = objName.find("cumulus:/")
if ndx >= 0:
pycb.log(logging.ERROR, "someone tried to make a key named cumulus://... why would someone do that? %d" % (ndx))
raise cbException('InvalidURI')
self.checkMD5 = None
self.bucketName = bucketName
self.objectName = objName
def init_redirector(req, bucketName, objectName):
redir_host = pycb.config.redirector.new_connection(req)
req.notifyFinish().addBoth(end_redirector, req)
if redir_host:
pycb.log(logging.INFO, "REDIRECT %s" % (redir_host))
ex = cbException('TemporaryRedirect')
req.setHeader('location', "http://%s%s" % (redir_host, req.uri))
ex.add_custom_xml("Bucket", bucketName)
ex.add_custom_xml("Endpoint", redir_host)
raise ex
def grant(self, user_id, bucketName, objectName=None, perms="Rr"):
(o_perms, key) = self.getPerms(bucketName, objectName)
ndx = o_perms.find("W")
if ndx < 0:
raise cbException('AccessDenied')
if objectName == None:
f = self.getBucketPermFile(bucketName)
else:
f = self.getObjectFile(bucketName, objectName)
self.add_perms(f, user_id, perms)
pycb.log(logging.INFO, "granted: %s to %s" % (perms, user_id))
def check_permissions(self):
srcExists = self.user.exists(self.srcBucketName, self.srcObjectName)
if not srcExists:
raise cbException('NoSuchKey')
(perms, src_data_key) = self.user.get_perms(self.srcBucketName,
self.srcObjectName)
# make sure that we can read the source
ndx = perms.find("r")
if ndx < 0:
raise cbException('AccessDenied')
# make sure we can write to the destination
dstExists = self.user.exists(self.dstBucketName, self.dstObjectName)
(bperms, bdata_key) = self.user.get_perms(self.dstBucketName)
ndx = bperms.find("w")
if ndx < 0:
raise cbException('AccessDenied')
dst_data_key = None
dst_size = 0
if dstExists:
(perms,
dst_data_key) = self.user.get_perms(self.dstBucketName,
self.dstObjectName)
ndx = perms.find("w")
if ndx < 0:
raise cbException('AccessDenied')
(dst_size, ctm, md5) = self.user.get_info(self.dstBucketName,
self.dstObjectName)
(src_size, self.src_ctm,
self.src_md5) = self.user.get_info(self.srcBucketName,
self.srcObjectName)
# check the quota
remaining_quota = self.user.get_remaining_quota()
if remaining_quota != User.UNLIMITED:
if remaining_quota < src_size - dst_size:
pycb.log(
logging.INFO,
"user %s did not pass quota. file size %d quota %d" %
(self.user, src_size, remaining_quota))
raise cbException('AccountProblem')
# if we get to here we are allowed to do the copy
if dst_data_key == None:
self.dst_file = self.bucketIface.put_object(
self.dstBucketName, self.dstObjectName)
else:
self.dst_file = self.bucketIface.get_object(dst_data_key)
self.dst_file.set_delete_on_close(True)
self.src_file = self.bucketIface.get_object(src_data_key)
def __init__(self, request, user, requestId, bucketIface, srcBucket, srcObject, dstBucket, dstObject):
cbRequest.__init__(self, request, user, requestId, bucketIface)
ndx = dstObject.find("cumulus:/")
if ndx >= 0:
pycb.log(logging.ERROR, "someone tried to make a key named cumulus://... why would someone do that? %d" % (ndx))
raise cbException('InvalidURI')
self.dstBucketName = dstBucket
self.dstObjectName = dstObject
self.srcBucketName = srcBucket
self.srcObjectName = srcObject
def get_user(self, id):
usr_ath_filename = getPosixAuthDir() + id
if os.path.exists(usr_ath_filename) == 0:
ex = cbException('InvalidAccessKeyId')
pycb.log(logging.ERROR, "could not open: %s" % (usr_ath_filename))
raise ex
auth_file = open(usr_ath_filename, 'r')
key = auth_file.readline()
key = key.strip()
display_name = auth_file.readline()
display_name = display_name.strip()
auth_file.close()
return cbPosixUserObject(id, display_name, key)
def __init__(self):
self.done = False
self.cb = CBService()
self.site = CumulusSite(self.cb)
# figure out if we need http of https
if pycb.config.use_https:
pycb.log(logging.INFO, "using https")
sslContext = ssl.DefaultOpenSSLContextFactory(
pycb.config.https_key, pycb.config.https_cert)
self.iconnector = reactor.listenSSL(self.cb.get_port(), self.site,
sslContext)
else:
pycb.log(logging.INFO, "using http")
self.iconnector = reactor.listenTCP(self.cb.get_port(), self.site)
def put_bucket(self, bucketName, perms="RWrw"):
rc = 'InvalidBucketName'
rc_msg = "Failed to delete"
try:
f = self.getBucketPermFile(bucketName)
d = self.getBucketDir(bucketName)
os.mkdir(d)
self.setowner_perms(f, bucketName, perms)
pycb.log(logging.INFO, "created bucket %s" % (bucketName))
return d
except OSError, (OsEx):
if OsEx.errno == errno.EEXIST:
rc = 'BucketAlreadyExists'
elif OsEx.errno == errno.EINVAL:
rc = 'InvalidBucketName'
pycb.log(logging.ERROR, "%s %s %s" % (rc, d, str(OsEx)))
def close(self):
hashValue = self.get_md5()
if hashValue != None:
try:
mFile = open(self.metafname, 'w')
mFile.write(hashValue)
mFile.close()
except:
pass
try:
self.file.close()
except Exception, ex:
pycb.log(logging.WARNING,
"error closing data object %s %s" %
(self.fname, sys.exc_info()[0]),
tb=traceback)
def sendErrorResponse(self, request, requestId):
try:
request.setHeader('x-amz-request-id', str(requestId))
request.setHeader('x-amz-id-2:', str(uuid.uuid1()))
request.setResponseCode(self.httpCode, self.httpDesc)
# Create the minidom document
xml = self.make_xml_string(request.path, requestId)
request.write(xml)
request.finish()
return xml
except:
# XXX LOG ERROR
pycb.log(logging.ERROR, sys.exc_info()[0], traceback)
return cbException.panicError
def authorize(headers, message_type, path, uri):
sent_auth = headers['authorization']
auth_A = sent_auth.split(':')
auth_hash = auth_A[1]
id = auth_A[0].split()[1].strip()
user = pycb.config.auth.get_user(id)
key = user.get_password()
pycb.log(logging.INFO, "AUTHORIZING %s %s %s" % (message_type, path, headers))
b64_hmac = pycb.get_auth_hash(key, message_type, path, headers, uri)
if auth_hash == b64_hmac:
return user
pycb.log(logging.ERROR, "%s %s %s" % (key, b64_hmac, auth_hash))
ec = 'AccessDenied'
ex = cbException(ec)
raise ex
def get_next_host(self):
try:
hosts = []
f = open(self.host_file, "r")
for l in f.readlines():
hosts.append(l.strip())
f.close()
my_host = "%s:%d" % (pycb.config.hostname, pycb.config.port)
for i in range(0, 5):
ndx = random.randint(0, len(hosts)-1)
h = hosts[ndx]
pycb.log(logging.INFO, "redirector found %s, my host %s" % (h, my_host))
if h != my_host:
return h
return None
except Exception, ex:
pycb.log(logging.ERROR, "get next host error %s" % (str(ex)))
return None
def endGet(self, dataObj):
try:
eTag = dataObj.get_md5()
mSum = base64.encodestring(base64.b16decode(eTag.upper()))
self.checkMD5 = mSum
pycb.log(logging.INFO, "sent %s etag %s" % (self.objectName, self.checkMD5))
self.setHeader(self.request, 'ETag', '"%s"' % (eTag))
# now that we have the file set delete on close to false
# it will now be safe to deal with dropped connections
# without having large files left around
dataObj.set_delete_on_close(False)
# dataObj.close() do no need to close for now. twisted will
# do this for us
self.user.put_object(dataObj, self.bucketName, self.objectName)
self.grant_public_permissions(self.bucketName, self.objectName)
self.finish(self.request)
except cbException, (ex):
ex.sendErrorResponse(self.request, self.requestId)
def list_bucket(self, bucketIface, bucketName, args):
d = self.getBucketDir(bucketName)
if not os.path.isdir(d):
pycb.log(logging.ERROR, "did not find %s" % (d))
raise cbException('NoSuchBucket')
if 'prefix' in args:
p = args['prefix'][0]
p = self.hashFile(p)
prefix = d + "/" + p + "*"
else:
prefix = d + "/" + "*"
dir_list = glob.glob(prefix)
if 'max-keys' in args:
max_a = args['max-keys']
max = int(max_a[0])
else:
max = len(dir_list) + 1
results = []
for f in dir_list:
if len(results) >= max:
break
file = f.replace(d, "")
st = os.stat(f)
if stat.S_ISREG(st.st_mode):
# XXX just pick the size of the first one
dir_list = os.listdir(d)
if len(dir_list) > 0:
data_key = self.getDataKey(d + "/" + dir_list[0])
size = bucketIface.getSize(data_key)
tm = bucketIface.getModTime(data_key)
key = self.unhashFile(file)
display_name = key
obj = cbObject(tm, size, key, display_name, self)
results.append(obj)
return results
def get_my_buckets(self):
d = getPosixBuckerMetaDir()
dir_list = os.listdir(d)
results = []
for f in dir_list:
file = d + "/" + f
st = os.stat(file)
tm = time.gmtime(st.st_ctime)
size = -1
try:
if self.isOwner(f):
mds = bucketIFace.getMD5(data_key)
obj = cbObject(tm, size, f, f, self, md5sum=mds)
results.append(obj)
except:
traceback.print_exc(file=sys.stdout)
pycb.log(logging.WARNING,
"error checking ownership of %s" % (f),
tb=traceback)
return results
def get_perms(self, bucketName, objectName=None):
global authed_user
global public_user
try:
ufa = authed_user.get_uf(bucketName, objectName)
ufp = public_user.get_uf(bucketName, objectName)
p1 = ufa.get_perms(force=True)
p2 = ufp.get_perms(force=True)
gperms = merge_permissions(p1, p2)
except:
pycb.log(logging.ERROR,
"error getting global permissions %s" %
(sys.exc_info()[0]),
tb=traceback)
gperms = ""
try:
uf = self.get_uf(bucketName, objectName)
p = uf.get_perms(force=True)
p = merge_permissions(p, gperms)
return (p, uf.get_file().get_data_key())
finally:
self.db_obj.commit()
def work(self):
dataObj = self.request.content
exists = self.user.exists(self.bucketName, self.objectName)
if self.acl:
if not exists:
raise cbException('NoSuchKey')
(perms, data_key) = self.user.get_perms(self.bucketName, self.objectName)
ndx = perms.find("W")
if ndx < 0:
raise cbException('AccessDenied')
rc = self.grant_public_permissions(self.bucketName, self.objectName)
if not rc:
xml = self.request.content.read()
pycb.log(logging.ERROR, "acl xml %s" % (xml))
grants = parse_acl_request(xml)
for g in grants:
pycb.log(logging.INFO, "granting %s to %s" % (g[2], g[0]))
self.user.grant(g[0], self.bucketName, self.objectName, perms=g[2])
self.set_common_headers()
self.setHeader(self.request, 'Content-Length', 0)
self.setHeader(self.request, 'Connection', 'close')
self.setHeader(self.request, 'Location', "/" + self.bucketName)
self.setResponseCode(self.request, 200, 'OK')
self.finish(self.request)
else:
(bperms, bdata_key) = self.user.get_perms(self.bucketName)
ndx = bperms.find("w")
if ndx < 0:
raise cbException('AccessDenied')
file_size = 0
if exists:
(perms, data_key) = self.user.get_perms(self.bucketName, self.objectName)
ndx = perms.find("w")
if ndx < 0:
raise cbException('AccessDenied')
# make sure they can write to the bucket
(perms, data_key) = self.user.get_perms(self.bucketName)
ndx = perms.find("w")
if ndx < 0:
raise cbException('AccessDenied')
(file_size, ctm, md5) = self.user.get_info(self.bucketName, self.objectName)
# gotta decide quota, if existed should get credit for the
# existing size
remaining_quota = self.user.get_remaining_quota()
if remaining_quota != User.UNLIMITED:
new_file_len = int(self.request.getHeader('content-length'))
if remaining_quota + file_size < new_file_len:
pycb.log(logging.INFO, "user %s did not pass quota. file size %d quota %d" % (self.user, new_file_len, remaining_quota))
raise cbException('AccountProblem')
obj = self.request.content
self.recvObject(self.request, obj)
def request_object_factory(self, request, user, path, requestId):
pycb.log(logging.INFO, "path %s" % (path))
# handle the one service operation
if path == "/":
if request.method == 'GET':
cbR = cbGetService(request, user, requestId,
pycb.config.bucket)
return cbR
raise cbException('InvalidArgument')
(bucketName, objectName) = path_to_bucket_object(path)
init_redirector(request, bucketName, objectName)
pycb.log(
logging.INFO, "path %s bucket %s object %s" %
(path, bucketName, str(objectName)))
if request.method == 'GET':
if objectName == None:
cbR = cbGetBucket(request, user, bucketName, requestId,
pycb.config.bucket)
else:
cbR = cbGetObject(request, user, bucketName, objectName,
requestId, pycb.config.bucket)
return cbR
elif request.method == 'PUT':
if objectName == None:
cbR = cbPutBucket(request, user, bucketName, requestId,
pycb.config.bucket)
else:
args = request.getAllHeaders()
if 'x-amz-copy-source' in args:
(srcBucketName, srcObjectName) = path_to_bucket_object(
args['x-amz-copy-source'])
cbR = cbCopyObject(request, user, requestId,
pycb.config.bucket, srcBucketName,
srcObjectName, bucketName, objectName)
else:
cbR = cbPutObject(request, user, bucketName, objectName,
requestId, pycb.config.bucket)
return cbR
elif request.method == 'POST':
pycb.log(logging.ERROR, "Nothing to handle POST")
elif request.method == 'DELETE':
if objectName == None:
cbR = cbDeleteBucket(request, user, bucketName, requestId,
pycb.config.bucket)
else:
cbR = cbDeleteObject(request, user, bucketName, objectName,
requestId, pycb.config.bucket)
return cbR
elif request.method == 'HEAD' and objectName != None:
cbR = cbHeadObject(request, user, bucketName, objectName,
requestId, pycb.config.bucket)
return cbR
raise cbException('InvalidArgument')
class CBService(resource.Resource):
isLeaf = True
def __init__(self):
pass
def get_port(self):
return pycb.config.port
# amazon uses some smaller num generator, might have to match theres
# for clients that make assumptions
def next_request_id(self):
return str(uuid.uuid1()).replace("-", "")
# figure out if the operation is targeted at a service, bucket, or
# object
def request_object_factory(self, request, user, path, requestId):
pycb.log(logging.INFO, "path %s" % (path))
# handle the one service operation
if path == "/":
if request.method == 'GET':
cbR = cbGetService(request, user, requestId, pycb.config.bucket)
return cbR
raise cbException('InvalidArgument')
(bucketName, objectName) = path_to_bucket_object(path)
init_redirector(request, bucketName, objectName)
pycb.log(logging.INFO, "path %s bucket %s object %s" % (path, bucketName, str(objectName)))
if request.method == 'GET':
if objectName == None:
cbR = cbGetBucket(request, user, bucketName, requestId, pycb.config.bucket)
else:
cbR = cbGetObject(request, user, bucketName, objectName, requestId, pycb.config.bucket)
return cbR
elif request.method == 'PUT':
if objectName == None:
cbR = cbPutBucket(request, user, bucketName, requestId, pycb.config.bucket)
else:
args = request.getAllHeaders()
if 'x-amz-copy-source' in args:
(srcBucketName, srcObjectName) = path_to_bucket_object(args['x-amz-copy-source'])
cbR = cbCopyObject(request, user, requestId, pycb.config.bucket, srcBucketName, srcObjectName, bucketName, objectName)
else:
cbR = cbPutObject(request, user, bucketName, objectName, requestId, pycb.config.bucket)
return cbR
elif request.method == 'POST':
pycb.log(logging.ERROR, "Nothing to handle POST")
elif request.method == 'DELETE':
if objectName == None:
cbR = cbDeleteBucket(request, user, bucketName, requestId, pycb.config.bucket)
else:
cbR = cbDeleteObject(request, user, bucketName, objectName, requestId, pycb.config.bucket)
return cbR
elif request.method == 'HEAD' and objectName != None:
cbR = cbHeadObject(request, user, bucketName, objectName, requestId, pycb.config.bucket)
return cbR
raise cbException('InvalidArgument')
# everything does through here to localize access control
def process_event(self, request):
try:
rPath = createPath(request.getAllHeaders(), request.path)
requestId = self.next_request_id()
pycb.log(logging.INFO, "%s %s Incoming" % (requestId, str(datetime.now())))
pycb.log(logging.INFO, "%s %s" % (requestId, str(request)))
pycb.log(logging.INFO, "%s %s" % (requestId, rPath))
pycb.log(logging.INFO, "%s %s" %(requestId, request.getAllHeaders()))
pycb.log(logging.INFO, "request URI %s method %s %s" %(request.uri, request.method, str(request.args)))
user = authorize(request.getAllHeaders(), request.method, rPath, request.uri)
self.allowed_event(request, user, requestId, rPath)
except cbException, ex:
eMsg = ex.sendErrorResponse(request, requestId)
pycb.log(logging.ERROR, eMsg, traceback)
except Exception, ex2:
traceback.print_exc(file=sys.stdout)
gdEx = cbException('InternalError')
eMsg = gdEx.sendErrorResponse(request, requestId)
pycb.log(logging.ERROR, eMsg, traceback)
def process_event(self, request):
try:
rPath = createPath(request.getAllHeaders(), request.path)
requestId = self.next_request_id()
pycb.log(logging.INFO, "%s %s Incoming" % (requestId, str(datetime.now())))
pycb.log(logging.INFO, "%s %s" % (requestId, str(request)))
pycb.log(logging.INFO, "%s %s" % (requestId, rPath))
pycb.log(logging.INFO, "%s %s" %(requestId, request.getAllHeaders()))
pycb.log(logging.INFO, "request URI %s method %s %s" %(request.uri, request.method, str(request.args)))
user = authorize(request.getAllHeaders(), request.method, rPath, request.uri)
self.allowed_event(request, user, requestId, rPath)
except cbException, ex:
eMsg = ex.sendErrorResponse(request, requestId)
pycb.log(logging.ERROR, eMsg, traceback)
return self.iconnector.getHost()
def get_port(self):
l = self.getListener()
return l.port
def run(self):
reactor.suggestThreadPoolSize(10)
reactor.run()
def stop(self):
self.iconnector.stopListening()
def main(argv=sys.argv[0:]):
pycb.config.parse_cmdline(argv)
try:
cumulus = CumulusRunner()
except Exception, ex:
pycb.log(logging.ERROR, "error starting the server, check that the port is not already taken: %s" % (str(ex)), tb=traceback)
raise ex
pycb.log(logging.INFO, "listening at %s" % (str(cumulus.getListener())))
cumulus.run()
return 0
if __name__ == "__main__":
rc = main()
sys.exit(rc)
