def destroyObject(bucket, key):
'''destroy's object'''
conn = Connection()
try:
#Validate the bucket
_verifyBucket(conn, bucket, True)
#Check for object and get information from database
query = "SELECT hashfield FROM object WHERE bucket = %s AND object = %s"
result = conn.executeStatement(query, (escape_string(str(bucket)), escape_string(str(key))))
if len(result) == 0:
raise NotFoundException.NoSuchKeyException(bucket, key)
#Delete the object from the database and the filesystem
query = "DELETE FROM object_metadata WHERE bucket = %s AND object = %s"
conn.executeStatement(query, (escape_string(str(bucket)), escape_string(str(key))))
query = "DELETE FROM object WHERE bucket = %s AND object = %s"
conn.executeStatement(query, (escape_string(str(bucket)), escape_string(str(key))))
except:
conn.cancelAndClose()
raise
conn.close()
hashString = result[0][0]
path = Config.get('common','filesystem_path')
path += str(bucket)
path += "/"+hashString[0:3]+"/"+hashString[3:6]+"/"+hashString[6:9]
os.remove(path+"/"+hashString)
try:
os.removedirs(path)
except OSError, e:
if e.errno != errno.ENOTEMPTY:
raise
def checkUserPermission(user, bucket, action):
'''checks if a user is permitted to perform action on bucket'''
if action in ('write_log_status', 'read_log_status', 'destroy'):
if not user:
return False
else:
conn = Connection()
try:
result = conn.executeStatement('SELECT userid from bucket where bucket = %s', (bucket,))
finally:
conn.close()
if len(result) == 0:
raise NotFoundException.NoSuchBucketException(bucket)
else:
return result[0][0] == user
elif action in ('read', 'write', 'read_acp', 'write_acp'):
conn = Connection()
try:
if user:
result = conn.executeStatement('''SELECT (SELECT COUNT(*) FROM bucket WHERE bucket = %s) +
(SELECT COUNT(*) FROM bucket_permission WHERE userid IN(2, %s) and bucket = %s and permission IN(%s, "full_control"))''', (bucket, user, bucket, action))
else:
result = conn.executeStatement('''SELECT (SELECT COUNT(*) FROM bucket WHERE bucket = %s) +
(SELECT COUNT(*) FROM bucket_permission WHERE userid = 1 and bucket = %s and permission IN(%s, 'full_control'))''', (bucket, bucket, action))
finally:
conn.close()
if result[0][0] == 0:
raise NotFoundException.NoSuchBucketException(bucket)
else:
return result[0][0] > 1
else:
raise InternalErrorException.BadArgumentException('action', str(action),
'Invalid action for BucketACP.checkUserPermission: action must be IN ("write_log_status", "read_log_status", "destroy", "write", "read", "write_acp", "read_acp").')
def setBucket(bucket, userid):
'''creates a new empty bucket'''
MAX_BUCKETS_PER_USER = 100
conn = Connection()
#Validate the bucket
try:
_verifyBucket(conn, bucket, False, userid)
#Check if user has too many buckets
query = "SELECT bucket FROM bucket WHERE userid = %s"
result = conn.executeStatement(query, (int(userid)))
if len(result) >= MAX_BUCKETS_PER_USER:
raise BadRequestException.TooManyBucketsException()
#Write bucket to database and filesystem
query = "INSERT INTO bucket (bucket, userid, bucket_creation_time) VALUES (%s, %s, NOW())"
conn.executeStatement(query, (escape_string(str(bucket)), int(userid)))
path = Config.get('common','filesystem_path')
path += str(bucket)
os.mkdir(path)
except:
conn.cancelAndClose()
raise
else:
conn.close()
def setService(username, isAdmin=False):
'''adds new user'''
conn = Connection()
try:
import random
import hashlib
access = hashlib.sha1()
secret = hashlib.sha1()
access.update(username + str(random.getrandbits(16)))
secret.update(str(random.getrandbits(16)) + username)
accessHexDigest = access.hexdigest()
secretHexDigest = secret.hexdigest()
success = False
for i in range(3):
try:
conn.executeStatement('insert into user(username, accesskey, secretkey, isAdmin) values(%s, %s, %s, %s)', (username, accessHexDigest, secretHexDigest, bool(isAdmin)))
except InternalErrorException.DatabaseIntegrityErrorException:
access.update(str(random.getrandbits(16)))
secret.update(str(random.getrandbits(16)))
accessHexDigest = access.hexdigest()
secretHexDigest = secret.hexdigest()
else:
success = True
break
if not success:
raise InternalErrorException.KeyCollisionErrorException()
except:
conn.cancelAndClose()
raise
else:
conn.close()
return (accessHexDigest, secretHexDigest)
def checkUserPermission(user, bucket, key, action):
if action in ('read', 'read_acp', 'write_acp'):
conn = Connection()
try:
if user:
result = conn.executeStatement(
'''SELECT (SELECT COUNT(*) from object WHERE bucket = %s and object = %s) +
(SELECT COUNT(*) from object_permission
WHERE bucket = %s and object = %s and userid = %s and permission IN(%s, 'full_control'))
''', (bucket, key, bucket, key, user, action))
else:
result = conn.executeStatement(
'''SELECT (SELECT COUNT(*) FROM object WHERE bucket = %s and object = %s) +
(SELECT COUNT(*) FROM object_permission WHERE bucket = %s and object = %s and userid = 1 and permission IN(%s, 'full_control'))
''', (bucket, key, bucket, key, action))
finally:
conn.close()
if result[0][0] == 0:
raise NotFoundException.NoSuchKeyException(bucket, key)
else:
return result[0][0] > 1
elif action in('write'):
conn = Connection()
try:
if user:
result = conn.executeStatement('''SELECT (SELECT COUNT(*) FROM bucket WHERE bucket = %s) +
(SELECT COUNT(*) FROM bucket_permission where userid IN(2, %s) and bucket = %s and permission IN('write', 'full_control')) +
(SELECT COUNT(*) FROM object_permission where userid IN(2, %s) and bucket = %s and object = %s and permission IN('write', 'full_control'))
''', (bucket, user, bucket, user, bucket, key))
else:
result = conn.executeStatement('''SELECT (SELECT COUNT(*) FROM bucket WHERE bucket = %s) +
(SELECT COUNT(*) FROM bucket_permission where userid = 1 and bucket = %s and permission IN('write', 'full_control')) +
(SELECT COUNT(*) FROM object_permission where userid = 1 and bucket = %s and object = %s and permission IN('write', 'full_control'))
''', (bucket, bucket, bucket, key))
finally:
conn.close()
if result[0][0] == 0:
raise NotFoundException.NoSuchBucketException(bucket)
return result[0][0] > 1
else:
raise InternalErrorException.BadArgumentException('action', str(action),
'Invalid action for ObjectACP.checkUserPermission: action must be IN ("write", "read", "write_acp", "read_acp").')
def getUser(signature, accessKey, stringToSign):
conn = Connection(useDictCursor = True)
try:
rs = conn.executeStatement('select userid, secretKey, isAdmin from user where accessKey = %s', (accessKey,))
finally:
conn.close()
if len(rs) == 0:
raise ForbiddenException.InvalidAccessKeyIdException(accessKey)
user = rs[0]['userid']
pkey = rs[0]['secretKey']
isAdmin = rs[0]['isAdmin']
computedSig = __computeBase64Signature(pkey, stringToSign)
mode = Config.get('common', 'mode')
if computedSig == signature or (mode == 'debug' and signature == 'free'):
return user, isAdmin, computedSig
else:
stringToSignByteSeq = []
for c in stringToSign:
stringToSignByteSeq.append(c.encode('hex'))
raise ForbiddenException.SignatureDoesNotMatchException(' '.join(stringToSignByteSeq), stringToSign, signature, accessKey)
def getBucketACP(bucket):
'''returns dictionary representation of the bucket's access control policy'''
conn = Connection(useDictCursor = True)
try:
rs = conn.executeStatement('''SELECT userid, username, 'owner' as permission
FROM user JOIN bucket USING(userid) WHERE bucket = %s
UNION
SELECT userid, username, permission
FROM bucket_permission JOIN user USING(userid)
WHERE bucket = %s''', (bucket, bucket))
finally:
conn.close()
acp = {}
if len(rs) > 0:
acp['owner'] = {'userid':rs[0]['userid'], 'username':rs[0]['username']}
acp['acl'] = []
for grant in rs[1:]:
acp['acl'].append({'grantee':{'userid':grant['userid'], 'username':grant['username']}, 'permission':grant['permission']})
return acp
else:
raise NotFoundException.NoSuchBucketException(bucket)
def getObjectACP(bucket, key):
conn = Connection(useDictCursor = True)
try:
rs = conn.executeStatement(
'''SELECT userid, username, 'owner' as permission
FROM user JOIN object USING(userid)
WHERE object = %s and bucket = %s
UNION
SELECT userid, username, permission
FROM object_permission JOIN user USING(userid)
WHERE object = %s and bucket = %s
''', (key, bucket, key, bucket))
finally:
conn.close()
acp = {}
if len(rs) > 0:
acp['owner'] = {'userid':rs[0]['userid'], 'username':rs[0]['username']}
acp['acl'] = []
for grant in rs[1:]:
acp['acl'].append({'grantee':{'userid':grant['userid'], 'username':grant['username']}, 'permission':grant['permission']})
else:
raise NotFoundException.NoSuchKeyException(bucket, key)
return acp
def getService(userid):
'''returns list of buckets owned by user'''
conn = Connection(True)
try:
result = conn.executeStatement("SELECT bucket, bucket_creation_time, username FROM bucket RIGHT JOIN user USING(userid) WHERE userid = %s", (userid,))
except:
conn.cancelAndClose()
raise
else:
conn.close()
buckets = []
for bucket in result:
if bucket['bucket'] != None:
buckets.append({'bucketName':bucket['bucket'], 'creationDate':((bucket['bucket_creation_time']).isoformat('T') + 'Z')})
return {'user':{'userid':userid,'username':result[0]['username']},'buckets':buckets}
def destroyBucket(bucket):
'''destroys a bucket if empty'''
conn = Connection()
try:
#Validate the bucket
_verifyBucket(conn, bucket, True)
#Check if the bucket is empty
query = "SELECT COUNT(*) FROM object WHERE bucket = %s"
result = conn.executeStatement(query, (escape_string(str(bucket))))
if result[0][0] > 0:
raise ConflictException.BucketNotEmptyException(bucket)
#Delete the bucket from the database and the filesystem
query = "DELETE FROM bucket WHERE bucket = %s"
conn.executeStatement(query, (escape_string(str(bucket))))
path = Config.get('common','filesystem_path')
path += str(bucket)
os.rmdir(path)
except:
conn.cancelAndClose()
raise
else:
conn.close()
def setBucketACP(bucket, accessControlPolicy):
'''resets a bucket's acp to the passed parameter'''
conn = Connection()
try:
removeString = 'delete from bucket_permission where bucket = %s'
insertString = 'insert into bucket_permission (userid, bucket, permission) VALUES '
aclWildcardList = []
aclValueList = []
for entry in accessControlPolicy['acl']:
aclWildcardList.append('(%s, %s, %s)')
aclValueList.append(entry['grantee']['userid'])
aclValueList.append(bucket)
aclValueList.append(entry['permission'])
insertString += ', '.join(aclWildcardList)
removeRS = conn.executeStatement(removeString, (bucket,))
insertRS = conn.executeStatement(insertString, aclValueList)
except:
conn.cancelAndClose()
raise
else:
conn.close()
def setObjectACP(bucket, key, accessControlPolicy):
conn = Connection()
try:
removeString = 'delete from object_permission where bucket = %s and object = %s'
insertString = 'insert into object_permission (userid, bucket, object, permission) VALUES'
aclWildcardList = []
aclValueList = []
for entry in accessControlPolicy['acl']:
aclWildcardList.append('(%s, %s, %s, %s)')
aclValueList.append(entry['grantee']['userid'])
aclValueList.append(bucket)
aclValueList.append(key)
aclValueList.append(entry['permission'])
insertString += ', '.join(aclWildcardList)
removeRS = conn.executeStatement(removeString, (bucket, key))
if len(accessControlPolicy) > 0:
insertRS = conn.executeStatement(insertString, aclValueList)
except:
conn.cancelAndClose()
raise
conn.close()
def getObject(bucket, key, getMetadata, getData, byteRangeStart = None, byteRangeEnd = None, ifMatch = None, ifNotMatch = None, ifModifiedSince = None, ifNotModifiedSince = None, ifRange = None):
'''returns object'''
conn = Connection()
try:
#Validate the bucket
_verifyBucket(conn, bucket, True)
#Check for object and get information from database
query = "SELECT o.object, o.bucket, o.hashfield, o.object_create_time, o.eTag, o.object_mod_time, o.size, o.content_type, o.content_encoding, o.content_disposition, o.userid, u.username FROM object as o, user as u WHERE o.bucket = %s AND o.object = %s AND o.userid = u.userid"
result = conn.executeStatement(query, (escape_string(str(bucket)), escape_string(str(key))))
if len(result) == 0:
raise NotFoundException.NoSuchKeyException(bucket, key)
result = result[0]
#if _passPrecondition(str(result[4]), str(result[5]), str(ifMatch), str(ifNotMatch), str(ifModifiedSince), str(ifNotModifiedSince), str(ifRange)) == False:
# byteRangeStart = None
# byteRangeEnd = None
#Get metadata from database
query = "SELECT type, value FROM object_metadata WHERE bucket = %s AND object = %s"
metadata = conn.executeStatement(query, (escape_string(str(bucket)), escape_string(str(key))))
except:
conn.cancelAndClose()
raise
else:
conn.close()
metadataDict = {}
for tag in metadata:
metadataDict[str(tag[0])] = unicode(tag[1], encoding='utf8')
content_range = {}
size = 0
hashfield = str(result[2])
if getData:
#Get data from filesystem and build content_range
path = Config.get('common','filesystem_path')
path += str(bucket)
path += "/"+hashfield[0:3]+"/"+hashfield[3:6]+"/"+hashfield[6:9]+"/"+hashfield
fileReader = open(path, 'rb')
try:
data = ""
if byteRangeStart != None and byteRangeStart > 0:
fileReader.seek(byteRangeStart)
content_range['start'] = byteRangeStart
if byteRangeEnd != None and byteRangeEnd > byteRangeStart:
data = fileReader.read(byteRangeEnd-byteRangeStart)
content_range['end'] = fileReader.tell()
fileReader.read()
content_range['total'] = fileReader.tell()
size = byteRangeEnd-byteRangeStart
else:
data = fileReader.read()
content_range['end'] = fileReader.tell()
content_range['total'] = fileReader.tell()
size = content_range['total']
else:
if byteRangeEnd != None:
content_range['start'] = 0
data = fileReader.read(byteRangeEnd)
content_range['end'] = fileReader.tell()
fileReader.read()
content_range['total'] = fileReader.tell()
size = byteRangeEnd
else:
data = fileReader.read()
size = fileReader.tell()
finally:
fileReader.close()
#print data
if content_range.has_key('start'):
content_range['string'] = str(content_range['start'])+"-"+str(content_range['end'])+"/"+str(content_range['total'])
returnDict = {'key':str(result[0]),
'bucket':str(result[1]),
'hash':hashfield,
'creationTime':((result[3]).isoformat('T') + 'Z'),
'eTag':str(result[4]),
'lastModified':((result[5]).isoformat('T') + 'Z'),
'size':size,
'content-type':str(result[7]),
'owner':{'id':int(result[10]),
'name':unicode(result[11], encoding='utf8')}}
if str(result[8]) != "" and result[8] != None:
returnDict['content-encoding'] = str(result[8])
if str(result[9]) != "" and result[9] != None:
returnDict['content-disposition'] = str(result[9])
if content_range.has_key('string'):
returnDict['content-range'] = content_range['string']
if getMetadata:
returnDict['metadata'] = metadataDict
if getData:
returnDict['data'] = data
return returnDict
def setObject(userid, bucket, key, metadata, data, content_md5 = None, content_type = None, content_disposition = None, content_encoding = None):
'''setObject'''
if not userid:
userid = 1
hashString = None
conn = Connection()
try:
#Validate the bucket
_verifyBucket(conn, bucket, userid, True)
#Check for object and get information from database
calculatedMD5 = md5.new(data)
calculatedMD5HexDigest = calculatedMD5.hexdigest()
if content_md5 != None and content_md5 != calculatedMD5HexDigest:
raise BadRequestException.BadDigestException(content_md5, calculatedMD5HexDigest)
#Generate hashfield
hashfield = hashlib.sha1()
hashfield.update(key)
hashfieldHexDigest = ''
success = False
query = "SELECT COUNT(*) FROM object WHERE hashfield = %s"
attemptedHashfieldList = []
for i in range(3):
hashfield.update(str(time.time()))
hashfieldHexDigest = hashfield.hexdigest()
attemptedHashfieldList.append(str(hashfieldHexDigest))
count = conn.executeStatement(query, (str(hashfieldHexDigest)))[0][0]
if count == 0:
success = True
break
if success == False:
raise InternalErrorException.HashfieldCollisionErrorException(attemptedHashfieldList)
#Get size of file
size = len(data)
if content_type == None:
content_type = "binary/octet-stream"
if content_encoding == None:
content_encoding = ""
if content_disposition == None:
content_disposition = ""
#Build metadata query
metadataQuery = ""
if metadata != None and metadata != {}:
metadataQuery = "INSERT INTO object_metadata (bucket, object, type, value) VALUES ("+"'"
for tag, value in metadata.iteritems():
if type(value) == str or type(value) == unicode:
value = value.encode('utf8')
else:
value = str(value)
metadataQuery += escape_string(str(bucket))+"', '"+escape_string(str(key))+"', '"+escape_string(tag)+"', '"+escape_string(value)+"'), ('"
metadataQuery = metadataQuery[0:-4]
#Write to database and filesystem
result = conn.executeStatement("SELECT hashfield FROM object WHERE bucket = %s AND object = %s", (escape_string(str(bucket)), escape_string(str(key))))
if len(result) > 0:
hashString = result[0][0]
path = Config.get('common','filesystem_path')
path += str(bucket)
path += "/"+hashString[0:3]+"/"+hashString[3:6]+"/"+hashString[6:9]
os.remove(path+"/"+hashString)
try:
os.removedirs(path)
except OSError, e:
if e.errno != errno.ENOTEMPTY:
raise
hashString = str(hashfieldHexDigest)
query = "UPDATE object SET userid = %s, hashfield = %s, eTag = %s, object_mod_time = NOW(), size = %s, content_type = %s, content_encoding = %s, content_disposition = %s WHERE bucket = %s AND object = %s"
conn.executeStatement(query, (int(userid), hashString, str(calculatedMD5HexDigest), int(size), escape_string(str(content_type)), escape_string(str(content_encoding)), escape_string(str(content_disposition)), escape_string(str(bucket)), escape_string(str(key))))
conn.executeStatement("DELETE FROM object_metadata WHERE bucket = %s AND object = %s", (escape_string(str(bucket)), escape_string(str(key))))
else:
def destroyService(userid):
'''deletes an owner'''
if not userid or userid <3:
raise BadRequestExceptin.UseridNotValidException(userid)
conn = Connection()
try:
#Check if user exists
checkUserResult = conn.executeStatement('select count(*) from user where userid = %s', (userid,))
if checkUserResult[0][0] == 0:
raise BadRequestException.UseridNotFoundException(userid)
#Give ownership of existing objects to enclosing bucket owners
conn.executeStatement('update object, bucket set object.userid = bucket.userid where object.bucket = bucket.bucket and object.userid = %s', (userid,))
#Empty existing buckets
conn.executeStatement('delete from object where userid = %s', (userid,))
#Delete buckets
conn.executeStatement('delete from bucket where userid = %s', (userid,))
#Delete user
conn.executeStatement('delete from user where userid = %s', (userid,))
except:
conn.cancelAndClose()
raise
conn.close()
def getBucket(bucket, prefix, marker, maxKeys, delimiter):
'''returns listing of objects inside a bucket'''
conn = Connection()
try:
#Validate the bucket
_verifyBucket(conn, bucket, True)
#get objects
group = False
if prefix != None:
if delimiter != None and delimiter != "":
delimiter = escape_string(str(delimiter))
count = prefix.count(delimiter) + 1
queryGroup = " GROUP BY SUBSTRING_INDEX(o.object, '"+delimiter+"', "+str(count)+")"
group = True
query = "SELECT o.userid, o.object, o.bucket, o.object_create_time, o.eTag, o.object_mod_time, o.size, u.username, COUNT(*), CONCAT(SUBSTRING_INDEX(o.object, '"+delimiter+"', "+str(count)+"), '"+delimiter+"') FROM object as o, user as u WHERE o.bucket = %s AND o.userid = u.userid"
else:
query = "SELECT o.userid, o.object, o.bucket, o.object_create_time, o.eTag, o.object_mod_time, o.size, u.username, 1 FROM object as o, user as u WHERE o.bucket = %s AND o.userid = u.userid"
prefix = escape_string(str(prefix))
prefix.replace('%','%%')
prefix += '%'
query += " AND o.object LIKE %s"
else:
query = "SELECT o.userid, o.object, o.bucket, o.object_create_time, o.eTag, o.object_mod_time, o.size, u.username, 1 FROM object as o, user as u WHERE o.bucket = %s AND o.userid = u.userid"
if marker != None:
marker = escape_string(str(marker))
query += " AND STRCMP(o.object, '"+marker+"') > 0"
if group == True:
query += queryGroup
else:
query += " ORDER BY o.object"
if maxKeys and int(maxKeys) > -1:
query += " LIMIT "+str(int(maxKeys))
if prefix != None:
print (query % ("'%s'", "'%s'")) % (escape_string(str(bucket)), prefix)
result = conn.executeStatement(query, (escape_string(str(bucket)), prefix))
else:
print (query % ("'%s'")) % (escape_string(str(bucket)))
result = conn.executeStatement(query, (escape_string(str(bucket))))
contents = []
commonPrefixes = []
for row in result:
if int(row[8]) == 1:
contents.append({'key':str(row[1]),
'lastModified':((row[5]).isoformat('T') + 'Z'),
'eTag':str(row[4]),
'size':int(row[6]),
'storageClass':'STANDARD',
'owner':{'id':int(row[0]),
'name':unicode(row[7], encoding='utf8')}})
else:
commonPrefixes.append(str(row[9]))
query = "SELECT COUNT(*) FROM object WHERE bucket = %s"
count = conn.executeStatement(query, (escape_string(str(bucket))))[0][0]
if count > len(contents):
isTruncated = True
else:
isTruncated = False
except:
conn.cancelAndClose()
raise
conn.close()
return (contents, commonPrefixes, isTruncated)