def __init__(self, req, virtualBucket=False):
self.req = req
self.bucket = self.key = self.user = self.accesskey = self.signature = self.stringToSign = self.computedSig = None
self.isUserAdmin = False
self.subresources = {}
self.__writeBuffer = ''
self.virtualBucket = False
#Query string digest
if self.req.args:
self.subresources = util.parse_qs(self.req.args, True)
#URI digest
basehost = Config.get('server', 'hostname')
if self.req.hostname == basehost:
uriDigestResults = self.uriDigest(req.uri)
self.bucket = uriDigestResults.get('bucket')
self.key = uriDigestResults.get('key')
else:
splitHost = self.req.hostname.split("." + basehost)
if len(splitHost) == 2:
uriDigestResults = self.uriDigest(splitHost[0] + '/' + req.uri)
self.bucket = uriDigestResults.get('bucket')
self.key = uriDigestResults.get('key')
self.virtualBucket = True
else:
self.req.write("HOST: " + self.req.hostname + "\r\n")
raise Exception, 'wrong hostname?'
#custom header table
self.customHeaderPrefix = Config.get('common', 'customHeaderPrefix').lower()
self.customHeaderTable = {}
for tag, val in self.req.headers_in.iteritems():
if tag.lower().startswith(self.customHeaderPrefix):
self.customHeaderTable[tag.lower()[len(self.customHeaderPrefix):]] = val
#authenticate -- must happen after custom header table is created
self.accesskey, self.signature = self.__getAccessKeyAndSignature()
if self.accesskey:
self.stringToSign = self.__buildStringToSign()
self.user, self.isUserAdmin, self.computedSig = getUser(self.signature, self.accesskey, self.stringToSign)
#Check date
#check customDateHeader then date header
if 'signature' in self.subresources:
self.req.headers_out['Signature'] = str(self.computedSig)
self.write(str(self.computedSig) + "\r\n")
self.write(str(self.stringToSign) + "\r\n")
self.send()
def destroyObject(bucket, key):
'''destroy's object'''
conn = Connection()
try:
#Validate the bucket
_verifyBucket(conn, bucket, True)
#Check for object and get information from database
query = "SELECT hashfield FROM object WHERE bucket = %s AND object = %s"
result = conn.executeStatement(query, (escape_string(str(bucket)), escape_string(str(key))))
if len(result) == 0:
raise NotFoundException.NoSuchKeyException(bucket, key)
#Delete the object from the database and the filesystem
query = "DELETE FROM object_metadata WHERE bucket = %s AND object = %s"
conn.executeStatement(query, (escape_string(str(bucket)), escape_string(str(key))))
query = "DELETE FROM object WHERE bucket = %s AND object = %s"
conn.executeStatement(query, (escape_string(str(bucket)), escape_string(str(key))))
except:
conn.cancelAndClose()
raise
conn.close()
hashString = result[0][0]
path = Config.get('common','filesystem_path')
path += str(bucket)
path += "/"+hashString[0:3]+"/"+hashString[3:6]+"/"+hashString[6:9]
os.remove(path+"/"+hashString)
try:
os.removedirs(path)
except OSError, e:
if e.errno != errno.ENOTEMPTY:
raise
def setBucket(bucket, userid):
'''creates a new empty bucket'''
MAX_BUCKETS_PER_USER = 100
conn = Connection()
#Validate the bucket
try:
_verifyBucket(conn, bucket, False, userid)
#Check if user has too many buckets
query = "SELECT bucket FROM bucket WHERE userid = %s"
result = conn.executeStatement(query, (int(userid)))
if len(result) >= MAX_BUCKETS_PER_USER:
raise BadRequestException.TooManyBucketsException()
#Write bucket to database and filesystem
query = "INSERT INTO bucket (bucket, userid, bucket_creation_time) VALUES (%s, %s, NOW())"
conn.executeStatement(query, (escape_string(str(bucket)), int(userid)))
path = Config.get('common','filesystem_path')
path += str(bucket)
os.mkdir(path)
except:
conn.cancelAndClose()
raise
else:
conn.close()
def __init__(self, stringToSignBytes, stringToSign, signatureProvided, accessKey):
accesskeyIdPrefix = str(Config.get('authentication', 'prefix'))
ForbiddenException.__init__(self,
{'Message' : 'The request signature we calculated des not match the signature you provided. Check your key and signing method.',
'Code' : 'SignatureDoesNotMatch',
'StringToSignBytes' : str(stringToSignBytes),
'StringToSign' : str(stringToSign),
'SignatureProvided' : str(signatureProvided),
accesskeyIdPrefix + 'AccessKeyId' : str(accessKey)})
def __getAccessKeyAndSignature(self):
header = Config.get('authentication', 'header')
prefix = Config.get('authentication', 'prefix') + ' '
accesskey = signature = None
try:
authString = self.req.headers_in[header]
except KeyError:
pass
else:
splitAuth = authString.split(prefix)
if len(splitAuth) == 2 and len(splitAuth[0]) == 0 and not splitAuth[1].startswith(' '):
try:
accesskey, signature = splitAuth[1].split(':')
except ValueError:
raise BadRequestException.InvalidArgumentAuthorizationException(argValue = authString)
elif not len(splitAuth) == 2:
raise BadRequestException.InvalidArgumentAuthorizationException(argValue = authString)
else:
raise BadRequestException.InvalidArgumentAuthorizationSpacingException(argValue = authString)
return accesskey, signature
def __init__(self, useDictCursor = False):
host = Config.get('database','mysql_utaka_hostname')
user = Config.get('database','mysql_utaka_username')
passwd = Config.get('database','mysql_utaka_password')
db = Config.get('database','mysql_utaka_database')
self.conn = MySQLdb.connect(host = host, user = user, passwd = passwd, db = db)
self.connectTime = datetime.datetime.today()
if useDictCursor:
self._usingDictCursor = True
self.cursor = self.conn.cursor(MySQLdb.cursors.DictCursor)
else:
self._usingDictCursor = False
self.cursor = self.conn.cursor()
utakaLog = open('/var/www/html/utaka/utakaLog', 'a')
try:
if self._usingDictCursor:
utakaLog.write('Dictionary Database Connection Made\r\n')
else:
utakaLog.write('Regular Database Connection Made\r\n')
finally:
utakaLog.close()
def __buildStringToSign(self):
nl = '\n'
#http headers
methodString = self.req.method
contentMd5String = self.req.headers_in.get('content-md5', '')
contentTypeString = self.req.headers_in.get('content-type', '')
dateString = self.req.headers_in.get('date', '')
#Canonicalize Custom Headers
__customHeaderPrefix = Config.get('common', 'customHeaderPrefix').lower()
__customDateHeader = __customHeaderPrefix + "-date"
customHeaderList = []
canCustomHeaders = ''
for tag, val in self.customHeaderTable.iteritems():
#self.req.write(tag + ":" + value + "\r\n")
customHeaderList.append(self.customHeaderPrefix + tag + ":" + val.lstrip() + nl)
if val == 'date':
dateString = ''
customHeaderList.sort()
for val in customHeaderList:
canCustomHeaders += val
#Canoicalize URI
import urllib
uriString = ""
if self.virtualBucket:
uriString = "/" + urllib.quote(self.bucket)
uriString += (self.req.unparsed_uri).split('?')[0]
for val in ('location', 'acl', 'logging', 'torrent'):
#self.write("CHECKING FOR ACL\r\n")
if val in self.subresources:
#self.write("FOUND ACL\r\n")
uriString += '?' + val
return (methodString + nl + contentMd5String + nl +
contentTypeString + nl + dateString + nl + canCustomHeaders + uriString)
def destroyBucket(bucket):
'''destroys a bucket if empty'''
conn = Connection()
try:
#Validate the bucket
_verifyBucket(conn, bucket, True)
#Check if the bucket is empty
query = "SELECT COUNT(*) FROM object WHERE bucket = %s"
result = conn.executeStatement(query, (escape_string(str(bucket))))
if result[0][0] > 0:
raise ConflictException.BucketNotEmptyException(bucket)
#Delete the bucket from the database and the filesystem
query = "DELETE FROM bucket WHERE bucket = %s"
conn.executeStatement(query, (escape_string(str(bucket))))
path = Config.get('common','filesystem_path')
path += str(bucket)
os.rmdir(path)
except:
conn.cancelAndClose()
raise
else:
conn.close()
def getObject(bucket, key, getMetadata, getData, byteRangeStart = None, byteRangeEnd = None, ifMatch = None, ifNotMatch = None, ifModifiedSince = None, ifNotModifiedSince = None, ifRange = None):
'''returns object'''
conn = Connection()
try:
#Validate the bucket
_verifyBucket(conn, bucket, True)
#Check for object and get information from database
query = "SELECT o.object, o.bucket, o.hashfield, o.object_create_time, o.eTag, o.object_mod_time, o.size, o.content_type, o.content_encoding, o.content_disposition, o.userid, u.username FROM object as o, user as u WHERE o.bucket = %s AND o.object = %s AND o.userid = u.userid"
result = conn.executeStatement(query, (escape_string(str(bucket)), escape_string(str(key))))
if len(result) == 0:
raise NotFoundException.NoSuchKeyException(bucket, key)
result = result[0]
#if _passPrecondition(str(result[4]), str(result[5]), str(ifMatch), str(ifNotMatch), str(ifModifiedSince), str(ifNotModifiedSince), str(ifRange)) == False:
# byteRangeStart = None
# byteRangeEnd = None
#Get metadata from database
query = "SELECT type, value FROM object_metadata WHERE bucket = %s AND object = %s"
metadata = conn.executeStatement(query, (escape_string(str(bucket)), escape_string(str(key))))
except:
conn.cancelAndClose()
raise
else:
conn.close()
metadataDict = {}
for tag in metadata:
metadataDict[str(tag[0])] = unicode(tag[1], encoding='utf8')
content_range = {}
size = 0
hashfield = str(result[2])
if getData:
#Get data from filesystem and build content_range
path = Config.get('common','filesystem_path')
path += str(bucket)
path += "/"+hashfield[0:3]+"/"+hashfield[3:6]+"/"+hashfield[6:9]+"/"+hashfield
fileReader = open(path, 'rb')
try:
data = ""
if byteRangeStart != None and byteRangeStart > 0:
fileReader.seek(byteRangeStart)
content_range['start'] = byteRangeStart
if byteRangeEnd != None and byteRangeEnd > byteRangeStart:
data = fileReader.read(byteRangeEnd-byteRangeStart)
content_range['end'] = fileReader.tell()
fileReader.read()
content_range['total'] = fileReader.tell()
size = byteRangeEnd-byteRangeStart
else:
data = fileReader.read()
content_range['end'] = fileReader.tell()
content_range['total'] = fileReader.tell()
size = content_range['total']
else:
if byteRangeEnd != None:
content_range['start'] = 0
data = fileReader.read(byteRangeEnd)
content_range['end'] = fileReader.tell()
fileReader.read()
content_range['total'] = fileReader.tell()
size = byteRangeEnd
else:
data = fileReader.read()
size = fileReader.tell()
finally:
fileReader.close()
#print data
if content_range.has_key('start'):
content_range['string'] = str(content_range['start'])+"-"+str(content_range['end'])+"/"+str(content_range['total'])
returnDict = {'key':str(result[0]),
'bucket':str(result[1]),
'hash':hashfield,
'creationTime':((result[3]).isoformat('T') + 'Z'),
'eTag':str(result[4]),
'lastModified':((result[5]).isoformat('T') + 'Z'),
'size':size,
'content-type':str(result[7]),
'owner':{'id':int(result[10]),
'name':unicode(result[11], encoding='utf8')}}
if str(result[8]) != "" and result[8] != None:
returnDict['content-encoding'] = str(result[8])
if str(result[9]) != "" and result[9] != None:
returnDict['content-disposition'] = str(result[9])
if content_range.has_key('string'):
returnDict['content-range'] = content_range['string']
if getMetadata:
returnDict['metadata'] = metadataDict
if getData:
returnDict['data'] = data
return returnDict
def setObject(userid, bucket, key, metadata, data, content_md5 = None, content_type = None, content_disposition = None, content_encoding = None):
'''setObject'''
if not userid:
userid = 1
hashString = None
conn = Connection()
try:
#Validate the bucket
_verifyBucket(conn, bucket, userid, True)
#Check for object and get information from database
calculatedMD5 = md5.new(data)
calculatedMD5HexDigest = calculatedMD5.hexdigest()
if content_md5 != None and content_md5 != calculatedMD5HexDigest:
raise BadRequestException.BadDigestException(content_md5, calculatedMD5HexDigest)
#Generate hashfield
hashfield = hashlib.sha1()
hashfield.update(key)
hashfieldHexDigest = ''
success = False
query = "SELECT COUNT(*) FROM object WHERE hashfield = %s"
attemptedHashfieldList = []
for i in range(3):
hashfield.update(str(time.time()))
hashfieldHexDigest = hashfield.hexdigest()
attemptedHashfieldList.append(str(hashfieldHexDigest))
count = conn.executeStatement(query, (str(hashfieldHexDigest)))[0][0]
if count == 0:
success = True
break
if success == False:
raise InternalErrorException.HashfieldCollisionErrorException(attemptedHashfieldList)
#Get size of file
size = len(data)
if content_type == None:
content_type = "binary/octet-stream"
if content_encoding == None:
content_encoding = ""
if content_disposition == None:
content_disposition = ""
#Build metadata query
metadataQuery = ""
if metadata != None and metadata != {}:
metadataQuery = "INSERT INTO object_metadata (bucket, object, type, value) VALUES ("+"'"
for tag, value in metadata.iteritems():
if type(value) == str or type(value) == unicode:
value = value.encode('utf8')
else:
value = str(value)
metadataQuery += escape_string(str(bucket))+"', '"+escape_string(str(key))+"', '"+escape_string(tag)+"', '"+escape_string(value)+"'), ('"
metadataQuery = metadataQuery[0:-4]
#Write to database and filesystem
result = conn.executeStatement("SELECT hashfield FROM object WHERE bucket = %s AND object = %s", (escape_string(str(bucket)), escape_string(str(key))))
if len(result) > 0:
hashString = result[0][0]
path = Config.get('common','filesystem_path')
path += str(bucket)
path += "/"+hashString[0:3]+"/"+hashString[3:6]+"/"+hashString[6:9]
os.remove(path+"/"+hashString)
try:
os.removedirs(path)
except OSError, e:
if e.errno != errno.ENOTEMPTY:
raise
hashString = str(hashfieldHexDigest)
query = "UPDATE object SET userid = %s, hashfield = %s, eTag = %s, object_mod_time = NOW(), size = %s, content_type = %s, content_encoding = %s, content_disposition = %s WHERE bucket = %s AND object = %s"
conn.executeStatement(query, (int(userid), hashString, str(calculatedMD5HexDigest), int(size), escape_string(str(content_type)), escape_string(str(content_encoding)), escape_string(str(content_disposition)), escape_string(str(bucket)), escape_string(str(key))))
conn.executeStatement("DELETE FROM object_metadata WHERE bucket = %s AND object = %s", (escape_string(str(bucket)), escape_string(str(key))))
else:
raise
hashString = str(hashfieldHexDigest)
query = "UPDATE object SET userid = %s, hashfield = %s, eTag = %s, object_mod_time = NOW(), size = %s, content_type = %s, content_encoding = %s, content_disposition = %s WHERE bucket = %s AND object = %s"
conn.executeStatement(query, (int(userid), hashString, str(calculatedMD5HexDigest), int(size), escape_string(str(content_type)), escape_string(str(content_encoding)), escape_string(str(content_disposition)), escape_string(str(bucket)), escape_string(str(key))))
conn.executeStatement("DELETE FROM object_metadata WHERE bucket = %s AND object = %s", (escape_string(str(bucket)), escape_string(str(key))))
else:
query = "INSERT INTO object (userid, object, bucket, hashfield, object_create_time, eTag, object_mod_time, size, content_type, content_encoding, content_disposition) VALUES (%s, %s, %s, %s, NOW(), %s, NOW(), %s, %s, %s, %s)"
hashString = str(hashfieldHexDigest)
conn.executeStatement(query, (int(userid), escape_string(str(key)), escape_string(str(bucket)), hashString, str(calculatedMD5HexDigest), int(size), escape_string(str(content_type)), escape_string(str(content_encoding)), escape_string(str(content_disposition))))
if metadataQuery != "":
conn.executeStatement(metadataQuery, ())
except:
conn.cancelAndClose()
raise
conn.close()
path = Config.get('common','filesystem_path')
path += str(bucket)
path += "/"+hashString[0:3]+"/"+hashString[3:6]+"/"+hashString[6:9]+"/"
os.makedirs(path)
path += hashString
fileReader = open(path, 'wb')
try:
fileReader.write(data)
finally:
fileReader.close()
return content_type, str(calculatedMD5HexDigest), hashString
def __init__(self, accessKey):
accesskeyIdPrefix = str(Config.get('authentication', 'prefix'))
ForbiddenException.__init__(self,
{'Message' : 'The ' + accesskeyIdPrefix + ' Access Key Id you provided does not exist in our records',
'Code' : 'InvalidAccessKeyId',
accesskeyIdPrefix + 'AccessKey' : str(accessKey)})
connection pool abstraction over previous Connection.py which is now SingleConnection.py
sets up module scope connection pool, currently with no size limit
pool for both connections with dictionary cursors and regular cursors
reconnects to db every x hours depending on config file
@author: Andrew
'''
from utaka.src.dataAccess.SingleConnection import Connection as SingleConnection
import utaka.src.Config as Config
import MySQLdb
import datetime
dcp = [SingleConnection(True)]
rcp = [SingleConnection(False)]
dbTimer = datetime.datetime.today()
dbTimeout = datetime.timedelta(hours = int(Config.get('database', 'connection_timeout_in_hours')))
class Connection:
def __init__(self, useDictCursor = False):
if len(dcp) > 0:
if useDictCursor:
self.innerConn = dcp.pop()
else:
self.innerConn = rcp.pop()
now = datetime.datetime.today()
if (now - dbTimeout) > self.innerConn.connectTime:
self.innerConn.close()
self.innerConn = SingleConnection(useDictCursor)
else:
self.innerConn = SingleConnection(useDictCursor)
def __init__(self, argValue):
headerPrefix = str(Config.get('authentication', 'prefix'))
InvalidArgumentException.__init__(self, argValue, 'Authorization', ("Authorization header is invalid. Expected " + headerPrefix + " AccessKeyId:signature"))
#distributed under the License is distributed on an "AS IS" BASIS,
#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#See the License for the specific language governing permissions and
#limitations under the License.
'''
Created August 6, 2009
Logging
@author: Andrew
'''
import logging
import utaka.src.Config as Config
import datetime
LOG_FILEPATH = Config.get('logging', 'path')
LOG_LEVEL = logging.DEBUG
BUCKET_LOG_LEVEL = logging.DEBUG
LOG_FORMAT = '%(asctime)s - %(levelname)s - %(message)s'
def logDebug(msg):
__logging_(msg, logging.DEBUG)
def logInfo(msg):
__logging_(msg, logging.INFO)
def logWarn(msg):
__logging_(msg, logging.WARN)
def logError(msg):
__logging_(msg, logging.ERROR)
def logCritical(msg):
__logging_(msg, logging.CRITICAL)
def __logging_(msg, lvl):