Ejemplo n.º 1
0
	def __init__(self, req, virtualBucket=False):

		self.req = req
		self.bucket = self.key = self.user = self.accesskey = self.signature = self.stringToSign = self.computedSig = None
		self.isUserAdmin = False
		self.subresources = {}
		self.__writeBuffer = ''
		self.virtualBucket = False

		#Query string digest
		if self.req.args:
			self.subresources = util.parse_qs(self.req.args, True)

		#URI digest
		basehost = Config.get('server', 'hostname')
		if self.req.hostname == basehost:
			uriDigestResults = self.uriDigest(req.uri)
			self.bucket = uriDigestResults.get('bucket')
			self.key = uriDigestResults.get('key')
		else:
			splitHost = self.req.hostname.split("." + basehost)
			if len(splitHost) == 2:
				uriDigestResults = self.uriDigest(splitHost[0] + '/' + req.uri)
				self.bucket = uriDigestResults.get('bucket')
				self.key = uriDigestResults.get('key')
				self.virtualBucket = True
			else:
				self.req.write("HOST: " + self.req.hostname + "\r\n")
				raise Exception, 'wrong hostname?'

		#custom header table
		self.customHeaderPrefix = Config.get('common', 'customHeaderPrefix').lower()
		self.customHeaderTable = {}
		for tag, val in self.req.headers_in.iteritems():
			if tag.lower().startswith(self.customHeaderPrefix):
				self.customHeaderTable[tag.lower()[len(self.customHeaderPrefix):]] = val

		#authenticate -- must happen after custom header table is created
		self.accesskey, self.signature = self.__getAccessKeyAndSignature()
		if self.accesskey:
			self.stringToSign = self.__buildStringToSign()
			self.user, self.isUserAdmin, self.computedSig = getUser(self.signature, self.accesskey, self.stringToSign)

		#Check date
		#check customDateHeader then date header

		if 'signature' in self.subresources:
			self.req.headers_out['Signature'] = str(self.computedSig)
			self.write(str(self.computedSig) + "\r\n")
			self.write(str(self.stringToSign) + "\r\n")
			self.send()
Ejemplo n.º 2
0
Archivo: Object.py Proyecto: HISG/utaka
def destroyObject(bucket, key):
	'''destroy's object'''

	conn = Connection()
	try:
		#Validate the bucket
		_verifyBucket(conn, bucket, True)

		#Check for object and get information from database
		query = "SELECT hashfield FROM object WHERE bucket = %s AND object = %s"
		result = conn.executeStatement(query, (escape_string(str(bucket)), escape_string(str(key))))
		if len(result) == 0:
			raise NotFoundException.NoSuchKeyException(bucket, key)

		#Delete the object from the database and the filesystem
		query = "DELETE FROM object_metadata WHERE bucket = %s AND object = %s"
		conn.executeStatement(query, (escape_string(str(bucket)), escape_string(str(key))))
		query = "DELETE FROM object WHERE bucket = %s AND object = %s"
		conn.executeStatement(query, (escape_string(str(bucket)), escape_string(str(key))))
	except:
		conn.cancelAndClose()
		raise
	conn.close()
	hashString = result[0][0]
	path = Config.get('common','filesystem_path')
	path += str(bucket)
	path += "/"+hashString[0:3]+"/"+hashString[3:6]+"/"+hashString[6:9]
	os.remove(path+"/"+hashString)
	try:
		os.removedirs(path)
	except OSError, e:
		if e.errno != errno.ENOTEMPTY:
			raise
Ejemplo n.º 3
0
Archivo: Bucket.py Proyecto: HISG/utaka
def setBucket(bucket, userid):
	'''creates a new empty bucket'''
	MAX_BUCKETS_PER_USER = 100

	conn = Connection()
	#Validate the bucket
	try:
		_verifyBucket(conn, bucket, False, userid)

		#Check if user has too many buckets
		query = "SELECT bucket FROM bucket WHERE userid = %s"
		result = conn.executeStatement(query, (int(userid)))
		if len(result) >= MAX_BUCKETS_PER_USER:
				raise BadRequestException.TooManyBucketsException()

		#Write bucket to database and filesystem
		query = "INSERT INTO bucket (bucket, userid, bucket_creation_time) VALUES (%s, %s, NOW())"
		conn.executeStatement(query, (escape_string(str(bucket)), int(userid)))
		path = Config.get('common','filesystem_path')
		path += str(bucket)
		os.mkdir(path)
	except:
		conn.cancelAndClose()
		raise
	else:
		conn.close()
Ejemplo n.º 4
0
	def __init__(self, stringToSignBytes, stringToSign, signatureProvided, accessKey):
		accesskeyIdPrefix = str(Config.get('authentication', 'prefix'))
		ForbiddenException.__init__(self,
			{'Message' : 'The request signature we calculated des not match the signature you provided. Check your key and signing method.',
			 'Code' : 'SignatureDoesNotMatch',
			 'StringToSignBytes' : str(stringToSignBytes),
			 'StringToSign' : str(stringToSign),
			 'SignatureProvided' : str(signatureProvided),
			 accesskeyIdPrefix + 'AccessKeyId' : str(accessKey)})
Ejemplo n.º 5
0
	def __getAccessKeyAndSignature(self):
		header = Config.get('authentication', 'header')
		prefix = Config.get('authentication', 'prefix') + ' '
		accesskey = signature = None
		try:
			authString = self.req.headers_in[header]
		except KeyError:
			pass
		else:
			splitAuth = authString.split(prefix)
			if len(splitAuth) == 2 and len(splitAuth[0]) == 0 and not splitAuth[1].startswith(' '):
				try:
					accesskey, signature = splitAuth[1].split(':')
				except ValueError:
					raise BadRequestException.InvalidArgumentAuthorizationException(argValue = authString)
			elif not len(splitAuth) == 2:
				raise BadRequestException.InvalidArgumentAuthorizationException(argValue = authString)
			else:
				raise BadRequestException.InvalidArgumentAuthorizationSpacingException(argValue = authString)
		return accesskey, signature
Ejemplo n.º 6
0
	def __init__(self, useDictCursor = False):
		host = Config.get('database','mysql_utaka_hostname')
		user = Config.get('database','mysql_utaka_username')
		passwd = Config.get('database','mysql_utaka_password')
		db = Config.get('database','mysql_utaka_database')
		self.conn = MySQLdb.connect(host = host, user = user, passwd = passwd, db = db)
		self.connectTime = datetime.datetime.today()
		if useDictCursor:
			self._usingDictCursor = True
			self.cursor = self.conn.cursor(MySQLdb.cursors.DictCursor)
		else:
			self._usingDictCursor = False
			self.cursor = self.conn.cursor()
		utakaLog = open('/var/www/html/utaka/utakaLog', 'a')
		try:
			if self._usingDictCursor:
				utakaLog.write('Dictionary Database Connection Made\r\n')
			else:
				utakaLog.write('Regular Database Connection Made\r\n')
		finally:
			utakaLog.close()
Ejemplo n.º 7
0
	def __buildStringToSign(self):
		nl = '\n'

		#http headers
		methodString = self.req.method
		contentMd5String = self.req.headers_in.get('content-md5', '')
		contentTypeString = self.req.headers_in.get('content-type', '')
		dateString = self.req.headers_in.get('date', '')

		#Canonicalize Custom Headers
		__customHeaderPrefix = Config.get('common', 'customHeaderPrefix').lower()
		__customDateHeader = __customHeaderPrefix + "-date"

		customHeaderList = []
		canCustomHeaders = ''

		for tag, val in self.customHeaderTable.iteritems():
			#self.req.write(tag + ":" + value + "\r\n")
			customHeaderList.append(self.customHeaderPrefix + tag + ":" + val.lstrip() + nl)
			if val == 'date':
					dateString = ''
		customHeaderList.sort()
		for val in customHeaderList:
			canCustomHeaders += val

		#Canoicalize URI
		import urllib
		uriString = ""
		if self.virtualBucket:
			uriString = "/" + urllib.quote(self.bucket)
		uriString += (self.req.unparsed_uri).split('?')[0]
		for val in ('location', 'acl', 'logging', 'torrent'):
			#self.write("CHECKING FOR ACL\r\n")
			if val in self.subresources:
				#self.write("FOUND ACL\r\n")
				uriString += '?' + val
		return (methodString + nl + contentMd5String + nl +
			contentTypeString + nl + dateString + nl + canCustomHeaders + uriString)
Ejemplo n.º 8
0
Archivo: Bucket.py Proyecto: HISG/utaka
def destroyBucket(bucket):
	'''destroys a bucket if empty'''
	conn = Connection()
	try:
		#Validate the bucket
		_verifyBucket(conn, bucket, True)

		#Check if the bucket is empty
		query = "SELECT COUNT(*) FROM object WHERE bucket = %s"
		result = conn.executeStatement(query, (escape_string(str(bucket))))
		if result[0][0] > 0:
			raise ConflictException.BucketNotEmptyException(bucket)

		#Delete the bucket from the database and the filesystem
		query = "DELETE FROM bucket WHERE bucket = %s"
		conn.executeStatement(query, (escape_string(str(bucket))))
		path = Config.get('common','filesystem_path')
		path += str(bucket)
		os.rmdir(path)
	except:
		conn.cancelAndClose()
		raise
	else:
		conn.close()
Ejemplo n.º 9
0
Archivo: Object.py Proyecto: HISG/utaka
def getObject(bucket, key, getMetadata, getData, byteRangeStart = None, byteRangeEnd = None, ifMatch = None, ifNotMatch = None, ifModifiedSince = None, ifNotModifiedSince = None, ifRange = None):
	'''returns object'''

	conn = Connection()
	try:
		#Validate the bucket
		_verifyBucket(conn, bucket, True)

		#Check for object and get information from database
		query = "SELECT o.object, o.bucket, o.hashfield, o.object_create_time, o.eTag, o.object_mod_time, o.size, o.content_type, o.content_encoding, o.content_disposition, o.userid, u.username FROM object as o, user as u WHERE o.bucket = %s AND o.object = %s AND o.userid = u.userid"
		result = conn.executeStatement(query, (escape_string(str(bucket)), escape_string(str(key))))
		if len(result) == 0:
				raise NotFoundException.NoSuchKeyException(bucket, key)
		result = result[0]


		#if _passPrecondition(str(result[4]), str(result[5]), str(ifMatch), str(ifNotMatch), str(ifModifiedSince), str(ifNotModifiedSince), str(ifRange)) == False:
		#    byteRangeStart = None
		#    byteRangeEnd = None


		#Get metadata from database
		query = "SELECT type, value FROM object_metadata WHERE bucket = %s AND object = %s"
		metadata = conn.executeStatement(query, (escape_string(str(bucket)), escape_string(str(key))))
	except:
		conn.cancelAndClose()
		raise
	else:
		conn.close()

	metadataDict = {}
	for tag in metadata:
		metadataDict[str(tag[0])] = unicode(tag[1], encoding='utf8')

	content_range = {}
	size = 0
	hashfield = str(result[2])
	if getData:
		#Get data from filesystem and build content_range
		path = Config.get('common','filesystem_path')
		path += str(bucket)
		path += "/"+hashfield[0:3]+"/"+hashfield[3:6]+"/"+hashfield[6:9]+"/"+hashfield
		fileReader = open(path, 'rb')
		try:
			data = ""
			if byteRangeStart != None and byteRangeStart > 0:
				fileReader.seek(byteRangeStart)
				content_range['start'] = byteRangeStart
				if byteRangeEnd != None and byteRangeEnd > byteRangeStart:
					data = fileReader.read(byteRangeEnd-byteRangeStart)
					content_range['end'] = fileReader.tell()
					fileReader.read()
					content_range['total'] = fileReader.tell()
					size = byteRangeEnd-byteRangeStart
				else:
					data = fileReader.read()
					content_range['end'] = fileReader.tell()
					content_range['total'] = fileReader.tell()
					size = content_range['total']
			else:
				if byteRangeEnd != None:
					content_range['start'] = 0
					data = fileReader.read(byteRangeEnd)
					content_range['end'] = fileReader.tell()
					fileReader.read()
					content_range['total'] = fileReader.tell()
					size = byteRangeEnd
				else:
					data = fileReader.read()
					size = fileReader.tell()
		finally:
			fileReader.close()
		#print data

		if content_range.has_key('start'):
			content_range['string'] = str(content_range['start'])+"-"+str(content_range['end'])+"/"+str(content_range['total'])

	returnDict = {'key':str(result[0]),
	              'bucket':str(result[1]),
	              'hash':hashfield,
	              'creationTime':((result[3]).isoformat('T') + 'Z'),
	              'eTag':str(result[4]),
	              'lastModified':((result[5]).isoformat('T') + 'Z'),
	              'size':size,
	              'content-type':str(result[7]),
	              'owner':{'id':int(result[10]),
	              'name':unicode(result[11], encoding='utf8')}}
	if str(result[8]) != "" and result[8] != None:
		returnDict['content-encoding'] = str(result[8])
	if str(result[9]) != "" and result[9] != None:
		returnDict['content-disposition'] = str(result[9])
	if content_range.has_key('string'):
		returnDict['content-range'] = content_range['string']
	if getMetadata:
		returnDict['metadata'] = metadataDict
	if getData:
		returnDict['data'] = data

	return returnDict
Ejemplo n.º 10
0
Archivo: Object.py Proyecto: HISG/utaka
def setObject(userid, bucket, key, metadata, data, content_md5 = None, content_type = None, content_disposition = None, content_encoding = None):
	'''setObject'''

	if not userid:
		userid = 1
	hashString = None
	conn = Connection()
	try:

		#Validate the bucket
		_verifyBucket(conn, bucket, userid, True)

		#Check for object and get information from database
		calculatedMD5 = md5.new(data)
		calculatedMD5HexDigest = calculatedMD5.hexdigest()
		if content_md5 != None and content_md5 != calculatedMD5HexDigest:
			raise BadRequestException.BadDigestException(content_md5, calculatedMD5HexDigest)

		#Generate hashfield
		hashfield = hashlib.sha1()
		hashfield.update(key)
		hashfieldHexDigest = ''
		success = False
		query = "SELECT COUNT(*) FROM object WHERE hashfield = %s"
		attemptedHashfieldList = []
		for i in range(3):
			hashfield.update(str(time.time()))
			hashfieldHexDigest = hashfield.hexdigest()
			attemptedHashfieldList.append(str(hashfieldHexDigest))
			count = conn.executeStatement(query, (str(hashfieldHexDigest)))[0][0]
			if count == 0:
				success = True
				break

		if success == False:
			raise InternalErrorException.HashfieldCollisionErrorException(attemptedHashfieldList)

		#Get size of file
		size = len(data)
		if content_type == None:
			content_type = "binary/octet-stream"
		if content_encoding == None:
			content_encoding = ""
		if content_disposition == None:
			content_disposition = ""

		#Build metadata query
		metadataQuery = ""
		if metadata != None and metadata != {}:
			metadataQuery = "INSERT INTO object_metadata (bucket, object, type, value) VALUES ("+"'"
			for tag, value in metadata.iteritems():
				if type(value) == str or type(value) == unicode:
					value = value.encode('utf8')
				else:
					value = str(value)
				metadataQuery += escape_string(str(bucket))+"', '"+escape_string(str(key))+"', '"+escape_string(tag)+"', '"+escape_string(value)+"'), ('"
			metadataQuery = metadataQuery[0:-4]

		#Write to database and filesystem
		result = conn.executeStatement("SELECT hashfield FROM object WHERE bucket = %s AND object = %s", (escape_string(str(bucket)), escape_string(str(key))))
		if len(result) > 0:
			hashString = result[0][0]
			path = Config.get('common','filesystem_path')
			path += str(bucket)
			path += "/"+hashString[0:3]+"/"+hashString[3:6]+"/"+hashString[6:9]
			os.remove(path+"/"+hashString)
			try:
				os.removedirs(path)
			except OSError, e:
				if e.errno != errno.ENOTEMPTY:
					raise
			hashString = str(hashfieldHexDigest)
			query = "UPDATE object SET userid = %s, hashfield = %s, eTag = %s, object_mod_time = NOW(), size = %s, content_type = %s, content_encoding = %s, content_disposition = %s WHERE bucket = %s AND object = %s"
			conn.executeStatement(query, (int(userid), hashString, str(calculatedMD5HexDigest), int(size), escape_string(str(content_type)), escape_string(str(content_encoding)), escape_string(str(content_disposition)), escape_string(str(bucket)), escape_string(str(key))))
			conn.executeStatement("DELETE FROM object_metadata WHERE bucket = %s AND object = %s", (escape_string(str(bucket)), escape_string(str(key))))
		else:
Ejemplo n.º 11
0
Archivo: Object.py Proyecto: HISG/utaka
					raise
			hashString = str(hashfieldHexDigest)
			query = "UPDATE object SET userid = %s, hashfield = %s, eTag = %s, object_mod_time = NOW(), size = %s, content_type = %s, content_encoding = %s, content_disposition = %s WHERE bucket = %s AND object = %s"
			conn.executeStatement(query, (int(userid), hashString, str(calculatedMD5HexDigest), int(size), escape_string(str(content_type)), escape_string(str(content_encoding)), escape_string(str(content_disposition)), escape_string(str(bucket)), escape_string(str(key))))
			conn.executeStatement("DELETE FROM object_metadata WHERE bucket = %s AND object = %s", (escape_string(str(bucket)), escape_string(str(key))))
		else:
			query = "INSERT INTO object (userid, object, bucket, hashfield, object_create_time, eTag, object_mod_time, size, content_type, content_encoding, content_disposition) VALUES (%s, %s, %s, %s, NOW(), %s, NOW(), %s, %s, %s, %s)"
			hashString = str(hashfieldHexDigest)
			conn.executeStatement(query, (int(userid), escape_string(str(key)), escape_string(str(bucket)), hashString, str(calculatedMD5HexDigest), int(size), escape_string(str(content_type)), escape_string(str(content_encoding)), escape_string(str(content_disposition))))
		if metadataQuery != "":
			conn.executeStatement(metadataQuery, ())
	except:
		conn.cancelAndClose()
		raise
	conn.close()
	path = Config.get('common','filesystem_path')
	path += str(bucket)
	path += "/"+hashString[0:3]+"/"+hashString[3:6]+"/"+hashString[6:9]+"/"
	os.makedirs(path)
	path += hashString
	fileReader = open(path, 'wb')
	try:
		fileReader.write(data)
	finally:
		fileReader.close()
	return content_type, str(calculatedMD5HexDigest), hashString




Ejemplo n.º 12
0
	def __init__(self, accessKey):
		accesskeyIdPrefix = str(Config.get('authentication', 'prefix'))
		ForbiddenException.__init__(self,
			{'Message' : 'The ' + accesskeyIdPrefix + ' Access Key Id you provided does not exist in our records',
			 'Code' : 'InvalidAccessKeyId',
			 accesskeyIdPrefix + 'AccessKey' : str(accessKey)})
Ejemplo n.º 13
0
connection pool abstraction over previous Connection.py which is now SingleConnection.py
sets up module scope connection pool, currently with no size limit
	pool for both connections with dictionary cursors and regular cursors
reconnects to db every x hours depending on config file
@author: Andrew
'''

from utaka.src.dataAccess.SingleConnection import Connection as SingleConnection
import utaka.src.Config as Config
import MySQLdb
import datetime

dcp = [SingleConnection(True)]
rcp = [SingleConnection(False)]
dbTimer = datetime.datetime.today()
dbTimeout = datetime.timedelta(hours = int(Config.get('database', 'connection_timeout_in_hours')))

class Connection:
	def __init__(self, useDictCursor = False):
		if len(dcp) > 0:
			if useDictCursor:
				self.innerConn = dcp.pop()
			else:
				self.innerConn = rcp.pop()
			now = datetime.datetime.today()
			if (now - dbTimeout) > self.innerConn.connectTime:
				self.innerConn.close()
				self.innerConn = SingleConnection(useDictCursor)
		else:
			self.innerConn = SingleConnection(useDictCursor)
Ejemplo n.º 14
0
	def __init__(self, argValue):
		headerPrefix = str(Config.get('authentication', 'prefix'))
		InvalidArgumentException.__init__(self, argValue, 'Authorization', ("Authorization header is invalid. Expected " + headerPrefix + " AccessKeyId:signature"))
Ejemplo n.º 15
0
#distributed under the License is distributed on an "AS IS" BASIS,
#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#See the License for the specific language governing permissions and
#limitations under the License.

'''
Created August 6, 2009

Logging
@author: Andrew
'''
import logging
import utaka.src.Config as Config
import datetime

LOG_FILEPATH = Config.get('logging', 'path')
LOG_LEVEL = logging.DEBUG
BUCKET_LOG_LEVEL = logging.DEBUG
LOG_FORMAT = '%(asctime)s - %(levelname)s - %(message)s'

def logDebug(msg):
	__logging_(msg, logging.DEBUG)
def logInfo(msg):
	__logging_(msg, logging.INFO)
def logWarn(msg):
	__logging_(msg, logging.WARN)
def logError(msg):
	__logging_(msg, logging.ERROR)
def logCritical(msg):
	__logging_(msg, logging.CRITICAL)
def __logging_(msg, lvl):