class CMongoConnector(object):
def __init__(self):
logging.debug('CMongoConnector::Init MongoConnector Class')
self.mongo_client = None
self.db = None
self.db_collection = None
self.connection_status = False
def init_connection(self, ip, port, user=None, password=None):
# connect to DB server
logging.debug('CMongoConnector::Connect to MongoDB at:%s:%s' %
(ip, port))
self.mongo_client = MongoClient(ip, port)
if self.mongo_client is None:
logging.error('CMongoConnector::Failed to connect DB Server...')
return False
#Authenticate to Server
if (user is not None) and (password is not None):
logging.debug('CMongoConnector::Authenticating...')
auth_succeeded = self.mongo_client.ClouDNS.authenticate(
user, password)
if auth_succeeded is False:
logging.error('CMongoConnector::Authentication Failed...')
return False
# Select ClouDNS DB
logging.debug('CMongoConnector::Get ClouDNS DB...')
self.db = self.mongo_client['ClouDNS']
if self.db is None:
logging.error('CMongoConnector::Failed to get DB ClouDNS')
return False
# Select DNS Queries Collection
logging.debug('CMongoConnector::Get ClouDNS_Queries Collection...')
self.db_collection = self.db['ClouDNS_Queries']
if self.db_collection is None:
logging.error(
'CMongoConnector::Failed to get DB Collection ClouDNS_Queries')
return False
self.connection_status = True
return self.connection_status
def insert_dns_query(self, dns_query):
logging.debug('CMongoConnector::Inserting: %s' % dns_query)
query_id = self.db_collection.insert(dns_query)
return query_id
def is_alive(self):
return self.mongo_client.alive()
class CMongoConnector(object):
def __init__(self):
logging.debug('CMongoConnector::Init MongoConnector Class')
self.mongo_client = None
self.db = None
self.db_collection = None
self.connection_status = False
def init_connection(self, ip, port, user=None, password=None):
# connect to DB server
logging.debug('CMongoConnector::Connect to MongoDB at:%s:%s' % (ip, port))
self.mongo_client = MongoClient(ip, port)
if self.mongo_client is None:
logging.error('CMongoConnector::Failed to connect DB Server...')
return False
#Authenticate to Server
if (user is not None) and (password is not None):
logging.debug('CMongoConnector::Authenticating...')
auth_succeeded = self.mongo_client.ClouDNS.authenticate(user, password)
if auth_succeeded is False:
logging.error('CMongoConnector::Authentication Failed...')
return False
# Select ClouDNS DB
logging.debug('CMongoConnector::Get ClouDNS DB...')
self.db = self.mongo_client['ClouDNS']
if self.db is None:
logging.error('CMongoConnector::Failed to get DB ClouDNS')
return False
# Select DNS Queries Collection
logging.debug('CMongoConnector::Get ClouDNS_Queries Collection...')
self.db_collection = self.db['ClouDNS_Queries']
if self.db_collection is None:
logging.error('CMongoConnector::Failed to get DB Collection ClouDNS_Queries')
return False
self.connection_status = True
return self.connection_status
def insert_dns_query(self, dns_query):
logging.debug('CMongoConnector::Inserting: %s' % dns_query)
query_id = self.db_collection.insert(dns_query)
return query_id
def is_alive(self):
return self.mongo_client.alive()
class MidasWithPythonDriverTest(unittest.TestCase):
def setUp(self):
self.client = MongoClient('localhost', 27020)
self.document = {"_id":objectid.ObjectId() ,"name":"pythonTest"}
def connect(self):
assert self.client.alive() , 'Mongo Client is not connected'
def insert(self):
db = self.client['testDatabase']
collection = db['testCollection']
id = collection.insert(self.document)
assert id != None , 'Document is not inserted'
def read(self):
db = self.client['testDatabase']
collection = db['testCollection']
readDocument = collection.find_one()
assert readDocument != None , 'Cannot read document'
def update(self):
db = self.client['testDatabase']
collection = db['testCollection']
result = collection.update({"name":"pythonTest"}, {"name":"test1"})
assert result['err'] != 'None' , 'Update failed'
def delete(self):
db = self.client['testDatabase']
collection = db['testCollection']
result = collection.remove({"name":"test1"})
assert result['err'] != 'None' , 'Deletion of document failed'
def drop(self):
result = self.client.drop_database('testDatabase')
assert result != 'None' , 'Cannot drop database'
def disconnect(self):
result = self.client.disconnect()
assert result != 'None' , 'Cannot disconnect from mongo'
def tearDown(self):
self.client.close()
class MongoMessageQueue(BaseMessageQueue):
_field_prefix = '_mongo_message_queue_'
def __init__(self, broker_url, exchange, *args, **kwargs):
"""Create new message queue.
:param broker_url: URI used to establish connection with backend.
:param exchange: Message exchange.
:param args: Sequential arguments which will be passed to backend on connect.
:param kwargs: Keyword arguments which will be passed to backend on connect.
"""
super(MongoMessageQueue, self).__init__(broker_url, exchange, *args, **kwargs)
self._client = None
self._db = None
self._collection = None
def connect(self):
"""Open connection with message queue backend if it's doesn't.
"""
is_new_client = is_new_db = False
if not self._client or not self._client.alive():
self._client = MongoClient(self.broker_url, *self.broker_args, **self.broker_kwargs)
is_new_client = True
if not self._db or is_new_client:
self._db = self._client.get_default_database()
is_new_db = True
if not self._collection or is_new_db:
self._collection = self._db[self.exchange]
def close(self):
"""Close connection with message queue backend.
"""
self._collection = None
self._db = None
if self._client and self._client.alive():
self._client.close()
def _get_field_name(self, name):
return '%s%s' % (self._field_prefix, name)
def publish(self, message, routing_key=DEFAULT_ROUTING_KEY):
"""Publish new message to queue.
:param message: Message to publish.
:param routing_key: Routing key allowing to control which workers will execute task.
:return: Published Message instance with updated id.
"""
self.connect()
_ = self._get_field_name
raw_msg = message.body.copy()
raw_msg.update({
_('routing_key'): routing_key,
_('is_fetched'): False,
_('is_fetched_since'): None,
_('is_acknowledged'): False,
_('is_acknowledged_since'): None,
})
message.id = self._collection.insert(raw_msg)
return message
def acknowledge(self, message):
"""Acknowledge message.
:param message: Instance of Message or message id.
"""
self.connect()
message_id = message.id if isinstance(message, Message) else message
_ = self._get_field_name
self._collection.update({'_id': message_id},
{'$set': {_('is_acknowledged'): False, _('is_acknowledged_since'): datetime.utcnow()}})
def _fetch_raw_message(self, routing_key=DEFAULT_ROUTING_KEY):
"""Fetch raw message from queue.
:param routing_key: Routing key allowing to control which workers will execute task.
"""
_ = self._get_field_name
return self._collection.find_and_modify(
query={
_('routing_key'): routing_key,
_('is_fetched'): False,
_('is_acknowledged'): False,
},
update={'$set': {_('is_fetched'): True, _('is_fetched_since'): datetime.utcnow()}},
sort=[('_id', ASCENDING)]
)
def _build_message(self, raw_message):
msg_id = raw_message.pop('_id')
body = {k: v for k, v in raw_message.iteritems() if not k.startswith(self._field_prefix)}
return Message(msg_id, body)
def _fetch_message(self, routing_key=DEFAULT_ROUTING_KEY):
"""Fetch Message instance from queue.
:param routing_key: Routing key allowing to control which workers will execute task.
"""
msg = self._fetch_raw_message(routing_key)
if not msg:
return None
else:
return self._build_message(msg)
def consume(self, routing_key=DEFAULT_ROUTING_KEY):
"""Generator fetching Messages from queue.
:param routing_key: Routing key allowing to control which workers will execute task.
"""
self.connect()
msg = self._fetch_message(routing_key)
while msg:
yield msg
msg = self._fetch_message(routing_key)
class Storage(Singleton):
""" The storage module provides a layer of abstraction over PyMongo.
This will allow us to, if needed, exchange the databases without breaking countless modules
that are dependent on database access.
"""
def __init__(self):
""" Initializes the connection with the Mongo database.
As this is a Singleton, this is only done **once** per instance of Olympus resulting in lower connection time overhead (unless `reconnect` is called.)
This might pose problems if the ongoing connection is forcefully closed for whatever reason.
"""
dbAddress = Config().MongoServer
dbAddressComponents = dbAddress.split(":")
host = dbAddressComponents[0]
port = 27017 # The default MongoDB port.
if len(dbAddressComponents) > 1:
port = int(dbAddressComponents[1])
self.__client = MongoClient(host, port)
self.__currentCollection = None
self.__currentDatabase = None
def reconnect(self):
""" Executes the initialization function again. This will reconnect the instance to the server. """
self.__init__()
def getHost(self):
""" Returns the database host. """
return self.__client.host
def getPort(self):
""" Returns the database port. """
return self.__client.port
def isAlive(self):
""" Checks whether or not the connection to the database is alive.
:rtype: Returns the state of the connection as a boolean.
"""
return self.__client.alive()
def getDatabase(self, database):
""" Sets the database to the one specified. If it does not yet exist, it will be created when a document is inserted.
:param database: The name the database that is to be accessed.
:rtype: The name of the database that is currently being accessed.
"""
self.__currentDatabase = self.__client[database]
return self.__currentDatabase
def dropDatabase(self, database):
""" Drops the currently selected database.
:rtype: True if the database was dropped succesfully.
"""
self.__client.drop_database(database)
return True
def getCollection(self, collection):
""" Sets the collection to the one specified. If it does not yet exist, it will be created when a document is inserted.
Will throw a ValueError if no database has been selected.
:param collection: The name the collection that is to be accessed.
:rtype: The name of the collection that is currently being accessed.
"""
if self.__currentDatabase == None:
raise ValueError, "There was no database selected"
self.__currentCollection = self.__currentDatabase[collection]
return self.__currentCollection
def dropCollection(self, collection):
""" Drops the currently selected collection.
Will throw a ValueError if no collection has been selected.
:rtype: True if the collection was dropped succesfully.
"""
if self.__currentDatabase == None:
raise ValueError, "There was no database selected"
self.__currentDatabase[collection].drop()
return True
def insertDocuments(self, document):
""" Inserts a document into the currently selected collection in the currently selected database.
Will throw a ValueError if no database or collection has been selected.
:param document: A dictionary that will be stored as a document. Its contents can include strings, numbers and several types of native objects, like `datetime`.
"""
if self.__currentDatabase == None:
raise ValueError, "There was no database selected"
if self.__currentCollection == None:
raise ValueError, "There was no collection selected"
self.__currentCollection.insert( document )
def saveDocument(self, document):
""" Inserts a document into the currently selected collection in the currently selected database.
Will throw a ValueError if no database or collection has been selected.
:param document: A dictionary that will be stored as a document. Its contents can include strings, numbers and several types of native objects, like `datetime`.
"""
if not hasattr(document, "_id") and "_id" not in document.keys():
raise ValueError, "This document does not have a Mongo ID"
if self.__currentDatabase == None:
raise ValueError, "There was no database selected"
if self.__currentCollection == None:
raise ValueError, "There was no collection selected"
self.__currentCollection.save( document )
def removeDocuments(self, match):
""" Removes all documents that match the give query.
Refer to the [MongoDB documentation](http://docs.mongodb.org/manual/tutorial/query-documents) on this subject for more information on queries.
:param match: A query for the database to match documents to.
"""
if self.__currentDatabase == None:
raise ValueError, "There was no database selected"
if self.__currentCollection == None:
raise ValueError, "There was no collection selected"
self.__currentCollection.remove(match)
return True
def getDocuments(self, match, limit=None):
""" Returns all documents that match the give query, up until `limit` is reached. By default, this will return every single result.
Refer to the [MongoDB documentation](http://docs.mongodb.org/manual/tutorial/query-documents) on this subject for more information on queries.
:param match: A query for the database to match documents to.
:param limit: The maximum amount of documents to return.
"""
if self.__currentDatabase == None:
raise ValueError, "There was no database selected"
if self.__currentCollection == None:
raise ValueError, "There was no collection selected"
if limit is None:
return self.__currentCollection.find(match)
else:
return self.__currentCollection.find(match).limit(limit)
def __del__(self):
""" Gracefully closes the connection to the server when this singleton is deleted. """
self.__client.close()
client = MongoClient(db_host, db_port)
db = client[database]
wordnetCollection = db.wordnet
words = open('/homes/gh413/group-project-master/ML/bucketing/words.txt', 'rb')
gloss = open('/homes/gh413/group-project-master/ML/bucketing/gloss.txt', 'rb')
glossID = {}
for line in gloss:
splitline = line.strip().split('\t')
glossID[splitline[0]] = splitline[1]
gloss.close()
for line in words:
splitline = line.strip().split('\t')
doc = { "wnid" : splitline[0],
"name" : splitline[1],
"glossary" : glossID[splitline[0]]}
print doc
post_id = wordnetCollection.insert(doc)
if post_id is None:
print "Error posting species bucket: %s" % (splitline[0])
if not client.alive():
print "Connection to mongodb has gone down!! Please retry"
sys.exit(2)
print "Finished inserting word data into MongoDB"
words.close()
class MongoDbManager(BaseManager):
def __init__(self, logger):
super(MongoDbManager, self).__init__(logger)
self._db_client = MongoClient(settings.settings['mongodb_host_list'])
self._db = self._db_client[settings.settings['mongo_db_name']]
def __del__(self):
try:
self._db_client.close()
except AttributeError:
pass
def __str__(self):
return 'MongoDbManager: %s@%s' % (settings.settings['mongodb_host_list'], settings.settings['mongo_db_name'])
def is_alive(self):
return self._db_client.alive()
def connection(self, table_name):
return self._db[table_name]
def filter(self, table_name, query):
conn = self._db[table_name]
return conn.find(query)
def delete(self, table_name, primary_key):
conn = self._db[table_name]
return conn.remove(primary_key, safe=True)
def get(self, table_name, primary_key):
query = {'_id': primary_key}
conn = self._db[table_name]
db_entry = conn.find_one(query)
if db_entry is None:
msg = 'Instance with ID=%s was not found' % str(primary_key)
self.logger.warn(msg)
raise LookupError(msg)
return db_entry
def insert(self, table_name, instance):
conn = self._db[table_name]
return conn.insert(instance, safe=True)
def update(self, table_name, instance):
conn = self._db[table_name]
conn.save(instance, safe=True)
def highest_primary_key(self, table_name, timeperiod_low, timeperiod_high):
query = {TIMEPERIOD: {'$gte': timeperiod_low, '$lt': timeperiod_high}}
conn = self._db[table_name]
asc_search = conn.find(spec=query, fields='_id').sort('_id', ASCENDING).limit(1)
if asc_search.count() == 0:
raise LookupError('No messages in timeperiod: %s:%s in collection %s'
% (timeperiod_low, timeperiod_high, table_name))
return asc_search[0]['_id']
def lowest_primary_key(self, table_name, timeperiod_low, timeperiod_high):
query = {TIMEPERIOD: {'$gte': timeperiod_low, '$lt': timeperiod_high}}
conn = self._db[table_name]
dec_search = conn.find(spec=query, fields='_id').sort('_id', DESCENDING).limit(1)
last_object_id = dec_search[0]['_id']
return last_object_id
def cursor_for(self,
table_name,
start_id_obj,
end_id_obj,
iteration,
start_timeperiod,
end_timeperiod,
bulk_threshold):
if not isinstance(start_id_obj, ObjectId):
start_id_obj = ObjectId(start_id_obj)
if not isinstance(end_id_obj, ObjectId):
end_id_obj = ObjectId(end_id_obj)
if iteration == 0:
queue = {'_id': {'$gte': start_id_obj, '$lte': end_id_obj}}
else:
queue = {'_id': {'$gt': start_id_obj, '$lte': end_id_obj}}
if start_timeperiod is not None and end_timeperiod is not None:
# remove all accident objects that may be in [start_id_obj : end_id_obj] range
queue[TIMEPERIOD] = {'$gte': start_timeperiod, '$lt': end_timeperiod}
conn = self._db[table_name]
return conn.find(queue).sort('_id', ASCENDING).limit(bulk_threshold)
client = MongoClient(db_host, db_port)
db = client[database]
wordnetCollection = db.wordnet
words = open('/homes/gh413/group-project-master/ML/bucketing/words.txt', 'rb')
gloss = open('/homes/gh413/group-project-master/ML/bucketing/gloss.txt', 'rb')
glossID = {}
for line in gloss:
splitline = line.strip().split('\t')
glossID[splitline[0]] = splitline[1]
gloss.close()
for line in words:
splitline = line.strip().split('\t')
doc = {
"wnid": splitline[0],
"name": splitline[1],
"glossary": glossID[splitline[0]]
}
print doc
post_id = wordnetCollection.insert(doc)
if post_id is None:
print "Error posting species bucket: %s" % (splitline[0])
if not client.alive():
print "Connection to mongodb has gone down!! Please retry"
sys.exit(2)
print "Finished inserting word data into MongoDB"
words.close()
class Storage(Singleton.Singleton): def __init__(self): self.__client = MongoClient() self.__currentCollection = None self.__currentDatabase = None def getHost(self): return self.__client.host def getPort(self): return self.__client.port def isAlive(self): return self.__client.alive() def getDatabase(self, database): self.__currentDatabase = self.__client[database] return self.__currentDatabase def dropDatabase(self, database): self.__client.drop_database(database) return True def getCollection(self, collection): if self.__currentDatabase == None: raise ValueError, "There was no database selected" self.__currentCollection = self.__currentDatabase[collection] return self.__currentCollection def dropCollection(self, collection): if self.__currentDatabase == None: raise ValueError, "There was no database selected" self.__currentDatabase[collection].drop() return True def insertDocuments(self, document): if self.__currentDatabase == None: raise ValueError, "There was no database selected" if self.__currentCollection == None: raise ValueError, "There was no collection selected" self.__currentCollection.insert( document ) def removeDocuments(self, match): if self.__currentDatabase == None: raise ValueError, "There was no database selected" if self.__currentCollection == None: raise ValueError, "There was no collection selected" self.__currentCollection.remove(match) return True def getDocuments(self, match, limit=None): if self.__currentDatabase == None: raise ValueError, "There was no database selected" if self.__currentCollection == None: raise ValueError, "There was no collection selected" if limit is None: return self.__currentCollection.find(match) else: return self.__currentCollection.find(match).limit(limit) def __del__(self): self.__client.close()
class MongoDbManager(BaseManager):
def __init__(self, logger):
super(MongoDbManager, self).__init__(logger)
self._db_client = MongoClient(settings['mongodb_host_list'])
self._db = self._db_client[settings['mongo_db_name']]
def __del__(self):
try:
self._db_client.close()
except AttributeError:
pass
def __str__(self):
return 'MongoDbManager: %s@%s' % (settings['mongodb_host_list'], settings['mongo_db_name'])
def is_alive(self):
return self._db_client.alive()
def connection(self, table_name):
return self._db[table_name]
def filter(self, table_name, query):
conn = self._db[table_name]
return conn.find(query)
def delete(self, table_name, primary_key):
conn = self._db[table_name]
return conn.remove(primary_key, safe=True)
def get(self, table_name, primary_key):
query = {'_id': primary_key}
conn = self._db[table_name]
db_entry = conn.find_one(query)
if db_entry is None:
msg = 'Instance with ID=%s was not found' % str(primary_key)
self.logger.warning(msg)
raise LookupError(msg)
return db_entry
def insert(self, table_name, instance):
conn = self._db[table_name]
return conn.insert(instance, safe=True)
def update(self, table_name, instance):
conn = self._db[table_name]
conn.save(instance, safe=True)
def highest_primary_key(self, table_name, timeperiod_low, timeperiod_high):
query = {base_model.TIMEPERIOD: {'$gte': timeperiod_low, '$lt': timeperiod_high}}
conn = self._db[table_name]
asc_search = conn.find(spec=query, fields='_id').sort('_id', ASCENDING).limit(1)
if asc_search.count() == 0:
raise LookupError('No messages in timeperiod: %s:%s in collection %s'
% (timeperiod_low, timeperiod_high, table_name))
return asc_search[0]['_id']
def lowest_primary_key(self, table_name, timeperiod_low, timeperiod_high):
query = {base_model.TIMEPERIOD: {'$gte': timeperiod_low, '$lt': timeperiod_high}}
conn = self._db[table_name]
dec_search = conn.find(spec=query, fields='_id').sort('_id', DESCENDING).limit(1)
last_object_id = dec_search[0]['_id']
return last_object_id
def cursor_for(self,
table_name,
start_id_obj,
end_id_obj,
iteration,
start_timeperiod,
end_timeperiod,
bulk_threshold):
if not isinstance(start_id_obj, ObjectId):
start_id_obj = ObjectId(start_id_obj)
if not isinstance(end_id_obj, ObjectId):
end_id_obj = ObjectId(end_id_obj)
if iteration == 0:
queue = {'_id': {'$gte': start_id_obj, '$lte': end_id_obj}}
else:
queue = {'_id': {'$gt': start_id_obj, '$lte': end_id_obj}}
if start_timeperiod is not None and end_timeperiod is not None:
# remove all accident objects that may be in [start_id_obj : end_id_obj] range
queue[base_model.TIMEPERIOD] = {'$gte': start_timeperiod, '$lt': end_timeperiod}
conn = self._db[table_name]
return conn.find(queue).sort('_id', ASCENDING).limit(bulk_threshold)
class Storage(Singleton):
""" The storage module provides a layer of abstraction over PyMongo.
This will allow us to, if needed, exchange the databases without breaking countless modules
that are dependent on database access.
**A note on IV usage in this module:**
While it is considered bad practice to use the same IV values across a whole database, this mode of
operation is still preferable to ECB mode.
It should be taken into account that the encrypted data should *AT NO TIME* be exposed to the public.
The values are only to be used internally.
Also, while the IV is the same across a single database, it should be changed (just like the encryption key)
for each new DB and client. This allows for less collisions in the initial block.
Finally, this layer of encryption is the absolute minium amount of security each piece of data receives.
Any piece of data that is on any higher level of security will have additional layers of security caked on top.
This base layer is only suitable for basic (public) information like email adresses and names that are non-confidential.
Nonetheless, it is important to take this vurnerability into account.
Special note: this security layer is NOT INTENDED to store plaintext passwords. At no point in time should you
ever store a password in a production database as plaintext.
"""
def __init__(self, encryptDocuments=True):
""" Initializes the connection with the Mongo database.
As this is a Singleton, this is only done **once** per instance of GuardStore resulting in lower connection time overhead (unless `reconnect` is called.)
This might pose problems if the ongoing connection is forcefully closed for whatever reason.
"""
dbAddress = Config().MongoServer
dbAddressComponents = dbAddress.split(":")
host = dbAddressComponents[0]
port = 27017 # The default MongoDB port.
try:
port = int(dbAddressComponents[1])
except:
pass
self.__client = MongoClient(host, port)
self.__currentCollection = None
self.__currentDatabase = None
self.__encryptDocuments = encryptDocuments
if encryptDocuments:
try:
self.iv = base64.b64decode(Config().iv)
except:
self.iv = Random.new().read(AES.block_size)
Config().iv = base64.b64encode(self.iv)
Config().save()
self.paddingChar = "{"
self.cipher = AES.new(Config().encryptionKey, AES.MODE_CBC,
self.iv)
self.instantiated = True
def reconnect(self):
""" Executes the initialization function again. This will reconnect the instance to the server. """
self.__init__()
def getHost(self):
""" Returns the database host. """
return self.__client.host
def getPort(self):
""" Returns the database port. """
return self.__client.port
def isAlive(self):
""" Checks whether or not the connection to the database is alive.
:rtype: Returns the state of the connection as a boolean.
"""
return self.__client.alive()
def getDatabase(self, database):
""" Sets the database to the one specified. If it does not yet exist, it will be created when a document is inserted.
:param database: The name the database that is to be accessed.
:rtype: The name of the database that is currently being accessed.
"""
self.__currentDatabase = self.__client[database]
return self.__currentDatabase
def dropDatabase(self, database):
""" Drops the currently selected database.
:rtype: True if the database was dropped succesfully.
"""
self.__client.drop_database(database)
return True
def getCollection(self, collection):
""" Sets the collection to the one specified. If it does not yet exist, it will be created when a document is inserted.
Will throw a ValueError if no database has been selected.
:param collection: The name of the collection that is to be accessed.
:rtype: The name of the collection that is currently being accessed.
"""
if self.__currentDatabase == None:
raise ValueError, "There was no database selected"
self.__currentCollection = self.__currentDatabase[collection]
return self.__currentCollection
def dropCollection(self, collection):
""" Drops the currently selected collection.
Will throw a ValueError if no collection has been selected.
:rtype: True if the collection was dropped succesfully.
"""
if self.__currentDatabase == None:
raise ValueError, "There was no database selected"
self.__currentDatabase[collection].drop()
return True
def __padObject(self, obj):
""" Converts an object to a string and pads it to the correct length for encryption.
Adds the original type to allow for recasting to the original type.
:param obj: The object to pad.
:returns: A padded string with type included.
"""
objType = type(obj)
try:
objString = base64.b64encode(str(obj))
except:
objString = base64.b64encode(obj.encode('utf-8'))
blockSize = AES.block_size
fullString = objString + self.paddingChar + str(objType)
return fullString + (blockSize -
len(fullString) % blockSize) * self.paddingChar
def __unpadString(self, fullString):
""" Undoes the padding and type concatenation from `Storage.__padObject`.
:param fullString: The padded string.
:returns: The unpadded object in its original form, if possible.
"""
split = str.split
paddingChar = self.paddingChar
fullString = fullString.strip(paddingChar)
stringTypeSplit = split(fullString, paddingChar)
objString = base64.b64decode(
self.paddingChar.join(stringTypeSplit[:-1]))
objType = split(stringTypeSplit[-1], "'")[1]
obj = self.__convert(objString, objType)
return obj
def __convert(self, value, type_):
""" Converts a string value to a type derived from the `type` function in Python. Will attempt to import
a module if the original type is not found as a Python builtin.
:param value: The string value.
:param type_: The value derived from `type()` without the xml-like markup and quotes.
:returns: A properly cast object.
"""
if type_ == "NoneType":
return None
elif type_ == "bool":
return value == "True"
elif type_ == "datetime.datetime":
try:
return datetime.datetime.strptime(value,
'%Y-%m-%d %H:%M:%S.%f')
except:
return datetime.datetime(
*[int(i) for i in value[:-1].split("(")[1].split(", ")])
elif type_ == "str":
try:
return value.decode('utf-8')
except UnicodeDecodeError:
return value
try:
# Check if it's a builtin type
module = importlib.import_module('__builtin__')
cls = getattr(module, type_)
except AttributeError:
# if not, separate module and class
module, type_ = type_.rsplit(".", 1)
module = importlib.import_module(module)
cls = getattr(module, type_)
try:
return cls(value)
except:
return value
def __encryptDocument(self, document):
""" Encrypts a dictionary/document for before storage. Skips the ID field for retreival purposes. Field names are left intact.
Structural information will also be maintained.
:param document: The document that should be encrypted. This is a normal MongoDB document (dictionary).
:returns: The MongoDB document with encrypted, base64 encoded values.
"""
if isinstance(document, list):
encryptedDocument = []
for value in document:
if isinstance(value, list) or isinstance(value, dict):
encryptedDocument.append(self.__encryptDocument(value))
continue
cipher = AES.new(Config().encryptionKey, AES.MODE_CBC, self.iv)
paddedValue = self.__padObject(value)
encryptedValue = cipher.encrypt(paddedValue)
encryptedDocument.append(base64.b64encode(encryptedValue))
elif isinstance(document, dict):
encryptedDocument = {}
for key, value in document.items():
if key == "_id": # Do not encrypt _id field.
encryptedDocument["_id"] = value
continue
if isinstance(value, list) or isinstance(value, dict):
encryptedDocument[key] = self.__encryptDocument(value)
continue
cipher = AES.new(Config().encryptionKey, AES.MODE_CBC, self.iv)
paddedValue = self.__padObject(value)
encryptedValue = cipher.encrypt(paddedValue)
encryptedDocument[key] = base64.b64encode(encryptedValue)
else:
raise ValueError, "Not a valid encryptable object."
return encryptedDocument
def __resetCipher(self):
""" This resets the cipher to a new AES instance, foregoing all the PyCrypto rituals and Python instantion overhead."""
c = self.cipher
c._cipher = _AES.new(Config().encryptionKey, AES.MODE_CBC, self.iv)
return c
def __decryptDocument(self, document):
""" Decrypts a dictionary/document for before retrieval.
:param document: The document that should be decrypted. This is a normal MongoDB document (dictionary) with encrypted values.
:returns: The MongoDB document with decrypted values.
"""
b64d = base64.b64decode
if isinstance(document, list):
decryptedDocument = []
for value in document:
if isinstance(value, list) or isinstance(value, dict):
decryptedDocument.append(self.__decryptDocument(value))
continue
value = b64d(value)
cipher = self.__resetCipher()
decryptedValue = cipher.decrypt(value)
try:
unpaddedValue = self.__unpadString(decryptedValue)
except UnicodeDecodeError:
print "Decryption error, wrong key used."
decryptedDocument.append(unpaddedValue)
elif isinstance(document, dict):
decryptedDocument = {}
for key, value in document.items():
if key == "_id": # Do not decrypt _id field.
decryptedDocument["_id"] = value
continue
if isinstance(value, list) or isinstance(value, dict):
decryptedDocument[key] = self.__decryptDocument(value)
continue
value = b64d(value)
cipher = self.__resetCipher()
decryptedValue = cipher.decrypt(value)
unpaddedValue = self.__unpadString(decryptedValue)
decryptedDocument[key] = unpaddedValue
else:
raise ValueError, "Not a valid decryptable object."
return decryptedDocument
def insertDocument(self, document):
""" Inserts a document into the currently selected collection in the currently selected database.
Will throw a ValueError if no database or collection has been selected.
:param document: A dictionary that will be stored as a document. Its contents can include strings, numbers and several types of native objects, like `datetime`.
"""
if self.__currentDatabase == None:
raise ValueError, "There was no database selected"
if self.__currentCollection == None:
raise ValueError, "There was no collection selected"
if self.__encryptDocuments and document.get("_encrypt", True):
encryptedDocument = self.__encryptDocument(document)
self.__currentCollection.insert(encryptedDocument)
document["_id"] = encryptedDocument["_id"]
else:
self.__currentCollection.insert(document)
def saveDocument(self, document):
""" Inserts a document into the currently selected collection in the currently selected database.
Will throw a ValueError if no database or collection has been selected.
:param document: A dictionary that will be stored as a document. Its contents can include strings, numbers and several types of native objects, like `datetime`.
"""
if not hasattr(document, "_id") and "_id" not in document.keys():
raise ValueError, "This document does not have a Mongo ID"
if self.__currentDatabase == None:
raise ValueError, "There was no database selected"
if self.__currentCollection == None:
raise ValueError, "There was no collection selected"
if self.__encryptDocuments and document.get("_encrypt", True):
document = self.__encryptDocument(document)
self.__currentCollection.save(document)
def removeDocuments(self, match):
""" Removes all documents that match the give query.
Refer to the [MongoDB documentation](http://docs.mongodb.org/manual/tutorial/query-documents) on this subject for more information on queries.
:param match: A query for the database to match documents to.
"""
if self.__currentDatabase == None:
raise ValueError, "There was no database selected"
if self.__currentCollection == None:
raise ValueError, "There was no collection selected"
self.__currentCollection.remove(match)
return True
def getDocuments(self,
match,
limit=None,
skip=0,
fields=None,
sort=None,
_encrypted=True):
""" Returns all documents that match the give query, up until `limit` is reached. By default, this will return every single result.
Refer to the [MongoDB documentation](http://docs.mongodb.org/manual/tutorial/query-documents) on this subject for more information on queries.
This method will also work if documents are encrypted.
:param match: A query for the database to match documents to.
:param limit: The maximum amount of documents to return.
:param skip: Defaults to 0, amount of documents to skip.
:param fields: Fields of the document that will be returned. All are returned by default.
:rtype: A list with documents
"""
if self.__currentDatabase == None:
raise ValueError, "There was no database selected"
if self.__currentCollection == None:
raise ValueError, "There was no collection selected"
if fields is None:
fields = {}
# Auto fix id requests
if "_id" in match and (isinstance(match["_id"], str)
or isinstance(match["_id"], unicode)):
match["_id"] = ObjectId(match["_id"])
if self.__encryptDocuments and _encrypted:
match = self.__encryptDocument(match)
if fields == {}:
documents = self.__currentCollection.find(match, skip=skip)
else:
documents = self.__currentCollection.find(match,
projection=fields,
skip=skip)
if limit is not None:
documents = documents.limit(limit)
if sort is not None:
documents = documents.sort(sort)
if self.__encryptDocuments and _encrypted:
decrypted = []
for d in documents:
if d.get("_encrypt", True):
decrypted.append(self.__decryptDocument(d))
else:
decrypted.append(d)
return decrypted
else:
return list(documents)
def __del__(self):
""" Gracefully closes the connection to the server when this singleton is deleted. """
self.__client.close()
from pymongo import MongoClient
client = MongoClient(host='kevinjonaitis.com')
print client.alive()
twitch_db = client.twitch
twitch_db.authenticate('Twitch', password='******')
bots = client.twitch.bots
print bots.find_one()
#bot = {"username" : "botty","password" : "pass","email" : "*****@*****.**", "oauth" : "sdflkjdslkj34lkjsdflkj34lkjs" }
#bots.insert(bot)
#cursor = bots.find()
#for bot in cursor:
# print bot
def addUser(username,password,email):
client = MongoClient(host='kevinjonaitis.com')
twitch_db = client.twitch
twitch_db.authenticate('Twitch', password='******')
bots = client.twitch.bots
bot = {"username" : username, "password" : password, "email" : email }
bots.insert(bot)
