def getRecipes():
""" Static method to get the list of recipes """
if not DataBase.db_available:
return []
else:
try:
db = DataBase(Config.RECIPES_DB_FILE)
db.connect()
res = []
result = db.select('select * from recipes')
for d in result:
name = d[0]
version = d[1]
modules = d[2].split(",")
recipe = d[3]
isapp = d[4]
galaxy_module = d[5]
desc = d[6]
requirements = d[7]
res.append(
Recipe(name, version, modules, recipe, desc,
requirements, galaxy_module, isapp))
return res
except Exception:
return []
def _get_inf_ids_from_db():
try:
db = DataBase(Config.DATA_DB)
if db.connect():
inf_list = []
if db.db_type == DataBase.MONGO:
res = db.find("inf_list", {"deleted": 0}, {"id": True},
[('id', -1)])
else:
res = db.select(
"select id from inf_list where deleted = 0 order by rowid desc"
)
for elem in res:
if db.db_type == DataBase.MONGO:
inf_list.append(elem['id'])
else:
inf_list.append(elem[0])
db.close()
return inf_list
else:
InfrastructureList.logger.error(
"ERROR connecting with the database!.")
return []
except Exception:
InfrastructureList.logger.exception(
"ERROR loading data. Correct or delete it!!")
return []
def _save_data_to_db(db_url, inf_list, inf_id=None):
db = DataBase(db_url)
if db.connect():
infs_to_save = inf_list
if inf_id:
infs_to_save = {inf_id: inf_list[inf_id]}
for inf in infs_to_save.values():
data = inf.serialize()
if db.db_type == DataBase.MONGO:
res = db.replace("inf_list", {"id": inf.id}, {
"id": inf.id,
"deleted": int(inf.deleted),
"data": data,
"date": time.time()
})
else:
res = db.execute(
"replace into inf_list (id, deleted, data, date) values (%s, %s, %s, now())",
(inf.id, int(inf.deleted), data))
db.close()
return res
else:
InfrastructureList.logger.error(
"ERROR connecting with the database!.")
return None
def get_data_from_db(db_url):
db = DataBase(db_url)
if db.connect():
if not db.table_exists("inf_list"):
db.execute("CREATE TABLE inf_list(id VARCHAR(255) PRIMARY KEY, date TIMESTAMP, data LONGBLOB)")
db.close()
return {}
else:
inf_list = {}
res = db.select("select * from inf_list order by id desc")
if len(res) > 0:
for elem in res:
#inf_id = elem[0]
#date = elem[1]
try:
inf = pickle.loads(elem[2])
if not inf.deleted:
inf_list[inf.id] = inf
except:
InfrastructureManager.logger.exception("ERROR reading infrastructure %d from database, ignoring it!." % inf.id)
else:
InfrastructureManager.logger.error("ERROR getting inf_list from database!.")
db.close()
return inf_list
else:
InfrastructureManager.logger.error("ERROR connecting with the database!.")
return {}
def get_data_from_db(db_url):
db = DataBase(db_url)
if db.connect():
if not db.table_exists("inf_list"):
return {}
else:
inf_list = {}
res = db.select("select * from inf_list order by id desc")
if len(res) > 0:
for elem in res:
# inf_id = elem[0]
# date = elem[1]
try:
inf = pickle.loads(elem[2])
inf_list[inf.id] = inf
except:
sys.stderr.write(
"ERROR reading infrastructure from database, ignoring it!."
)
sys.exit(-1)
else:
sys.stderr.write("ERROR getting inf_list from database!.")
sys.exit(-1)
db.close()
return inf_list
else:
sys.stderr.write("ERROR connecting with the database!.")
sys.exit(-1)
def _reinit():
"""Restart the class attributes to initial values."""
InfrastructureList.infrastructure_list = {}
InfrastructureList._lock = threading.Lock()
db = DataBase(Config.DATA_DB)
if db.connect():
db.execute("delete from inf_list")
db.close()
def delete_data_from_db(db_url, date):
db = DataBase(db_url)
if db.connect():
db.execute("DELETE FROM inf_list WHERE deleted = 1 and date < '%s';" % date)
db.close()
else:
sys.stderr.write("ERROR connecting with the database!.")
sys.exit(-1)
def test_db(self, mdb_conn):
filename = "/tmp/inf.dat"
if os.path.exists(filename):
os.unlink(filename)
db_url = "sqlite://" + filename
db = DataBase(db_url)
self.assertTrue(db.connect())
if not db.table_exists("test"):
db.execute(
"CREATE TABLE test(id int PRIMARY KEY, date TIMESTAMP, data LONGBLOB)"
)
self.assertTrue(db.table_exists("test"))
db.execute("insert into test (id, data, date) values (%s, %s, now())",
(1, "Data"))
res = db.select("select data from test where id = %s", (1, ))
self.assertEqual(res, [("Data", )])
db.close()
connection = MagicMock()
mdb_conn.return_value = connection
db_url = "mysql://*****:*****@server/db_name"
db = DataBase(db_url)
self.assertTrue(db.connect())
if not db.table_exists("test"):
db.execute(
"CREATE TABLE test(id int PRIMARY KEY, date TIMESTAMP, data LONGBLOB)"
)
db.execute("insert into test (id, data, date) values (%s, %s, now())",
(1, "Data"))
cursor = MagicMock()
cursor.fetchall.return_value = [("Data", )]
connection.cursor.return_value = cursor
res = db.select("select data from test where id = %s", (1, ))
self.assertEqual(res, [("Data", )])
db.close()
def insertRecipe(name,
version,
module,
recipe,
desc,
requirements,
galaxy_module=None,
isapp=0):
""" Static method to insert a recipe in the DB """
if not DataBase.db_available:
return False
else:
try:
db = DataBase(Config.RECIPES_DB_FILE)
db.connect()
res = db.execute(
'''insert into recipes values ("%s", "%s", "%s", "%s", %d, %d, "%s", "%s")'''
% (name, version, module, recipe, isapp, galaxy_module,
desc, requirements))
return res
except Exception:
return False
def test_sqlite_db(self):
filename = "/tmp/inf.dat"
if os.path.exists(filename):
os.unlink(filename)
db_url = "sqlite://" + filename
db = DataBase(db_url)
self.assertTrue(db.connect())
if not db.table_exists("test"):
db.execute("CREATE TABLE test(id int PRIMARY KEY, date TIMESTAMP, data LONGBLOB)")
self.assertTrue(db.table_exists("test"))
db.execute("insert into test (id, data, date) values (%s, %s, now())", (1, "Data"))
res = db.select("select data from test where id = %s", (1,))
self.assertEqual(res, [("Data",)])
db.close()
def getInstallableApps():
""" Static method to get the list of avalible apps """
if not DataBase.db_available:
return []
else:
try:
db = DataBase(Config.RECIPES_DB_FILE)
db.connect()
res = []
result = db.select('select * from recipes where isapp = 1')
for d in result:
name = d[0]
version = d[1]
module = d[2]
recipe = d[3]
galaxy_module = d[5]
requirements = d[7]
res.append((FeaturesApp.from_str(name, version), module,
galaxy_module, recipe, requirements))
return res
except Exception:
return []
def save_data_to_db(db_url, inf_list, inf_id = None):
db = DataBase(db_url)
if db.connect():
infs_to_save = inf_list
if inf_id:
infs_to_save = {inf_id: inf_list[inf_id]}
for inf in infs_to_save.values():
res = db.execute("replace into inf_list set id = %s, data = %s, date = now()", (inf.id, pickle.dumps(inf)))
db.close()
return res
else:
InfrastructureManager.logger.error("ERROR connecting with the database!.")
return None
def init_table():
""" Creates de database """
db = DataBase(Config.DATA_DB)
if db.connect():
if not db.table_exists("inf_list"):
db.execute(
"CREATE TABLE inf_list(id VARCHAR(255) PRIMARY KEY, deleted INTEGER,"
" date TIMESTAMP, data LONGBLOB)")
db.close()
return True
else:
InfrastructureList.logger.error(
"ERROR connecting with the database!.")
return False
def _save_data_to_db(db_url, inf_list, inf_id=None):
db = DataBase(db_url)
if db.connect():
infs_to_save = inf_list
if inf_id:
infs_to_save = {inf_id: inf_list[inf_id]}
for inf in infs_to_save.values():
res = db.execute("replace into inf_list (id, deleted, data, date) values (%s, %s, %s, now())",
(inf.id, int(inf.deleted), inf.serialize()))
db.close()
return res
else:
InfrastructureList.logger.error("ERROR connecting with the database!.")
return None
def _get_data_from_db(db_url, inf_id=None, auth=None):
"""
Get data from DB.
If no inf_id specified all Infrastructures are loaded.
If auth is specified only auth data will be loaded.
"""
if InfrastructureList.init_table():
db = DataBase(db_url)
if db.connect():
inf_list = {}
if inf_id:
if db.db_type == DataBase.MONGO:
res = db.find("inf_list", {"id": inf_id}, {"data": True})
else:
res = db.select("select data from inf_list where id = %s", (inf_id,))
else:
if db.db_type == DataBase.MONGO:
res = db.find("inf_list", {"deleted": 0}, {"data": True}, [('_id', -1)])
else:
res = db.select("select data from inf_list where deleted = 0 order by rowid desc")
if len(res) > 0:
for elem in res:
if db.db_type == DataBase.MONGO:
data = elem['data']
else:
data = elem[0]
try:
if auth:
inf = IM.InfrastructureInfo.InfrastructureInfo.deserialize_auth(data)
else:
inf = IM.InfrastructureInfo.InfrastructureInfo.deserialize(data)
inf_list[inf.id] = inf
except Exception:
InfrastructureList.logger.exception(
"ERROR reading infrastructure from database, ignoring it!.")
else:
InfrastructureList.logger.warn("No data in database!.")
db.close()
return inf_list
else:
InfrastructureList.logger.error("ERROR connecting with the database!.")
return {}
else:
InfrastructureList.logger.error("ERROR connecting with the database!.")
return {}
def _get_inf_ids_from_db():
try:
db = DataBase(Config.DATA_DB)
if db.connect():
inf_list = []
res = db.select("select id from inf_list where deleted = 0 order by rowid desc")
for elem in res:
inf_list.append(elem[0])
db.close()
return inf_list
else:
InfrastructureList.logger.error("ERROR connecting with the database!.")
return []
except Exception:
InfrastructureList.logger.exception("ERROR loading data. Correct or delete it!!")
return []
def _get_data_from_db(db_url, inf_id=None, auth=None):
"""
Get data from DB.
If no inf_id specified all Infrastructures are loaded.
If auth is specified only auth data will be loaded.
"""
if InfrastructureList.init_table():
return {}
else:
db = DataBase(db_url)
if db.connect():
inf_list = {}
if inf_id:
res = db.select("select * from inf_list where id = '%s'" %
inf_id)
else:
res = db.select(
"select * from inf_list where deleted = 0 order by id desc"
)
if len(res) > 0:
for elem in res:
# inf_id = elem[0]
# date = elem[1]
# deleted = elem[2]
try:
if auth:
inf = IM.InfrastructureInfo.InfrastructureInfo.deserialize_auth(
elem[3])
else:
inf = IM.InfrastructureInfo.InfrastructureInfo.deserialize(
elem[3])
inf_list[inf.id] = inf
except:
InfrastructureList.logger.exception(
"ERROR reading infrastructure from database, ignoring it!."
)
else:
InfrastructureList.logger.warn("No data in database!.")
db.close()
return inf_list
else:
InfrastructureList.logger.error(
"ERROR connecting with the database!.")
return {}
def init_table():
""" Creates de database """
db = DataBase(Config.DATA_DB)
if db.connect():
if not db.table_exists("inf_list"):
InfrastructureList.logger.debug("Creating the IM database!.")
if db.db_type == DataBase.MYSQL:
db.execute("CREATE TABLE inf_list(rowid INTEGER NOT NULL AUTO_INCREMENT UNIQUE,"
" id VARCHAR(255) PRIMARY KEY, deleted INTEGER, date TIMESTAMP, data LONGBLOB)")
elif db.db_type == DataBase.SQLITE:
db.execute("CREATE TABLE inf_list(id VARCHAR(255) PRIMARY KEY, deleted INTEGER,"
" date TIMESTAMP, data LONGBLOB)")
db.close()
return True
else:
InfrastructureList.logger.error("ERROR connecting with the database!.")
return False
def rename_old_data():
db = DataBase(Config.DATA_DB)
if db.connect():
if db.table_exists("inf_list"):
now = str(int(time.time() * 100))
if db.db_type == DataBase.SQLITE:
db.execute('ALTER TABLE inf_list RENAME TO inf_list_%s;' % now)
db.close()
elif db.db_type == DataBase.MYSQL:
db.execute('RENAME TABLE inf_list TO inf_list_%s;' % now)
db.close()
else:
db.close()
sys.stderr.write("ERROR connecting with the database!.")
sys.exit(-1)
else:
db.close()
else:
sys.stderr.write("ERROR connecting with the database!.")
sys.exit(-1)
def get_data_from_db(db_url):
db = DataBase(db_url)
if db.connect():
inf_list = {}
res = db.select("select * from inf_list where deleted = 0 order by id desc")
if len(res) > 0:
for elem in res:
try:
inf = DB150to151.deserialize_info(elem[3])
inf_list[inf.id] = inf
except:
sys.stderr.write("ERROR reading infrastructure from database, ignoring it!.")
else:
sys.stderr.write("No data in database!.")
db.close()
return inf_list
else:
sys.stderr.write("ERROR connecting with the database!.")
sys.exit(-1)
def test_mongo_db(self, mongo):
client = MagicMock()
mongo.return_value = client
database = MagicMock()
client.__getitem__.return_value = database
database.client = client
table = MagicMock()
database.__getitem__.return_value = table
table.database = database
db_url = "mongodb://*****:*****@server/db_name"
db = DataBase(db_url)
self.assertTrue(db.connect())
database.collection_names.return_value = ['table1']
res = db.table_exists("test")
self.assertFalse(res)
res = db.table_exists("table1")
self.assertTrue(res)
res = db.replace('table', {}, {'id': 1, 'data': 'test1'})
self.assertTrue(res)
self.assertEqual(table.replace_one.call_args_list[0][0], ({}, {'data': 'test1', 'id': 1}, True))
table.find.return_value = [{'id': 2, 'data': 'test2', '_id': 2}]
res = db.find('table', {'id': 2}, {'data': True})
self.assertEqual(len(res), 1)
self.assertEqual(table.find.call_args_list[0][0], ({'id': 2}, {'_id': False, 'data': True}))
del_res = MagicMock()
del_res.deleted_count = 1
table.delete_many.return_value = del_res
res = db.delete('table', {'id': 1})
self.assertEqual(res, 1)
self.assertEqual(table.delete_many.call_args_list[0][0], ({'id': 1},))
db.close()
import sys
sys.path.append("..")
sys.path.append(".")
from IM.config import Config
from IM.db import DataBase
if __name__ == "__main__":
if not Config.DATA_DB:
sys.stderr.write("No DATA_DB defined in the im.cfg file!!")
sys.exit(-1)
db = DataBase(Config.DATA_DB)
if db.connect():
if db.table_exists("inf_list"):
if db.db_type == DataBase.MYSQL:
sys.stdout.write("Updating DB: %s.\n" % Config.DATA_DB)
db.execute(
"ALTER TABLE `inf_list` ADD COLUMN `rowid` INT AUTO_INCREMENT UNIQUE FIRST;"
)
else:
sys.stdout.write("SQLite DB does not need to be updated.")
db.close()
else:
sys.stdout.write(
"There are no inf_list table. Do not need to update.")
else:
sys.stderr.write("Error connecting with DB: %s\n" % Config.DATA_DB)
sys.exit(-1)