def __init__(self, config):
clt = AC(
**{p[4:].lower(): config.get(p)
for p in config if p in AC_PARAMS})
self._adb = clt.database(config.get('ADB_NAME'))
self._clt = clt
self._colls = {}
self.log = logging
return
def get_new_connection(self, conn_params)->ArangoClient:
"""Opens a connection to the database.
connection = ArangoClient(
protocol='http',
host='localhost',
port=8529,
username='******',
password='',
enable_logging=True)
"""
database = conn_params.pop('database')
connection = ArangoClient(**conn_params)
database = connection.database(database)
# FIXME: Very dangerous and not intuitive to set a database property here.
# may be better to use an extra layer: A wrapper class that combine the functionality.
self.database = database
return connection
def main():
client = ArangoClient(host='192.168.0.37',
username='******',
password='******')
# create database
try:
client.create_database('CMDB')
except ArangoError:
print('database already created.')
db = client.database('CMDB')
# create collections
collection_list = [
# vertex
'Account',
'Application',
'Disk',
'IpAddress',
'LoadBalance',
'NoSQL',
'RDBMS',
'Server',
# edge
'InstanceAccount',
'InstanceApplication',
'IpaddrInstance',
'LoadBalanceBackendServers'
]
for col in collection_list:
try:
db.create_collection(col)
except ArangoError:
print('[%s] collection already created.' % col)
print('done.')
def __init__(self):
client = ArangoClient(host="192.168.0.37", password="******")
self.db = client.database('CMDB')
from arango import ArangoClient
client = ArangoClient(username='******', password='******')
db = client.database('gestion')
class DB(object):
"""DB"""
collection = None
database = None
graph = None
conn = None
edge = None
def __init__(self):
self._connection()
def _connection(self):
self.username = app.config['ARANGO_USERNAME']
self.password = app.config['ARANGO_PASSWORD']
self.arango_protocol = app.config['ARANGO_PROTOCOL']
self.arango_host = app.config['ARANGO_HOST']
self.arango_port = app.config['ARANGO_PORT']
self._conn = ArangoClient(
protocol=self.arango_protocol,
host=self.arango_host,
port=self.arango_port,
username=self.username,
password=self.password,
enable_logging=True
)
############
# DATABASE #
############
def has_database(self, name=''):
"""Return True if there database"""
try:
database = self._conn.database(name)
database.properties()
except exceptions.DatabasePropertiesError:
return False
else:
return True
def get_database(self, name=''):
"""Return database"""
if not name:
name = app.config['ARANGO_DB']
if self.has_database(name):
self.database = self._conn.database(name)
return self.database
else:
msg = db_err.get(1228).format(name)
raise gmap_exceptions.DatabaseNotExist(msg)
def create_database(self, name=''):
"""Create DB"""
try:
self.database = self._conn.create_database(name)
return self.database
except exceptions.DatabaseCreateError as err:
if err.error_code == 1207:
msg = db_err.get(1207).format(name)
raise gmap_exceptions.DatabaseAlreadyExist(msg)
else:
msg = db_err.get(0).format(name, err.message)
raise gmap_exceptions.DatabaseException(msg)
except Exception as err:
msg = db_err.get(0).format(name, err.message)
raise gmap_exceptions.DatabaseException(msg)
def delete_database(self, name=''):
"""Delete DB"""
try:
self._conn.delete_database(name)
self.database = None
return True
except exceptions.DatabaseDeleteError as err:
if err.error_code == 1228:
msg = db_err.get(1228).format(name)
raise gmap_exceptions.DatabaseNotExist(msg)
else:
msg = db_err.get(0).format(name, err.message)
raise gmap_exceptions.DatabaseException(msg)
except Exception as err:
msg = db_err.get(0).format(name, err.message)
raise gmap_exceptions.DatabaseException(msg)
def search_in_database(self, collection, field, value, offset=0, count=10):
"""Search Document"""
# TODO: To use a better way to search
try:
if field and value:
where = 'FILTER LOWER(doc.`{}`) like "%{}%"'.format(
field, value.lower())
else:
where = ''
cursor = self.database.aql.execute('''
FOR doc IN {} {} LIMIT {}, {} RETURN doc'''.format(
collection, where, offset, count),
count=True,
batch_size=1,
ttl=10,
optimizer_rules=['+all']
)
return cursor
except Exception as err:
msg = db_err.get(1).format(err.message)
raise gmap_exceptions.DatabaseException(msg)
###############
# COLLECTIONS #
###############
def _has_collection(self, name='', edge=False):
"""Return True if there collection/edge"""
try:
collection = self.database.collection(name)
res = collection.properties().get('edge') is edge
# import pdb; pdb.Pdb().set_trace()
# if res is edge:
return res
except exceptions.CollectionPropertiesError:
return False
def has_collection(self, name=''):
"""Return True if there collection"""
return self._has_collection(name, False)
def get_collection(self, name=''):
"""Return Collection"""
if self.has_collection(name):
self.collection = self.database.collection(name)
return self.collection
else:
msg = coll_err.get(1228).format(name)
raise gmap_exceptions.CollectionNotExist(msg)
def create_collection(self, name='', edge=False):
"""Create Collection"""
try:
self.collection = self.database.create_collection(
name=name, edge=False)
return self.collection
except exceptions.CollectionCreateError as err:
if err.error_code == 1207:
msg = coll_err.get(1207).format(name)
raise gmap_exceptions.CollectionAlreadyExist(msg)
else:
msg = coll_err.get(0).format(name, err.message)
raise gmap_exceptions.CollectionException(msg)
except Exception as err:
msg = coll_err.get(0).format(name, err.message)
raise gmap_exceptions.CollectionException(msg)
else:
return True
def delete_collection(self, name=''):
"""Delete Collection """
try:
self.database.delete_collection(name=name)
self.collection = None
return True
except exceptions.CollectionDeleteError as err:
if err.error_code == 1203:
msg = coll_err.get(1228).format(name)
raise gmap_exceptions.CollectionNotExist(msg)
else:
msg = coll_err.get(0).format(name, err.message)
raise gmap_exceptions.CollectionException(msg)
except Exception as err:
msg = coll_err.get(0).format(name, err.message)
raise gmap_exceptions.CollectionException(msg)
def has_edge(self, name=''):
"""Return True if there edge"""
return self._has_collection(name, True)
def get_edge(self, name='', ):
"""Return Edge"""
if self.has_edge(name):
self.edge = self.database.collection(name)
return self.edge
else:
msg = edge_err.get(1228).format(name)
raise gmap_exceptions.EdgeNotExist(msg)
def create_edge(self, name=''):
"""Create Edge"""
try:
self.edge = self.database.create_collection(
name=name, edge=True)
return self.edge
except exceptions.CollectionCreateError as err:
if err.error_code == 1207:
msg = edge_err.get(1207).format(name)
raise gmap_exceptions.EdgeAlreadyExist(msg)
else:
msg = edge_err.get(0).format(name, err.message)
raise gmap_exceptions.EdgeException(msg)
except Exception as err:
msg = edge_err.get(0).format(name, err.message)
raise gmap_exceptions.EdgeException(msg)
def delete_edge(self, name=''):
"""Delete Edge """
try:
self.database.delete_collection(name=name)
self.edge = None
return True
except exceptions.CollectionDeleteError as err:
if err.error_code == 1203:
msg = edge_err.get(1228).format(name)
raise gmap_exceptions.EdgeNotExist(msg)
else:
msg = edge_err.get(0).format(name, err.message)
raise gmap_exceptions.EdgeException(msg)
except Exception as err:
msg = edge_err.get(0).format(name, err.message)
raise gmap_exceptions.EdgeException(msg)
else:
return True
##########
# GRAPHS #
##########
def has_graph(self, name=''):
"""Return True if there graph"""
try:
graph = self.database.graph(name)
graph.properties()
return True
except exceptions.GraphPropertiesError:
return False
def get_graph(self, name=''):
"""Return graph"""
if self.has_graph(name):
self.graph = self.database.graph(name)
return self.graph
else:
msg = gph_err.get(1924).format(name)
raise gmap_exceptions.GraphNotExist(msg)
def create_graph(self, name='', edge_definitions=None):
"""Create Graph"""
try:
self.graph = self.database.create_graph(name)
except exceptions.GraphCreateError as err:
if err.error_code == 1925:
msg = gph_err.get(1925).format(name, err)
raise gmap_exceptions.GraphAlreadyExist(msg)
else:
msg = gph_err.get(0).format(name, err.message)
raise gmap_exceptions.GraphException(msg)
except Exception as err:
msg = gph_err.get(0).format(name, err.message)
raise gmap_exceptions.GraphException(msg)
else:
if edge_definitions:
for edge in edge_definitions:
try:
self.graph.create_edge_definition(
name=edge.get('edge'),
from_collections=edge.get('from_collections'),
to_collections=edge.get('to_collections')
)
except Exception as err:
self.database.delete_graph(name)
msg = gph_err.get(1).format(name)
raise gmap_exceptions.GraphException(msg)
return self.graph
def delete_graph(self, name=''):
"""Delete Graph"""
try:
self.database.delete_graph(name)
return True
except exceptions.GraphDeleteError as err:
if err.error_code == 1924:
msg = gph_err.get(1924).format(name)
raise gmap_exceptions.GraphNotExist(msg)
else:
msg = gph_err.get(0).format(name, err.message)
raise gmap_exceptions.GraphException(msg)
except Exception as err:
msg = gph_err.get(0).format(name, err.message)
raise gmap_exceptions.GraphException(msg)
class ArangoHandler(object):
def __init__(self,
database,
protocol='http',
host='localhost',
port=8529,
username='******',
password=''):
self.BATCH_LIMIT = 10000
self.client = ArangoClient(protocol=protocol,
host=host,
port=port,
username=username,
password=password,
enable_logging=True)
self.db = self.retrieve_db(database)
def retrieve_db(self, database_name):
if self.has_db(database_name):
return self.client.database(database_name)
else:
return self.client.create_database(database_name)
def has_db(self, database_name):
database_list = self.client.databases()
if database_name in database_list:
return True
else:
return False
def insert_jsonl(self, collection_name, lines):
collection = self.retrieve_collection(collection_name)
commit = []
lines = lines.split(b"\n")
count = 0
for line in lines:
try:
dictionary = ujson.loads(line)
except Exception as error:
logging.error(u"Error loading jsonl: {}".format(line))
line = lines.pop()
continue
key_hash = hashlib.md5(line)
dictionary['_key'] = key_hash.hexdigest()
commit.append(dictionary)
if len(commit) == self.BATCH_LIMIT:
collection.import_bulk(commit,
sync=True,
on_duplicate="ignore")
commit[:] = []
count += 1
if len(commit) > 0:
collection.import_bulk(commit, sync=True, on_duplicate="ignore")
logging.info(u"Records Processed: {}".format(count))
def retrieve_collection(self, collection_name):
if self.has_collection(collection_name):
return self.db.collection(collection_name)
else:
return self.db.create_collection(collection_name)
def has_collection(self, collection_name):
collection_list = self.db.collections()
for collection_info in collection_list:
if collection_info['name'] == collection_name:
return True
return False
class DB(object):
"""DB"""
collection = None
database = None
graph = None
conn = None
edge = None
def __init__(self):
self._connection()
def _connection(self):
self.username = app.config['ARANGO_USERNAME']
self.password = app.config['ARANGO_PASSWORD']
self.arango_protocol = app.config['ARANGO_PROTOCOL']
self.arango_host = app.config['ARANGO_HOST']
self.arango_port = app.config['ARANGO_PORT']
self._conn = ArangoClient(protocol=self.arango_protocol,
host=self.arango_host,
port=self.arango_port,
username=self.username,
password=self.password,
enable_logging=True)
############
# DATABASE #
############
def has_database(self, name=''):
"""Return True if there database"""
try:
database = self._conn.database(name)
database.properties()
except exceptions.DatabasePropertiesError:
return False
else:
return True
def get_database(self, name=''):
"""Return database"""
if not name:
name = app.config['ARANGO_DB']
if self.has_database(name):
self.database = self._conn.database(name)
return self.database
else:
msg = db_err.get(1228).format(name)
raise gmap_exceptions.DatabaseNotExist(msg)
def create_database(self, name=''):
"""Create DB"""
try:
self.database = self._conn.create_database(name)
return self.database
except exceptions.DatabaseCreateError as err:
if err.error_code == 1207:
msg = db_err.get(1207).format(name)
raise gmap_exceptions.DatabaseAlreadyExist(msg)
else:
msg = db_err.get(0).format(name, err.message)
raise gmap_exceptions.DatabaseException(msg)
except Exception as err:
msg = db_err.get(0).format(name, err.message)
raise gmap_exceptions.DatabaseException(msg)
def delete_database(self, name=''):
"""Delete DB"""
try:
self._conn.delete_database(name)
self.database = None
return True
except exceptions.DatabaseDeleteError as err:
if err.error_code == 1228:
msg = db_err.get(1228).format(name)
raise gmap_exceptions.DatabaseNotExist(msg)
else:
msg = db_err.get(0).format(name, err.message)
raise gmap_exceptions.DatabaseException(msg)
except Exception as err:
msg = db_err.get(0).format(name, err.message)
raise gmap_exceptions.DatabaseException(msg)
def search_in_database(self, collection, field, value, offset=0, count=10):
"""Search Document"""
# TODO: To use a better way to search
try:
if field and value:
where = 'FILTER LOWER(doc.`{}`) like "%{}%"'.format(
field, value.lower())
else:
where = ''
cursor = self.database.aql.execute('''
FOR doc IN {} {} LIMIT {}, {} RETURN doc'''.format(
collection, where, offset, count),
count=True,
batch_size=1,
ttl=10,
optimizer_rules=['+all'])
return cursor
except Exception as err:
msg = db_err.get(1).format(err.message)
raise gmap_exceptions.DatabaseException(msg)
###############
# COLLECTIONS #
###############
def _has_collection(self, name='', edge=False):
"""Return True if there collection/edge"""
try:
collection = self.database.collection(name)
res = collection.properties().get('edge') is edge
# import pdb; pdb.Pdb().set_trace()
# if res is edge:
return res
except exceptions.CollectionPropertiesError:
return False
def has_collection(self, name=''):
"""Return True if there collection"""
return self._has_collection(name, False)
def get_collection(self, name=''):
"""Return Collection"""
if self.has_collection(name):
self.collection = self.database.collection(name)
return self.collection
else:
msg = coll_err.get(1228).format(name)
raise gmap_exceptions.CollectionNotExist(msg)
def create_collection(self, name='', edge=False):
"""Create Collection"""
try:
self.collection = self.database.create_collection(name=name,
edge=False)
return self.collection
except exceptions.CollectionCreateError as err:
if err.error_code == 1207:
msg = coll_err.get(1207).format(name)
raise gmap_exceptions.CollectionAlreadyExist(msg)
else:
msg = coll_err.get(0).format(name, err.message)
raise gmap_exceptions.CollectionException(msg)
except Exception as err:
msg = coll_err.get(0).format(name, err.message)
raise gmap_exceptions.CollectionException(msg)
else:
return True
def delete_collection(self, name=''):
"""Delete Collection """
try:
self.database.delete_collection(name=name)
self.collection = None
return True
except exceptions.CollectionDeleteError as err:
if err.error_code == 1203:
msg = coll_err.get(1228).format(name)
raise gmap_exceptions.CollectionNotExist(msg)
else:
msg = coll_err.get(0).format(name, err.message)
raise gmap_exceptions.CollectionException(msg)
except Exception as err:
msg = coll_err.get(0).format(name, err.message)
raise gmap_exceptions.CollectionException(msg)
def has_edge(self, name=''):
"""Return True if there edge"""
return self._has_collection(name, True)
def get_edge(
self,
name='',
):
"""Return Edge"""
if self.has_edge(name):
self.edge = self.database.collection(name)
return self.edge
else:
msg = edge_err.get(1228).format(name)
raise gmap_exceptions.EdgeNotExist(msg)
def create_edge(self, name=''):
"""Create Edge"""
try:
self.edge = self.database.create_collection(name=name, edge=True)
return self.edge
except exceptions.CollectionCreateError as err:
if err.error_code == 1207:
msg = edge_err.get(1207).format(name)
raise gmap_exceptions.EdgeAlreadyExist(msg)
else:
msg = edge_err.get(0).format(name, err.message)
raise gmap_exceptions.EdgeException(msg)
except Exception as err:
msg = edge_err.get(0).format(name, err.message)
raise gmap_exceptions.EdgeException(msg)
def delete_edge(self, name=''):
"""Delete Edge """
try:
self.database.delete_collection(name=name)
self.edge = None
return True
except exceptions.CollectionDeleteError as err:
if err.error_code == 1203:
msg = edge_err.get(1228).format(name)
raise gmap_exceptions.EdgeNotExist(msg)
else:
msg = edge_err.get(0).format(name, err.message)
raise gmap_exceptions.EdgeException(msg)
except Exception as err:
msg = edge_err.get(0).format(name, err.message)
raise gmap_exceptions.EdgeException(msg)
else:
return True
##########
# GRAPHS #
##########
def has_graph(self, name=''):
"""Return True if there graph"""
try:
graph = self.database.graph(name)
graph.properties()
return True
except exceptions.GraphPropertiesError:
return False
def get_graph(self, name=''):
"""Return graph"""
if self.has_graph(name):
self.graph = self.database.graph(name)
return self.graph
else:
msg = gph_err.get(1924).format(name)
raise gmap_exceptions.GraphNotExist(msg)
def create_graph(self, name='', edge_definitions=None):
"""Create Graph"""
try:
self.graph = self.database.create_graph(name)
except exceptions.GraphCreateError as err:
if err.error_code == 1925:
msg = gph_err.get(1925).format(name, err)
raise gmap_exceptions.GraphAlreadyExist(msg)
else:
msg = gph_err.get(0).format(name, err.message)
raise gmap_exceptions.GraphException(msg)
except Exception as err:
msg = gph_err.get(0).format(name, err.message)
raise gmap_exceptions.GraphException(msg)
else:
if edge_definitions:
for edge in edge_definitions:
try:
self.graph.create_edge_definition(
name=edge.get('edge'),
from_collections=edge.get('from_collections'),
to_collections=edge.get('to_collections'))
except Exception as err:
self.database.delete_graph(name)
msg = gph_err.get(1).format(name)
raise gmap_exceptions.GraphException(msg)
return self.graph
def delete_graph(self, name=''):
"""Delete Graph"""
try:
self.database.delete_graph(name)
return True
except exceptions.GraphDeleteError as err:
if err.error_code == 1924:
msg = gph_err.get(1924).format(name)
raise gmap_exceptions.GraphNotExist(msg)
else:
msg = gph_err.get(0).format(name, err.message)
raise gmap_exceptions.GraphException(msg)
except Exception as err:
msg = gph_err.get(0).format(name, err.message)
raise gmap_exceptions.GraphException(msg)
