def create_database() -> None:
try:
# try to access the system database with default credentials.
# this only works if arango has been started with default settings.
http_client = ArangoHTTPClient(args.graphdb_request_timeout,
not args.graphdb_no_ssl_verify)
root_pw = args.graphdb_root_password
secure_root = not args.graphdb_bootstrap_do_not_secure
root_db = ArangoClient(
hosts=args.graphdb_server,
http_client=http_client).db(password=root_pw)
root_db.echo(
) # this call will fail, if we are not allowed to access the system db
user = args.graphdb_username
passwd = args.graphdb_password
database = args.graphdb_database
change = False
if not root_db.has_user(user):
log.info(
"Configured graph db user does not exist. Create it.")
root_db.create_user(user, passwd, active=True)
change = True
if not root_db.has_database(database):
log.info(
"Configured graph db database does not exist. Create it."
)
root_db.create_database(
database,
[{
"username": user,
"password": passwd,
"active": True,
"extra": {
"generated": "resoto"
}
}],
)
change = True
if change and secure_root and root_pw == "" and passwd != "" and passwd not in {
"test"
}:
root_db.replace_user("root", passwd, True)
log.info(
"Database is using an empty password. "
"Secure the root account with the provided user password. "
"Login to the Resoto database via provided username and password. "
"Login to the System database via `root` and provided password!"
)
if not change:
log.info(
"Not allowed to access database, while user and database exist. Wrong password?"
)
except Exception as ex:
log.error(
"Database or user does not exist or does not have enough permissions. "
f"Attempt to create user/database via default system account is not possible. Reason: {ex}. "
"You can provide the password of the root user via --graphdb-root-password to setup "
"a Resoto user and database automatically.")
def setup_database_test(request):
username = os.environ.get('ARANGO_USERNAME', "test")
password = os.environ.get('ARANGO_PASSWORD', "test")
arango_hosts = os.environ.get('ARANGO_HOSTS', "http://arangodb:8529")
database_name = os.environ.get('ARANGO_DATABASE', "test")
sys_db = ArangoClient(hosts=arango_hosts).db('_system',
username=username,
password=password)
try:
sys_db.create_database(database_name)
except DatabaseCreateError as exp:
log.info("Could not create the test db, probably already exists. %s",
str(exp))
yield
def arangoConnect(getIp, getPort, getUser, getPass, getNewdb):
client = ArangoClient(protocol='http',
host=getIp,
port=getPort,
username=getUser,
password=getPass,
enable_logging=True)
print("connected?")
try:
client.create_database(getNewdb)
print('Created new db, ' + getNewdb)
return (client.databases())
except:
print('failed to create new DB, ' + getNewdb)
print(client.databases())
return (client.databases())
def main():
client = ArangoClient(host='192.168.0.37',
username='******',
password='******')
# create database
try:
client.create_database('CMDB')
except ArangoError:
print('database already created.')
db = client.database('CMDB')
# create collections
collection_list = [
# vertex
'Account',
'Application',
'Disk',
'IpAddress',
'LoadBalance',
'NoSQL',
'RDBMS',
'Server',
# edge
'InstanceAccount',
'InstanceApplication',
'IpaddrInstance',
'LoadBalanceBackendServers'
]
for col in collection_list:
try:
db.create_collection(col)
except ArangoError:
print('[%s] collection already created.' % col)
print('done.')
AsyncJobClearError,
AsyncJobResultError,
AsyncJobStatusError,
AsyncJobListError,
AQLQueryExecuteError
)
from arango.graph import Graph
from .utils import (
generate_db_name,
generate_col_name
)
arango_client = ArangoClient()
db_name = generate_db_name(arango_client)
db = arango_client.create_database(db_name)
col_name = generate_col_name(db)
col = db.create_collection(col_name)
col.add_fulltext_index(fields=['val'])
def teardown_module(*_):
arango_client.delete_database(db_name, ignore_missing=True)
def setup_function(*_):
col.truncate()
def wait_on_job(job):
while job.status() == 'pending':
from arango import ArangoClient
from datetime import tzinfo
from db_credentials import db_credentials
# Configure your ArangoDB server connection here
conn = ArangoClient(protocol=db_credentials['protocol'],
host=db_credentials['host'],
port=db_credentials['port'],
username=db_credentials['username'],
password=db_credentials['password'])
dbName = 'polBotCheck'
if dbName not in conn.databases():
db = conn.create_database(dbName)
db = conn.db(dbName)
def getCollection(collectionName, edge=False):
collectionNames = map(lambda c: c['name'], db.collections())
if collectionName not in collectionNames:
db.create_collection(collectionName, edge=edge)
return db.collection(collectionName)
def getVertexCollection(graph, collectionName):
collectionNames = map(lambda c: c['name'], db.collections())
if collectionName not in collectionNames:
graph.create_vertex_collection(collectionName)
return db.collection(collectionName)
class DB(object):
"""DB"""
collection = None
database = None
graph = None
conn = None
edge = None
def __init__(self):
self._connection()
def _connection(self):
self.username = app.config['ARANGO_USERNAME']
self.password = app.config['ARANGO_PASSWORD']
self.arango_protocol = app.config['ARANGO_PROTOCOL']
self.arango_host = app.config['ARANGO_HOST']
self.arango_port = app.config['ARANGO_PORT']
self._conn = ArangoClient(
protocol=self.arango_protocol,
host=self.arango_host,
port=self.arango_port,
username=self.username,
password=self.password,
enable_logging=True
)
############
# DATABASE #
############
def has_database(self, name=''):
"""Return True if there database"""
try:
database = self._conn.database(name)
database.properties()
except exceptions.DatabasePropertiesError:
return False
else:
return True
def get_database(self, name=''):
"""Return database"""
if not name:
name = app.config['ARANGO_DB']
if self.has_database(name):
self.database = self._conn.database(name)
return self.database
else:
msg = db_err.get(1228).format(name)
raise gmap_exceptions.DatabaseNotExist(msg)
def create_database(self, name=''):
"""Create DB"""
try:
self.database = self._conn.create_database(name)
return self.database
except exceptions.DatabaseCreateError as err:
if err.error_code == 1207:
msg = db_err.get(1207).format(name)
raise gmap_exceptions.DatabaseAlreadyExist(msg)
else:
msg = db_err.get(0).format(name, err.message)
raise gmap_exceptions.DatabaseException(msg)
except Exception as err:
msg = db_err.get(0).format(name, err.message)
raise gmap_exceptions.DatabaseException(msg)
def delete_database(self, name=''):
"""Delete DB"""
try:
self._conn.delete_database(name)
self.database = None
return True
except exceptions.DatabaseDeleteError as err:
if err.error_code == 1228:
msg = db_err.get(1228).format(name)
raise gmap_exceptions.DatabaseNotExist(msg)
else:
msg = db_err.get(0).format(name, err.message)
raise gmap_exceptions.DatabaseException(msg)
except Exception as err:
msg = db_err.get(0).format(name, err.message)
raise gmap_exceptions.DatabaseException(msg)
def search_in_database(self, collection, field, value, offset=0, count=10):
"""Search Document"""
# TODO: To use a better way to search
try:
if field and value:
where = 'FILTER LOWER(doc.`{}`) like "%{}%"'.format(
field, value.lower())
else:
where = ''
cursor = self.database.aql.execute('''
FOR doc IN {} {} LIMIT {}, {} RETURN doc'''.format(
collection, where, offset, count),
count=True,
batch_size=1,
ttl=10,
optimizer_rules=['+all']
)
return cursor
except Exception as err:
msg = db_err.get(1).format(err.message)
raise gmap_exceptions.DatabaseException(msg)
###############
# COLLECTIONS #
###############
def _has_collection(self, name='', edge=False):
"""Return True if there collection/edge"""
try:
collection = self.database.collection(name)
res = collection.properties().get('edge') is edge
# import pdb; pdb.Pdb().set_trace()
# if res is edge:
return res
except exceptions.CollectionPropertiesError:
return False
def has_collection(self, name=''):
"""Return True if there collection"""
return self._has_collection(name, False)
def get_collection(self, name=''):
"""Return Collection"""
if self.has_collection(name):
self.collection = self.database.collection(name)
return self.collection
else:
msg = coll_err.get(1228).format(name)
raise gmap_exceptions.CollectionNotExist(msg)
def create_collection(self, name='', edge=False):
"""Create Collection"""
try:
self.collection = self.database.create_collection(
name=name, edge=False)
return self.collection
except exceptions.CollectionCreateError as err:
if err.error_code == 1207:
msg = coll_err.get(1207).format(name)
raise gmap_exceptions.CollectionAlreadyExist(msg)
else:
msg = coll_err.get(0).format(name, err.message)
raise gmap_exceptions.CollectionException(msg)
except Exception as err:
msg = coll_err.get(0).format(name, err.message)
raise gmap_exceptions.CollectionException(msg)
else:
return True
def delete_collection(self, name=''):
"""Delete Collection """
try:
self.database.delete_collection(name=name)
self.collection = None
return True
except exceptions.CollectionDeleteError as err:
if err.error_code == 1203:
msg = coll_err.get(1228).format(name)
raise gmap_exceptions.CollectionNotExist(msg)
else:
msg = coll_err.get(0).format(name, err.message)
raise gmap_exceptions.CollectionException(msg)
except Exception as err:
msg = coll_err.get(0).format(name, err.message)
raise gmap_exceptions.CollectionException(msg)
def has_edge(self, name=''):
"""Return True if there edge"""
return self._has_collection(name, True)
def get_edge(self, name='', ):
"""Return Edge"""
if self.has_edge(name):
self.edge = self.database.collection(name)
return self.edge
else:
msg = edge_err.get(1228).format(name)
raise gmap_exceptions.EdgeNotExist(msg)
def create_edge(self, name=''):
"""Create Edge"""
try:
self.edge = self.database.create_collection(
name=name, edge=True)
return self.edge
except exceptions.CollectionCreateError as err:
if err.error_code == 1207:
msg = edge_err.get(1207).format(name)
raise gmap_exceptions.EdgeAlreadyExist(msg)
else:
msg = edge_err.get(0).format(name, err.message)
raise gmap_exceptions.EdgeException(msg)
except Exception as err:
msg = edge_err.get(0).format(name, err.message)
raise gmap_exceptions.EdgeException(msg)
def delete_edge(self, name=''):
"""Delete Edge """
try:
self.database.delete_collection(name=name)
self.edge = None
return True
except exceptions.CollectionDeleteError as err:
if err.error_code == 1203:
msg = edge_err.get(1228).format(name)
raise gmap_exceptions.EdgeNotExist(msg)
else:
msg = edge_err.get(0).format(name, err.message)
raise gmap_exceptions.EdgeException(msg)
except Exception as err:
msg = edge_err.get(0).format(name, err.message)
raise gmap_exceptions.EdgeException(msg)
else:
return True
##########
# GRAPHS #
##########
def has_graph(self, name=''):
"""Return True if there graph"""
try:
graph = self.database.graph(name)
graph.properties()
return True
except exceptions.GraphPropertiesError:
return False
def get_graph(self, name=''):
"""Return graph"""
if self.has_graph(name):
self.graph = self.database.graph(name)
return self.graph
else:
msg = gph_err.get(1924).format(name)
raise gmap_exceptions.GraphNotExist(msg)
def create_graph(self, name='', edge_definitions=None):
"""Create Graph"""
try:
self.graph = self.database.create_graph(name)
except exceptions.GraphCreateError as err:
if err.error_code == 1925:
msg = gph_err.get(1925).format(name, err)
raise gmap_exceptions.GraphAlreadyExist(msg)
else:
msg = gph_err.get(0).format(name, err.message)
raise gmap_exceptions.GraphException(msg)
except Exception as err:
msg = gph_err.get(0).format(name, err.message)
raise gmap_exceptions.GraphException(msg)
else:
if edge_definitions:
for edge in edge_definitions:
try:
self.graph.create_edge_definition(
name=edge.get('edge'),
from_collections=edge.get('from_collections'),
to_collections=edge.get('to_collections')
)
except Exception as err:
self.database.delete_graph(name)
msg = gph_err.get(1).format(name)
raise gmap_exceptions.GraphException(msg)
return self.graph
def delete_graph(self, name=''):
"""Delete Graph"""
try:
self.database.delete_graph(name)
return True
except exceptions.GraphDeleteError as err:
if err.error_code == 1924:
msg = gph_err.get(1924).format(name)
raise gmap_exceptions.GraphNotExist(msg)
else:
msg = gph_err.get(0).format(name, err.message)
raise gmap_exceptions.GraphException(msg)
except Exception as err:
msg = gph_err.get(0).format(name, err.message)
raise gmap_exceptions.GraphException(msg)
class ArangoHandler(object):
def __init__(self,
database,
protocol='http',
host='localhost',
port=8529,
username='******',
password=''):
self.BATCH_LIMIT = 10000
self.client = ArangoClient(protocol=protocol,
host=host,
port=port,
username=username,
password=password,
enable_logging=True)
self.db = self.retrieve_db(database)
def retrieve_db(self, database_name):
if self.has_db(database_name):
return self.client.database(database_name)
else:
return self.client.create_database(database_name)
def has_db(self, database_name):
database_list = self.client.databases()
if database_name in database_list:
return True
else:
return False
def insert_jsonl(self, collection_name, lines):
collection = self.retrieve_collection(collection_name)
commit = []
lines = lines.split(b"\n")
count = 0
for line in lines:
try:
dictionary = ujson.loads(line)
except Exception as error:
logging.error(u"Error loading jsonl: {}".format(line))
line = lines.pop()
continue
key_hash = hashlib.md5(line)
dictionary['_key'] = key_hash.hexdigest()
commit.append(dictionary)
if len(commit) == self.BATCH_LIMIT:
collection.import_bulk(commit,
sync=True,
on_duplicate="ignore")
commit[:] = []
count += 1
if len(commit) > 0:
collection.import_bulk(commit, sync=True, on_duplicate="ignore")
logging.info(u"Records Processed: {}".format(count))
def retrieve_collection(self, collection_name):
if self.has_collection(collection_name):
return self.db.collection(collection_name)
else:
return self.db.create_collection(collection_name)
def has_collection(self, collection_name):
collection_list = self.db.collections()
for collection_info in collection_list:
if collection_info['name'] == collection_name:
return True
return False
class DB(object):
"""DB"""
collection = None
database = None
graph = None
conn = None
edge = None
def __init__(self):
self._connection()
def _connection(self):
self.username = app.config['ARANGO_USERNAME']
self.password = app.config['ARANGO_PASSWORD']
self.arango_protocol = app.config['ARANGO_PROTOCOL']
self.arango_host = app.config['ARANGO_HOST']
self.arango_port = app.config['ARANGO_PORT']
self._conn = ArangoClient(protocol=self.arango_protocol,
host=self.arango_host,
port=self.arango_port,
username=self.username,
password=self.password,
enable_logging=True)
############
# DATABASE #
############
def has_database(self, name=''):
"""Return True if there database"""
try:
database = self._conn.database(name)
database.properties()
except exceptions.DatabasePropertiesError:
return False
else:
return True
def get_database(self, name=''):
"""Return database"""
if not name:
name = app.config['ARANGO_DB']
if self.has_database(name):
self.database = self._conn.database(name)
return self.database
else:
msg = db_err.get(1228).format(name)
raise gmap_exceptions.DatabaseNotExist(msg)
def create_database(self, name=''):
"""Create DB"""
try:
self.database = self._conn.create_database(name)
return self.database
except exceptions.DatabaseCreateError as err:
if err.error_code == 1207:
msg = db_err.get(1207).format(name)
raise gmap_exceptions.DatabaseAlreadyExist(msg)
else:
msg = db_err.get(0).format(name, err.message)
raise gmap_exceptions.DatabaseException(msg)
except Exception as err:
msg = db_err.get(0).format(name, err.message)
raise gmap_exceptions.DatabaseException(msg)
def delete_database(self, name=''):
"""Delete DB"""
try:
self._conn.delete_database(name)
self.database = None
return True
except exceptions.DatabaseDeleteError as err:
if err.error_code == 1228:
msg = db_err.get(1228).format(name)
raise gmap_exceptions.DatabaseNotExist(msg)
else:
msg = db_err.get(0).format(name, err.message)
raise gmap_exceptions.DatabaseException(msg)
except Exception as err:
msg = db_err.get(0).format(name, err.message)
raise gmap_exceptions.DatabaseException(msg)
def search_in_database(self, collection, field, value, offset=0, count=10):
"""Search Document"""
# TODO: To use a better way to search
try:
if field and value:
where = 'FILTER LOWER(doc.`{}`) like "%{}%"'.format(
field, value.lower())
else:
where = ''
cursor = self.database.aql.execute('''
FOR doc IN {} {} LIMIT {}, {} RETURN doc'''.format(
collection, where, offset, count),
count=True,
batch_size=1,
ttl=10,
optimizer_rules=['+all'])
return cursor
except Exception as err:
msg = db_err.get(1).format(err.message)
raise gmap_exceptions.DatabaseException(msg)
###############
# COLLECTIONS #
###############
def _has_collection(self, name='', edge=False):
"""Return True if there collection/edge"""
try:
collection = self.database.collection(name)
res = collection.properties().get('edge') is edge
# import pdb; pdb.Pdb().set_trace()
# if res is edge:
return res
except exceptions.CollectionPropertiesError:
return False
def has_collection(self, name=''):
"""Return True if there collection"""
return self._has_collection(name, False)
def get_collection(self, name=''):
"""Return Collection"""
if self.has_collection(name):
self.collection = self.database.collection(name)
return self.collection
else:
msg = coll_err.get(1228).format(name)
raise gmap_exceptions.CollectionNotExist(msg)
def create_collection(self, name='', edge=False):
"""Create Collection"""
try:
self.collection = self.database.create_collection(name=name,
edge=False)
return self.collection
except exceptions.CollectionCreateError as err:
if err.error_code == 1207:
msg = coll_err.get(1207).format(name)
raise gmap_exceptions.CollectionAlreadyExist(msg)
else:
msg = coll_err.get(0).format(name, err.message)
raise gmap_exceptions.CollectionException(msg)
except Exception as err:
msg = coll_err.get(0).format(name, err.message)
raise gmap_exceptions.CollectionException(msg)
else:
return True
def delete_collection(self, name=''):
"""Delete Collection """
try:
self.database.delete_collection(name=name)
self.collection = None
return True
except exceptions.CollectionDeleteError as err:
if err.error_code == 1203:
msg = coll_err.get(1228).format(name)
raise gmap_exceptions.CollectionNotExist(msg)
else:
msg = coll_err.get(0).format(name, err.message)
raise gmap_exceptions.CollectionException(msg)
except Exception as err:
msg = coll_err.get(0).format(name, err.message)
raise gmap_exceptions.CollectionException(msg)
def has_edge(self, name=''):
"""Return True if there edge"""
return self._has_collection(name, True)
def get_edge(
self,
name='',
):
"""Return Edge"""
if self.has_edge(name):
self.edge = self.database.collection(name)
return self.edge
else:
msg = edge_err.get(1228).format(name)
raise gmap_exceptions.EdgeNotExist(msg)
def create_edge(self, name=''):
"""Create Edge"""
try:
self.edge = self.database.create_collection(name=name, edge=True)
return self.edge
except exceptions.CollectionCreateError as err:
if err.error_code == 1207:
msg = edge_err.get(1207).format(name)
raise gmap_exceptions.EdgeAlreadyExist(msg)
else:
msg = edge_err.get(0).format(name, err.message)
raise gmap_exceptions.EdgeException(msg)
except Exception as err:
msg = edge_err.get(0).format(name, err.message)
raise gmap_exceptions.EdgeException(msg)
def delete_edge(self, name=''):
"""Delete Edge """
try:
self.database.delete_collection(name=name)
self.edge = None
return True
except exceptions.CollectionDeleteError as err:
if err.error_code == 1203:
msg = edge_err.get(1228).format(name)
raise gmap_exceptions.EdgeNotExist(msg)
else:
msg = edge_err.get(0).format(name, err.message)
raise gmap_exceptions.EdgeException(msg)
except Exception as err:
msg = edge_err.get(0).format(name, err.message)
raise gmap_exceptions.EdgeException(msg)
else:
return True
##########
# GRAPHS #
##########
def has_graph(self, name=''):
"""Return True if there graph"""
try:
graph = self.database.graph(name)
graph.properties()
return True
except exceptions.GraphPropertiesError:
return False
def get_graph(self, name=''):
"""Return graph"""
if self.has_graph(name):
self.graph = self.database.graph(name)
return self.graph
else:
msg = gph_err.get(1924).format(name)
raise gmap_exceptions.GraphNotExist(msg)
def create_graph(self, name='', edge_definitions=None):
"""Create Graph"""
try:
self.graph = self.database.create_graph(name)
except exceptions.GraphCreateError as err:
if err.error_code == 1925:
msg = gph_err.get(1925).format(name, err)
raise gmap_exceptions.GraphAlreadyExist(msg)
else:
msg = gph_err.get(0).format(name, err.message)
raise gmap_exceptions.GraphException(msg)
except Exception as err:
msg = gph_err.get(0).format(name, err.message)
raise gmap_exceptions.GraphException(msg)
else:
if edge_definitions:
for edge in edge_definitions:
try:
self.graph.create_edge_definition(
name=edge.get('edge'),
from_collections=edge.get('from_collections'),
to_collections=edge.get('to_collections'))
except Exception as err:
self.database.delete_graph(name)
msg = gph_err.get(1).format(name)
raise gmap_exceptions.GraphException(msg)
return self.graph
def delete_graph(self, name=''):
"""Delete Graph"""
try:
self.database.delete_graph(name)
return True
except exceptions.GraphDeleteError as err:
if err.error_code == 1924:
msg = gph_err.get(1924).format(name)
raise gmap_exceptions.GraphNotExist(msg)
else:
msg = gph_err.get(0).format(name, err.message)
raise gmap_exceptions.GraphException(msg)
except Exception as err:
msg = gph_err.get(0).format(name, err.message)
raise gmap_exceptions.GraphException(msg)
if len(header_line) > 1:
# each header line
for line in header_line:
val = line.split(r'=')
# add the id as the key
if val[0] == 'ID':
# set the
elm_ref = structure[val[1]] = {}
else:
# for this id, add the values
elm_ref[val[0].lower()] = re.sub('^\"|\"?$', '', val[1])
if len(sys.argv) > 2:
# Initialize the client for ArangoDB
client = ArangoClient(protocol='http',
host='localhost',
port=8529,
username='******',
password='******',
enable_logging=True)
if sys.argv[2] not in client.databases():
client.create_database(sys.argv[2])
# select the database
db = client.db(sys.argv[2])
# create our obj
loader = hapmap_load()
# start loading variants
loader.load_vcf(sys.argv[1], db)
else:
print "please specify a VCF and arango database"
delete_database - delete test database
""".format(sys.argv[0])
db_name = 'db_test'
username = '******'
password = ''
client = ArangoClient(username=username, password=password, verify=True)
# try:
# client.delete_database(name=db_name)
# except:
# pass
try:
test_db = client.create_database(name=db_name)
except:
test_db = client.db(db_name)
db = Database(test_db)
uni_graph = UniversityGraph(connection=db)
if len(sys.argv) > 1:
if 'test_data_create' == sys.argv[1]:
db.create_graph(uni_graph)
data_create()
elif 'drop_graph' == sys.argv[1]:
db.drop_graph(uni_graph)
elif 'delete_database' == sys.argv[1]:
client.delete_database(name=db_name)
from arango import ArangoClient
# Initialize the client for ArangoDB.
client = ArangoClient(protocol='http', host='localhost', port=8529)
try:
db = client.create_database('aops_database', username=None, password=None)
except:
client.delete_database('aops_database')
db = client.create_database('aops_database', username=None, password=None)
# Define a new graph
metadata = db.create_graph('operational_metadata')
fs_engines = metadata.create_vertex_collection('fs_engines')
er_models = metadata.create_vertex_collection('er_models')
model_code = metadata.create_vertex_collection('model_code')
datasets = metadata.create_vertex_collection('datasets')
edges = metadata.create_edge_definition(
name='edges',
from_collections=['fs_engines', 'er_models', 'model_code', 'datasets'],
to_collections=['fs_engines', 'er_models', 'model_code', 'datasets'])
# Insert vertices into the graph
#fs_engines.insert({'_key': 'engine-1'})
fs_engines.insert({
'_key': 'engine-2',
'uri': 'https://hub.docker.com/r/fastscore/engine/',
'tag': '1.7.1'
})
fs_engines.insert({
'_key': 'engine-3',
'uri': 'https://hub.docker.com/r/fastscore/engine/',
