def create_database() -> None: try: # try to access the system database with default credentials. # this only works if arango has been started with default settings. http_client = ArangoHTTPClient(args.graphdb_request_timeout, not args.graphdb_no_ssl_verify) root_pw = args.graphdb_root_password secure_root = not args.graphdb_bootstrap_do_not_secure root_db = ArangoClient( hosts=args.graphdb_server, http_client=http_client).db(password=root_pw) root_db.echo( ) # this call will fail, if we are not allowed to access the system db user = args.graphdb_username passwd = args.graphdb_password database = args.graphdb_database change = False if not root_db.has_user(user): log.info( "Configured graph db user does not exist. Create it.") root_db.create_user(user, passwd, active=True) change = True if not root_db.has_database(database): log.info( "Configured graph db database does not exist. Create it." ) root_db.create_database( database, [{ "username": user, "password": passwd, "active": True, "extra": { "generated": "resoto" } }], ) change = True if change and secure_root and root_pw == "" and passwd != "" and passwd not in { "test" }: root_db.replace_user("root", passwd, True) log.info( "Database is using an empty password. " "Secure the root account with the provided user password. " "Login to the Resoto database via provided username and password. " "Login to the System database via `root` and provided password!" ) if not change: log.info( "Not allowed to access database, while user and database exist. Wrong password?" ) except Exception as ex: log.error( "Database or user does not exist or does not have enough permissions. " f"Attempt to create user/database via default system account is not possible. Reason: {ex}. " "You can provide the password of the root user via --graphdb-root-password to setup " "a Resoto user and database automatically.")
def setup_database_test(request): username = os.environ.get('ARANGO_USERNAME', "test") password = os.environ.get('ARANGO_PASSWORD', "test") arango_hosts = os.environ.get('ARANGO_HOSTS', "http://arangodb:8529") database_name = os.environ.get('ARANGO_DATABASE', "test") sys_db = ArangoClient(hosts=arango_hosts).db('_system', username=username, password=password) try: sys_db.create_database(database_name) except DatabaseCreateError as exp: log.info("Could not create the test db, probably already exists. %s", str(exp)) yield
def arangoConnect(getIp, getPort, getUser, getPass, getNewdb): client = ArangoClient(protocol='http', host=getIp, port=getPort, username=getUser, password=getPass, enable_logging=True) print("connected?") try: client.create_database(getNewdb) print('Created new db, ' + getNewdb) return (client.databases()) except: print('failed to create new DB, ' + getNewdb) print(client.databases()) return (client.databases())
def main(): client = ArangoClient(host='192.168.0.37', username='******', password='******') # create database try: client.create_database('CMDB') except ArangoError: print('database already created.') db = client.database('CMDB') # create collections collection_list = [ # vertex 'Account', 'Application', 'Disk', 'IpAddress', 'LoadBalance', 'NoSQL', 'RDBMS', 'Server', # edge 'InstanceAccount', 'InstanceApplication', 'IpaddrInstance', 'LoadBalanceBackendServers' ] for col in collection_list: try: db.create_collection(col) except ArangoError: print('[%s] collection already created.' % col) print('done.')
AsyncJobClearError, AsyncJobResultError, AsyncJobStatusError, AsyncJobListError, AQLQueryExecuteError ) from arango.graph import Graph from .utils import ( generate_db_name, generate_col_name ) arango_client = ArangoClient() db_name = generate_db_name(arango_client) db = arango_client.create_database(db_name) col_name = generate_col_name(db) col = db.create_collection(col_name) col.add_fulltext_index(fields=['val']) def teardown_module(*_): arango_client.delete_database(db_name, ignore_missing=True) def setup_function(*_): col.truncate() def wait_on_job(job): while job.status() == 'pending':
from arango import ArangoClient from datetime import tzinfo from db_credentials import db_credentials # Configure your ArangoDB server connection here conn = ArangoClient(protocol=db_credentials['protocol'], host=db_credentials['host'], port=db_credentials['port'], username=db_credentials['username'], password=db_credentials['password']) dbName = 'polBotCheck' if dbName not in conn.databases(): db = conn.create_database(dbName) db = conn.db(dbName) def getCollection(collectionName, edge=False): collectionNames = map(lambda c: c['name'], db.collections()) if collectionName not in collectionNames: db.create_collection(collectionName, edge=edge) return db.collection(collectionName) def getVertexCollection(graph, collectionName): collectionNames = map(lambda c: c['name'], db.collections()) if collectionName not in collectionNames: graph.create_vertex_collection(collectionName) return db.collection(collectionName)
class DB(object): """DB""" collection = None database = None graph = None conn = None edge = None def __init__(self): self._connection() def _connection(self): self.username = app.config['ARANGO_USERNAME'] self.password = app.config['ARANGO_PASSWORD'] self.arango_protocol = app.config['ARANGO_PROTOCOL'] self.arango_host = app.config['ARANGO_HOST'] self.arango_port = app.config['ARANGO_PORT'] self._conn = ArangoClient( protocol=self.arango_protocol, host=self.arango_host, port=self.arango_port, username=self.username, password=self.password, enable_logging=True ) ############ # DATABASE # ############ def has_database(self, name=''): """Return True if there database""" try: database = self._conn.database(name) database.properties() except exceptions.DatabasePropertiesError: return False else: return True def get_database(self, name=''): """Return database""" if not name: name = app.config['ARANGO_DB'] if self.has_database(name): self.database = self._conn.database(name) return self.database else: msg = db_err.get(1228).format(name) raise gmap_exceptions.DatabaseNotExist(msg) def create_database(self, name=''): """Create DB""" try: self.database = self._conn.create_database(name) return self.database except exceptions.DatabaseCreateError as err: if err.error_code == 1207: msg = db_err.get(1207).format(name) raise gmap_exceptions.DatabaseAlreadyExist(msg) else: msg = db_err.get(0).format(name, err.message) raise gmap_exceptions.DatabaseException(msg) except Exception as err: msg = db_err.get(0).format(name, err.message) raise gmap_exceptions.DatabaseException(msg) def delete_database(self, name=''): """Delete DB""" try: self._conn.delete_database(name) self.database = None return True except exceptions.DatabaseDeleteError as err: if err.error_code == 1228: msg = db_err.get(1228).format(name) raise gmap_exceptions.DatabaseNotExist(msg) else: msg = db_err.get(0).format(name, err.message) raise gmap_exceptions.DatabaseException(msg) except Exception as err: msg = db_err.get(0).format(name, err.message) raise gmap_exceptions.DatabaseException(msg) def search_in_database(self, collection, field, value, offset=0, count=10): """Search Document""" # TODO: To use a better way to search try: if field and value: where = 'FILTER LOWER(doc.`{}`) like "%{}%"'.format( field, value.lower()) else: where = '' cursor = self.database.aql.execute(''' FOR doc IN {} {} LIMIT {}, {} RETURN doc'''.format( collection, where, offset, count), count=True, batch_size=1, ttl=10, optimizer_rules=['+all'] ) return cursor except Exception as err: msg = db_err.get(1).format(err.message) raise gmap_exceptions.DatabaseException(msg) ############### # COLLECTIONS # ############### def _has_collection(self, name='', edge=False): """Return True if there collection/edge""" try: collection = self.database.collection(name) res = collection.properties().get('edge') is edge # import pdb; pdb.Pdb().set_trace() # if res is edge: return res except exceptions.CollectionPropertiesError: return False def has_collection(self, name=''): """Return True if there collection""" return self._has_collection(name, False) def get_collection(self, name=''): """Return Collection""" if self.has_collection(name): self.collection = self.database.collection(name) return self.collection else: msg = coll_err.get(1228).format(name) raise gmap_exceptions.CollectionNotExist(msg) def create_collection(self, name='', edge=False): """Create Collection""" try: self.collection = self.database.create_collection( name=name, edge=False) return self.collection except exceptions.CollectionCreateError as err: if err.error_code == 1207: msg = coll_err.get(1207).format(name) raise gmap_exceptions.CollectionAlreadyExist(msg) else: msg = coll_err.get(0).format(name, err.message) raise gmap_exceptions.CollectionException(msg) except Exception as err: msg = coll_err.get(0).format(name, err.message) raise gmap_exceptions.CollectionException(msg) else: return True def delete_collection(self, name=''): """Delete Collection """ try: self.database.delete_collection(name=name) self.collection = None return True except exceptions.CollectionDeleteError as err: if err.error_code == 1203: msg = coll_err.get(1228).format(name) raise gmap_exceptions.CollectionNotExist(msg) else: msg = coll_err.get(0).format(name, err.message) raise gmap_exceptions.CollectionException(msg) except Exception as err: msg = coll_err.get(0).format(name, err.message) raise gmap_exceptions.CollectionException(msg) def has_edge(self, name=''): """Return True if there edge""" return self._has_collection(name, True) def get_edge(self, name='', ): """Return Edge""" if self.has_edge(name): self.edge = self.database.collection(name) return self.edge else: msg = edge_err.get(1228).format(name) raise gmap_exceptions.EdgeNotExist(msg) def create_edge(self, name=''): """Create Edge""" try: self.edge = self.database.create_collection( name=name, edge=True) return self.edge except exceptions.CollectionCreateError as err: if err.error_code == 1207: msg = edge_err.get(1207).format(name) raise gmap_exceptions.EdgeAlreadyExist(msg) else: msg = edge_err.get(0).format(name, err.message) raise gmap_exceptions.EdgeException(msg) except Exception as err: msg = edge_err.get(0).format(name, err.message) raise gmap_exceptions.EdgeException(msg) def delete_edge(self, name=''): """Delete Edge """ try: self.database.delete_collection(name=name) self.edge = None return True except exceptions.CollectionDeleteError as err: if err.error_code == 1203: msg = edge_err.get(1228).format(name) raise gmap_exceptions.EdgeNotExist(msg) else: msg = edge_err.get(0).format(name, err.message) raise gmap_exceptions.EdgeException(msg) except Exception as err: msg = edge_err.get(0).format(name, err.message) raise gmap_exceptions.EdgeException(msg) else: return True ########## # GRAPHS # ########## def has_graph(self, name=''): """Return True if there graph""" try: graph = self.database.graph(name) graph.properties() return True except exceptions.GraphPropertiesError: return False def get_graph(self, name=''): """Return graph""" if self.has_graph(name): self.graph = self.database.graph(name) return self.graph else: msg = gph_err.get(1924).format(name) raise gmap_exceptions.GraphNotExist(msg) def create_graph(self, name='', edge_definitions=None): """Create Graph""" try: self.graph = self.database.create_graph(name) except exceptions.GraphCreateError as err: if err.error_code == 1925: msg = gph_err.get(1925).format(name, err) raise gmap_exceptions.GraphAlreadyExist(msg) else: msg = gph_err.get(0).format(name, err.message) raise gmap_exceptions.GraphException(msg) except Exception as err: msg = gph_err.get(0).format(name, err.message) raise gmap_exceptions.GraphException(msg) else: if edge_definitions: for edge in edge_definitions: try: self.graph.create_edge_definition( name=edge.get('edge'), from_collections=edge.get('from_collections'), to_collections=edge.get('to_collections') ) except Exception as err: self.database.delete_graph(name) msg = gph_err.get(1).format(name) raise gmap_exceptions.GraphException(msg) return self.graph def delete_graph(self, name=''): """Delete Graph""" try: self.database.delete_graph(name) return True except exceptions.GraphDeleteError as err: if err.error_code == 1924: msg = gph_err.get(1924).format(name) raise gmap_exceptions.GraphNotExist(msg) else: msg = gph_err.get(0).format(name, err.message) raise gmap_exceptions.GraphException(msg) except Exception as err: msg = gph_err.get(0).format(name, err.message) raise gmap_exceptions.GraphException(msg)
class ArangoHandler(object): def __init__(self, database, protocol='http', host='localhost', port=8529, username='******', password=''): self.BATCH_LIMIT = 10000 self.client = ArangoClient(protocol=protocol, host=host, port=port, username=username, password=password, enable_logging=True) self.db = self.retrieve_db(database) def retrieve_db(self, database_name): if self.has_db(database_name): return self.client.database(database_name) else: return self.client.create_database(database_name) def has_db(self, database_name): database_list = self.client.databases() if database_name in database_list: return True else: return False def insert_jsonl(self, collection_name, lines): collection = self.retrieve_collection(collection_name) commit = [] lines = lines.split(b"\n") count = 0 for line in lines: try: dictionary = ujson.loads(line) except Exception as error: logging.error(u"Error loading jsonl: {}".format(line)) line = lines.pop() continue key_hash = hashlib.md5(line) dictionary['_key'] = key_hash.hexdigest() commit.append(dictionary) if len(commit) == self.BATCH_LIMIT: collection.import_bulk(commit, sync=True, on_duplicate="ignore") commit[:] = [] count += 1 if len(commit) > 0: collection.import_bulk(commit, sync=True, on_duplicate="ignore") logging.info(u"Records Processed: {}".format(count)) def retrieve_collection(self, collection_name): if self.has_collection(collection_name): return self.db.collection(collection_name) else: return self.db.create_collection(collection_name) def has_collection(self, collection_name): collection_list = self.db.collections() for collection_info in collection_list: if collection_info['name'] == collection_name: return True return False
class DB(object): """DB""" collection = None database = None graph = None conn = None edge = None def __init__(self): self._connection() def _connection(self): self.username = app.config['ARANGO_USERNAME'] self.password = app.config['ARANGO_PASSWORD'] self.arango_protocol = app.config['ARANGO_PROTOCOL'] self.arango_host = app.config['ARANGO_HOST'] self.arango_port = app.config['ARANGO_PORT'] self._conn = ArangoClient(protocol=self.arango_protocol, host=self.arango_host, port=self.arango_port, username=self.username, password=self.password, enable_logging=True) ############ # DATABASE # ############ def has_database(self, name=''): """Return True if there database""" try: database = self._conn.database(name) database.properties() except exceptions.DatabasePropertiesError: return False else: return True def get_database(self, name=''): """Return database""" if not name: name = app.config['ARANGO_DB'] if self.has_database(name): self.database = self._conn.database(name) return self.database else: msg = db_err.get(1228).format(name) raise gmap_exceptions.DatabaseNotExist(msg) def create_database(self, name=''): """Create DB""" try: self.database = self._conn.create_database(name) return self.database except exceptions.DatabaseCreateError as err: if err.error_code == 1207: msg = db_err.get(1207).format(name) raise gmap_exceptions.DatabaseAlreadyExist(msg) else: msg = db_err.get(0).format(name, err.message) raise gmap_exceptions.DatabaseException(msg) except Exception as err: msg = db_err.get(0).format(name, err.message) raise gmap_exceptions.DatabaseException(msg) def delete_database(self, name=''): """Delete DB""" try: self._conn.delete_database(name) self.database = None return True except exceptions.DatabaseDeleteError as err: if err.error_code == 1228: msg = db_err.get(1228).format(name) raise gmap_exceptions.DatabaseNotExist(msg) else: msg = db_err.get(0).format(name, err.message) raise gmap_exceptions.DatabaseException(msg) except Exception as err: msg = db_err.get(0).format(name, err.message) raise gmap_exceptions.DatabaseException(msg) def search_in_database(self, collection, field, value, offset=0, count=10): """Search Document""" # TODO: To use a better way to search try: if field and value: where = 'FILTER LOWER(doc.`{}`) like "%{}%"'.format( field, value.lower()) else: where = '' cursor = self.database.aql.execute(''' FOR doc IN {} {} LIMIT {}, {} RETURN doc'''.format( collection, where, offset, count), count=True, batch_size=1, ttl=10, optimizer_rules=['+all']) return cursor except Exception as err: msg = db_err.get(1).format(err.message) raise gmap_exceptions.DatabaseException(msg) ############### # COLLECTIONS # ############### def _has_collection(self, name='', edge=False): """Return True if there collection/edge""" try: collection = self.database.collection(name) res = collection.properties().get('edge') is edge # import pdb; pdb.Pdb().set_trace() # if res is edge: return res except exceptions.CollectionPropertiesError: return False def has_collection(self, name=''): """Return True if there collection""" return self._has_collection(name, False) def get_collection(self, name=''): """Return Collection""" if self.has_collection(name): self.collection = self.database.collection(name) return self.collection else: msg = coll_err.get(1228).format(name) raise gmap_exceptions.CollectionNotExist(msg) def create_collection(self, name='', edge=False): """Create Collection""" try: self.collection = self.database.create_collection(name=name, edge=False) return self.collection except exceptions.CollectionCreateError as err: if err.error_code == 1207: msg = coll_err.get(1207).format(name) raise gmap_exceptions.CollectionAlreadyExist(msg) else: msg = coll_err.get(0).format(name, err.message) raise gmap_exceptions.CollectionException(msg) except Exception as err: msg = coll_err.get(0).format(name, err.message) raise gmap_exceptions.CollectionException(msg) else: return True def delete_collection(self, name=''): """Delete Collection """ try: self.database.delete_collection(name=name) self.collection = None return True except exceptions.CollectionDeleteError as err: if err.error_code == 1203: msg = coll_err.get(1228).format(name) raise gmap_exceptions.CollectionNotExist(msg) else: msg = coll_err.get(0).format(name, err.message) raise gmap_exceptions.CollectionException(msg) except Exception as err: msg = coll_err.get(0).format(name, err.message) raise gmap_exceptions.CollectionException(msg) def has_edge(self, name=''): """Return True if there edge""" return self._has_collection(name, True) def get_edge( self, name='', ): """Return Edge""" if self.has_edge(name): self.edge = self.database.collection(name) return self.edge else: msg = edge_err.get(1228).format(name) raise gmap_exceptions.EdgeNotExist(msg) def create_edge(self, name=''): """Create Edge""" try: self.edge = self.database.create_collection(name=name, edge=True) return self.edge except exceptions.CollectionCreateError as err: if err.error_code == 1207: msg = edge_err.get(1207).format(name) raise gmap_exceptions.EdgeAlreadyExist(msg) else: msg = edge_err.get(0).format(name, err.message) raise gmap_exceptions.EdgeException(msg) except Exception as err: msg = edge_err.get(0).format(name, err.message) raise gmap_exceptions.EdgeException(msg) def delete_edge(self, name=''): """Delete Edge """ try: self.database.delete_collection(name=name) self.edge = None return True except exceptions.CollectionDeleteError as err: if err.error_code == 1203: msg = edge_err.get(1228).format(name) raise gmap_exceptions.EdgeNotExist(msg) else: msg = edge_err.get(0).format(name, err.message) raise gmap_exceptions.EdgeException(msg) except Exception as err: msg = edge_err.get(0).format(name, err.message) raise gmap_exceptions.EdgeException(msg) else: return True ########## # GRAPHS # ########## def has_graph(self, name=''): """Return True if there graph""" try: graph = self.database.graph(name) graph.properties() return True except exceptions.GraphPropertiesError: return False def get_graph(self, name=''): """Return graph""" if self.has_graph(name): self.graph = self.database.graph(name) return self.graph else: msg = gph_err.get(1924).format(name) raise gmap_exceptions.GraphNotExist(msg) def create_graph(self, name='', edge_definitions=None): """Create Graph""" try: self.graph = self.database.create_graph(name) except exceptions.GraphCreateError as err: if err.error_code == 1925: msg = gph_err.get(1925).format(name, err) raise gmap_exceptions.GraphAlreadyExist(msg) else: msg = gph_err.get(0).format(name, err.message) raise gmap_exceptions.GraphException(msg) except Exception as err: msg = gph_err.get(0).format(name, err.message) raise gmap_exceptions.GraphException(msg) else: if edge_definitions: for edge in edge_definitions: try: self.graph.create_edge_definition( name=edge.get('edge'), from_collections=edge.get('from_collections'), to_collections=edge.get('to_collections')) except Exception as err: self.database.delete_graph(name) msg = gph_err.get(1).format(name) raise gmap_exceptions.GraphException(msg) return self.graph def delete_graph(self, name=''): """Delete Graph""" try: self.database.delete_graph(name) return True except exceptions.GraphDeleteError as err: if err.error_code == 1924: msg = gph_err.get(1924).format(name) raise gmap_exceptions.GraphNotExist(msg) else: msg = gph_err.get(0).format(name, err.message) raise gmap_exceptions.GraphException(msg) except Exception as err: msg = gph_err.get(0).format(name, err.message) raise gmap_exceptions.GraphException(msg)
if len(header_line) > 1: # each header line for line in header_line: val = line.split(r'=') # add the id as the key if val[0] == 'ID': # set the elm_ref = structure[val[1]] = {} else: # for this id, add the values elm_ref[val[0].lower()] = re.sub('^\"|\"?$', '', val[1]) if len(sys.argv) > 2: # Initialize the client for ArangoDB client = ArangoClient(protocol='http', host='localhost', port=8529, username='******', password='******', enable_logging=True) if sys.argv[2] not in client.databases(): client.create_database(sys.argv[2]) # select the database db = client.db(sys.argv[2]) # create our obj loader = hapmap_load() # start loading variants loader.load_vcf(sys.argv[1], db) else: print "please specify a VCF and arango database"
delete_database - delete test database """.format(sys.argv[0]) db_name = 'db_test' username = '******' password = '' client = ArangoClient(username=username, password=password, verify=True) # try: # client.delete_database(name=db_name) # except: # pass try: test_db = client.create_database(name=db_name) except: test_db = client.db(db_name) db = Database(test_db) uni_graph = UniversityGraph(connection=db) if len(sys.argv) > 1: if 'test_data_create' == sys.argv[1]: db.create_graph(uni_graph) data_create() elif 'drop_graph' == sys.argv[1]: db.drop_graph(uni_graph) elif 'delete_database' == sys.argv[1]: client.delete_database(name=db_name)
from arango import ArangoClient # Initialize the client for ArangoDB. client = ArangoClient(protocol='http', host='localhost', port=8529) try: db = client.create_database('aops_database', username=None, password=None) except: client.delete_database('aops_database') db = client.create_database('aops_database', username=None, password=None) # Define a new graph metadata = db.create_graph('operational_metadata') fs_engines = metadata.create_vertex_collection('fs_engines') er_models = metadata.create_vertex_collection('er_models') model_code = metadata.create_vertex_collection('model_code') datasets = metadata.create_vertex_collection('datasets') edges = metadata.create_edge_definition( name='edges', from_collections=['fs_engines', 'er_models', 'model_code', 'datasets'], to_collections=['fs_engines', 'er_models', 'model_code', 'datasets']) # Insert vertices into the graph #fs_engines.insert({'_key': 'engine-1'}) fs_engines.insert({ '_key': 'engine-2', 'uri': 'https://hub.docker.com/r/fastscore/engine/', 'tag': '1.7.1' }) fs_engines.insert({ '_key': 'engine-3', 'uri': 'https://hub.docker.com/r/fastscore/engine/',