def get_connections(starting_points: List[str], collection_name: str, username: str, ip: str, password: str) -> Dict[str, Set["Document"]]: connections = collections.defaultdict(set) client = arango.ArangoClient(hosts=f"http://{ip}:8529") db = client.db(DB, username=username, password=password, auth_method="basic") edge_collections = [ get_edge_collection_name(*_) for _ in EDGE_KEYS if collection_name in _ ] assert 1 <= len(edge_collections) <= 2 for starting_point in starting_points: # TODO not getting inbound outbound right... Comes from how data is added to arangodb for edge_collection in edge_collections: for direction in DIRECTIONS: query = CONNECTIONS_QUERY.format(collection_name, starting_point, direction, edge_collection) assert db.aql.validate(query), query cursor = db.aql.execute(query) documents = set() for document in cursor: d = Document(document["datatype"], document["original_id"], document["name"]) documents.add(d) connections[starting_point].update(documents) return connections
def create_test_db(arango_host, test_db_name, test_user, test_user_password): arango_client = arango.ArangoClient(hosts=arango_host) systemdb = arango_client.db(verify=True) # default access to _system db systemdb.create_database( test_db_name, [{"username": test_user, "password": test_user_password}] ) return arango_client.db(test_db_name, test_user, test_user_password)
def create_graph(username: str, password: str, ip: str) -> None: host = HOST.format(ip) client = arango.ArangoClient(hosts=host) db = client.db(DB, username=username, password=password, auth_method="basic") if not db.has_graph(GRAPH): bron_graph = db.create_graph(GRAPH) else: bron_graph = db.graph(GRAPH) # Create vertex collections for vertex in NODE_KEYS: if not bron_graph.has_vertex_collection(vertex): _ = bron_graph.vertex_collection(vertex) logging.info(f"Done vertex_collection: {vertex}") edge_keys = get_edge_keys() for edge_key in edge_keys: edge_collection_key = get_edge_collection_name(*edge_key) if not bron_graph.has_edge_definition(edge_collection_key): _ = bron_graph.create_edge_definition( edge_collection=edge_collection_key, from_vertex_collections=[edge_key[0]], to_vertex_collections=[edge_key[1]] ) logging.info(f"Done edge_collection: {edge_collection_key}")
def connect_to_db(db_name='test', username='******', password='******', hosts='http://localhost:8529'): """ Expect a root level user to access the database and perform transactions :param db_name: str default to test :param username: str default to root :param password: str default to admin :param hosts: str default to http://localhost:8529 :return: client.db client that allows requests to the Arango system database. This is the root db to create other databases. arango.database.StandardDatabase """ client = arango.ArangoClient(hosts=hosts) sys_db = client.db('_system', username=username, password=password) if not sys_db.has_database(db_name): sys_db.create_database(name=db_name) return client.db(db_name, username=username, password=password)
def __init__(self, dbname=None, configfile=None): self._conf = self._read_conf(configfile) client_options = dict(self._conf['Arangodb']) self.client = arango.ArangoClient(**client_options) self.database_name = self._conf.get('Database', 'name', fallback=dbname) if self.database_name not in self.client.databases(): self.database = self.client.create_database(self.database_name) else: self.database = self.client.db(self.database_name)
def setup_arango_client(): client = arango.ArangoClient(protocol='http', host=g.arangohost, port='8529', username='******', password=g.arangopw1, enable_logging=True) db = client.database('skills') return db
def create_db(username: str, password: str, ip: str) -> None: host = HOST.format(ip) client = arango.ArangoClient(hosts=host) sys_db = client.db('_system', username=username, password=password, auth_method="basic") if not sys_db.has_database(DB): sys_db.create_database(DB)
def create_guest_user(username: str, password: str, ip: str) -> None: host = HOST.format(ip) client = arango.ArangoClient(hosts=host) sys_db = client.db('_system', username=username, password=password, auth_method="basic") if not sys_db.has_user(GUEST): sys_db.create_user(username=GUEST, password=GUEST) sys_db.update_permission(GUEST, 'ro', DB) logging.info(sys_db.permissions(GUEST))
def __init__(self, arangoexe: Path, arangojs: Path, root_temp_dir: Path) -> None: ''' Create and start a new ArangoDB database. An unused port will be selected for the server. :param arangoexe: The path to the ArangoDB server executable (e.g. arangod) to run. :param arangojs: The path to the ArangoDB javascript files (e.g. --javascript.startup-directory). :param root_temp_dir: A temporary directory in which to store ArangoDB data and log files. The files will be stored inside a child directory that is unique per invocation. ''' arangoexe = Path(os.path.expanduser(arangoexe)) arangojs = Path(os.path.expanduser(arangojs)) if not arangoexe or not os.access(arangoexe, os.X_OK): raise TestException( 'arangodb executable path {} does not exist or is not executable.' .format(arangoexe)) if not arangojs or not os.path.isdir(arangojs): raise TestException( 'arangodb javascript path {} does not exist or is not a directory.' .format(arangoexe)) if not root_temp_dir: raise ValueError('root_temp_dir is None') # make temp dirs root_temp_dir = root_temp_dir.absolute() os.makedirs(root_temp_dir, exist_ok=True) self.temp_dir = Path( tempfile.mkdtemp(prefix='ArangoController-', dir=str(root_temp_dir))) data_dir = self.temp_dir.joinpath('data') os.makedirs(data_dir) self.port = find_free_port() command = [ str(arangoexe), '--server.endpoint', f'tcp://localhost:{self.port}', '--configuration', 'none', '--database.directory', str(data_dir), '--javascript.startup-directory', str(arangojs), '--javascript.app-path', str(data_dir / 'apps'), '--log.file', str(self.temp_dir / 'arango.log') ] self._outfile = open(self.temp_dir.joinpath('arango.out'), 'w') self._proc = subprocess.Popen(command, stdout=self._outfile, stderr=subprocess.STDOUT) time.sleep(3) # wait for server to start up self.client = arango.ArangoClient( hosts=f'http://localhost:{self.port}') self.client.db( verify=True) # connect to the _system db with default creds
def network_import(network_import: str, username: str, password: str, ip: str) -> None: # TODO what is a good network file format... Now it is home made... with open(network_import, 'r') as fd: network = json.load(fd) print(network) host = HOST.format(ip) client = arango.ArangoClient(hosts=host) db = client.db(DB, username=username, password=password, auth_method="basic") # TODO faster to arangoimport? for node in network['nodes'].keys(): #TODO here pass
def __init__(self, database_name='ipe'): self.__database_name = database_name client_kwargs = { 'protocol': 'http', 'host': CONFIG.get('arango_host'), 'port': CONFIG.get('arango_port'), 'username': CONFIG.get('arango_username'), 'password': CONFIG.get('arango_password'), 'enable_logging': True, } self.__client = arango.ArangoClient(**client_kwargs) self.__graph = None
def get_arango_conn(): if ARANGODB_USER is None: LOGGER.error('Please set env variable "ARANGODB_USER"') sys.exit(1) a_client = arango.ArangoClient(protocol='http', host=ARANGODB_HOST, port=8529) LOGGER.info(f'Connecting to database {DB_NAME} on host {ARANGODB_HOST} ' f'as {ARANGODB_USER}') db_conn = a_client.db(DB_NAME, ARANGODB_USER, ARANGODB_PASSWORD) res = db_conn.ping() if res != 200: raise IOError(f'Unable to connect to {DB_NAME}: {res}') return db_conn
def get_client(url=None, port=None, username=None, password=None): """Get arango client db handle""" url = boltons.iterutils.first( [url, settings.ARANGO_URL, "http://localhost:8529"] # DevSkim: ignore DS137138 ) # DevSkim: ignore DS137138 (username, password) = get_user_credentials(username, password) try: client = arango.ArangoClient(hosts=url) client.db(verify=True) return client except Exception: logger.error(f"Cannot access arangodb at {url}") return None
def get_graph_traversal(starting_points: List[str], collection_name: str, username: str, ip: str, password: str) -> Dict[str, Dict[str, int]]: client = arango.ArangoClient(hosts=f"http://{ip}:8529") db = client.db(DB, username=username, password=password, auth_method="basic") graph = db.graph(GRAPH) data = {} for starting_point in starting_points: query = ID_QUERY.format(collection_name, starting_point, starting_point) assert db.aql.validate(query), query cursor = db.aql.execute(query) start_vertex = set() for document in cursor: start_vertex = document["_id"] # TODO Should get only one document logging.info(f"{collection_name} {starting_point} {start_vertex}") try: values = graph.traverse( start_vertex=start_vertex, direction="ANY", strategy='bfs', edge_uniqueness='global', vertex_uniqueness='global', # TODO Max depth number of edges? (More results are # provided when it is higher, what does that meann? max_depth=len(EDGE_KEYS)) except TypeError as e: logging.error(e) values = {} data[starting_point] = values logging.info(','.join(map(str, map(len, values.values())))) with open(f'tmp_{starting_point}.json', 'w') as fd: json.dump(values, fd, indent=2) # TODO what to return return data
def get_client(host=None, port=None, username=None, password=None, enable_logging=True): """Get arango client and edgestore db handle""" host = utils.first_true([host, config["bel_api"]["servers"]["arangodb_host"], "localhost"]) port = utils.first_true([port, config["bel_api"]["servers"]["arangodb_port"], 8529]) username = utils.first_true([username, config["bel_api"]["servers"]["arangodb_username"], ""]) password = utils.first_true( [ password, config.get("secrets", config["secrets"]["bel_api"]["servers"].get("arangodb_password")), "", ] ) arango_url = f"http://{host}:{port}" try: client = arango.ArangoClient(hosts=arango_url) client.db(verify=True) return client except Exception as e: log.error(f"Cannot access arangodb at {arango_url}") return None
def main(): web_site_urls_file = pathlib.Path(WEB_SITE_URLS_FILE_NAME) with web_site_urls_file.open() as fh: web_site_info = yaml.load(fh) a_client = arango.ArangoClient() q_db = a_client.db('quaerere', os.getenv('ARANGODB_USER'), os.getenv('ARANGODB_PASSWD')) if not q_db.has_collection(WEB_SITE_DB_COLLECTION): LOGGER.info(f'Creating website collection.', extra={'website_collection': WEB_SITE_DB_COLLECTION}) q_db.create_collection(WEB_SITE_DB_COLLECTION) web_sites = q_db.collection(WEB_SITE_DB_COLLECTION) for web_site in web_site_info['web_sites']: db_cur = web_sites.find({'url': web_site['url']}) if db_cur.count() == 0: LOGGER.info('Inserting web site.', extra={'web_site': web_site}) web_sites.insert(web_site) else: LOGGER.info('Found web site in DB, skipping', extra={'web_site': web_site})
import arango from arango.database import StandardCollection, StandardDatabase from loguru import logger from settings import ARANGO_ENDPOINT, DATABASE_NAME, COLLECTION_NAME, ARANGO_PASSWORD logger.info( f"connecting to {ARANGO_ENDPOINT}, db: {DATABASE_NAME}, coll: {COLLECTION_NAME}" ) client = arango.ArangoClient(hosts=ARANGO_ENDPOINT) _sys_db: StandardDatabase = client.db(password=ARANGO_PASSWORD) if not _sys_db.has_database(DATABASE_NAME): _sys_db.create_database(DATABASE_NAME) db = client.db(DATABASE_NAME, password=ARANGO_PASSWORD) if db.has_collection(COLLECTION_NAME): cases_collection: StandardCollection = db.collection(COLLECTION_NAME) else: cases_collection: StandardCollection = db.create_collection( COLLECTION_NAME) cases_collection.add_hash_index(fields=["id"], unique=False) logger.info("cases collection ready")
def build_samples( config: Dict[str, str]) -> Tuple[Samples, KBaseUserLookup, List[str]]: ''' Build the sample service instance from the SDK server provided parameters. :param cfg: The SDK generated configuration. :returns: A samples instance. ''' if not config: raise ValueError('config is empty, cannot start service') arango_url = _check_string_req(config.get('arango-url'), 'config param arango-url') arango_db = _check_string_req(config.get('arango-db'), 'config param arango-db') arango_user = _check_string_req(config.get('arango-user'), 'config param arango-user') arango_pwd = _check_string_req(config.get('arango-pwd'), 'config param arango-pwd') col_sample = _check_string_req(config.get('sample-collection'), 'config param sample-collection') col_version = _check_string_req(config.get('version-collection'), 'config param version-collection') col_ver_edge = _check_string_req(config.get('version-edge-collection'), 'config param version-edge-collection') col_node = _check_string_req(config.get('node-collection'), 'config param node-collection') col_node_edge = _check_string_req(config.get('node-edge-collection'), 'config param node-edge-collection') col_data_link = _check_string_req(config.get('data-link-collection'), 'config param data-link-collection') col_ws_obj_ver = _check_string_req( config.get('workspace-object-version-shadow-collection'), 'config param workspace-object-version-shadow-collection') col_schema = _check_string_req(config.get('schema-collection'), 'config param schema-collection') auth_root_url = _check_string_req(config.get('auth-root-url'), 'config param auth-root-url') auth_token = _check_string_req(config.get('auth-token'), 'config param auth-token') full_roles = split_value(config, 'auth-full-admin-roles') read_roles = split_value(config, 'auth-read-admin-roles') read_exempt_roles = split_value(config, 'auth-read-exempt-roles') ws_url = _check_string_req(config.get('workspace-url'), 'config param workspace-url') ws_token = _check_string_req(config.get('workspace-read-admin-token'), 'config param workspace-read-admin-token') kafka_servers = _check_string(config.get('kafka-bootstrap-servers'), 'config param kafka-bootstrap-servers', optional=True) kafka_topic = None if kafka_servers: # have to start the server twice to test no kafka scenario kafka_topic = _check_string(config.get('kafka-topic'), 'config param kafka-topic') metaval_url = _check_string(config.get('metadata-validator-config-url'), 'config param metadata-validator-config-url', optional=True) # meta params may have info that shouldn't be logged so don't log any for now. # Add code to deal with this later if needed print(f''' Starting server with config: arango-url: {arango_url} arango-db: {arango_db} arango-user: {arango_user} arango-pwd: [REDACTED FOR YOUR SAFETY AND COMFORT] sample-collection: {col_sample} version-collection: {col_version} version-edge-collection: {col_ver_edge} node-collection: {col_node} node-edge-collection: {col_node_edge} data-link-collection: {col_data_link} workspace-object-version-shadow-collection: {col_ws_obj_ver} schema-collection: {col_schema} auth-root-url: {auth_root_url} auth-token: [REDACTED FOR YOUR CONVENIENCE AND ENJOYMENT] auth-full-admin-roles: {', '.join(full_roles)} auth-read-admin-roles: {', '.join(read_roles)} auth-read-exempt-roles: {', '.join(read_exempt_roles)} workspace-url: {ws_url} workspace-read-admin-token: [REDACTED FOR YOUR ULTIMATE PLEASURE] kafka-bootstrap-servers: {kafka_servers} kafka-topic: {kafka_topic} metadata-validators-config-url: {metaval_url} ''') # build the validators before trying to connect to arango metaval = get_validators( metaval_url) if metaval_url else MetadataValidatorSet() arangoclient = _arango.ArangoClient(hosts=arango_url) arango_db = arangoclient.db(arango_db, username=arango_user, password=arango_pwd, verify=True) storage = _ArangoSampleStorage( arango_db, col_sample, col_version, col_ver_edge, col_node, col_node_edge, col_ws_obj_ver, col_data_link, col_schema, ) storage.start_consistency_checker() kafka = _KafkaNotifer(kafka_servers, _cast( str, kafka_topic)) if kafka_servers else None user_lookup = KBaseUserLookup(auth_root_url, auth_token, full_roles, read_roles) ws = _WS(_Workspace(ws_url, token=ws_token)) return Samples(storage, user_lookup, metaval, ws, kafka), user_lookup, read_exempt_roles
def _get_client(self): return arango.ArangoClient(hosts=self._get_db_uri())
import json import arango import re client = arango.ArangoClient(protocol='http', host='localhost', port='6969', username='******', password='******', enable_logging=True) def get_graph(): # create or get the database print(client.databases()) # get the user try: client.user('admin') except: client.create_user('admin', 'password') print("Creating user...") dbs = client.databases() print("Databases: ", dbs) if 'skills' not in dbs: db = client.create_database('skills', username='******', password='******') client.grant_user_access('admin', 'skills') else: db = client.database('skills')
def connect(): client = arango.ArangoClient(protocol='http', host='localhost', port=8529) test_db = client.db('books', username='******', password='******') return arango_orm.Database(test_db)
from translate.storage.po import pofile import lxml.html import arango import subprocess import pathlib import json import regex storage = pathlib.Path('./storage') if not storage.exists(): client = arango.ArangoClient(protocol='http', host='localhost', port=8529) db = client.db('suttacentral', username='******', password='******') subprocess.run(['docker', 'cp', 'sc-flask:/opt/sc/storage/', str(storage)], check=True) def make_clean_html(uid): markup_path = storage / f'{uid}_ms.html' strings_path = storage / f'{uid}_sujato.json' markup_string = markup_path.open().read() strings = json.load(strings_path.open()) def repl_fn(m): s = strings.get(m[1], '') if s: return s + ' ' return s
def create(): client = arango.ArangoClient(protocol='http', host='localhost', port=8529) sys_db = client.db('_system', username='******', password='******') sys_db.create_database('books')
import logging import arango import pm.settings as settings import xxhash log = logging.getLogger() username = "******" password = "" client = arango.ArangoClient(hosts=settings.ARANGO_URL) sys_db = client.db("_system", username=username, password=password) # Create a new database if not sys_db.has_database(settings.PUBMED_DB_NAME): sys_db.create_database( name=settings.PUBMED_DB_NAME, users=[{ "username": username, "password": password, "active": True }], ) pubmed_db = client.db(settings.PUBMED_DB_NAME, username=username, password=password) # xml collection