def get_connections(starting_points: List[str], collection_name: str,
username: str, ip: str,
password: str) -> Dict[str, Set["Document"]]:
connections = collections.defaultdict(set)
client = arango.ArangoClient(hosts=f"http://{ip}:8529")
db = client.db(DB,
username=username,
password=password,
auth_method="basic")
edge_collections = [
get_edge_collection_name(*_) for _ in EDGE_KEYS if collection_name in _
]
assert 1 <= len(edge_collections) <= 2
for starting_point in starting_points:
# TODO not getting inbound outbound right... Comes from how data is added to arangodb
for edge_collection in edge_collections:
for direction in DIRECTIONS:
query = CONNECTIONS_QUERY.format(collection_name,
starting_point, direction,
edge_collection)
assert db.aql.validate(query), query
cursor = db.aql.execute(query)
documents = set()
for document in cursor:
d = Document(document["datatype"], document["original_id"],
document["name"])
documents.add(d)
connections[starting_point].update(documents)
return connections
def create_test_db(arango_host, test_db_name, test_user, test_user_password):
arango_client = arango.ArangoClient(hosts=arango_host)
systemdb = arango_client.db(verify=True) # default access to _system db
systemdb.create_database(
test_db_name, [{"username": test_user, "password": test_user_password}]
)
return arango_client.db(test_db_name, test_user, test_user_password)
def create_graph(username: str, password: str, ip: str) -> None:
host = HOST.format(ip)
client = arango.ArangoClient(hosts=host)
db = client.db(DB, username=username, password=password, auth_method="basic")
if not db.has_graph(GRAPH):
bron_graph = db.create_graph(GRAPH)
else:
bron_graph = db.graph(GRAPH)
# Create vertex collections
for vertex in NODE_KEYS:
if not bron_graph.has_vertex_collection(vertex):
_ = bron_graph.vertex_collection(vertex)
logging.info(f"Done vertex_collection: {vertex}")
edge_keys = get_edge_keys()
for edge_key in edge_keys:
edge_collection_key = get_edge_collection_name(*edge_key)
if not bron_graph.has_edge_definition(edge_collection_key):
_ = bron_graph.create_edge_definition(
edge_collection=edge_collection_key,
from_vertex_collections=[edge_key[0]],
to_vertex_collections=[edge_key[1]]
)
logging.info(f"Done edge_collection: {edge_collection_key}")
def connect_to_db(db_name='test',
username='******',
password='******',
hosts='http://localhost:8529'):
"""
Expect a root level user to access the database and perform transactions
:param db_name: str
default to test
:param username: str
default to root
:param password: str
default to admin
:param hosts: str
default to http://localhost:8529
:return: client.db
client that allows requests to the Arango system database. This is the root db to create other databases.
arango.database.StandardDatabase
"""
client = arango.ArangoClient(hosts=hosts)
sys_db = client.db('_system', username=username, password=password)
if not sys_db.has_database(db_name):
sys_db.create_database(name=db_name)
return client.db(db_name, username=username, password=password)
def __init__(self, dbname=None, configfile=None):
self._conf = self._read_conf(configfile)
client_options = dict(self._conf['Arangodb'])
self.client = arango.ArangoClient(**client_options)
self.database_name = self._conf.get('Database', 'name', fallback=dbname)
if self.database_name not in self.client.databases():
self.database = self.client.create_database(self.database_name)
else:
self.database = self.client.db(self.database_name)
def setup_arango_client():
client = arango.ArangoClient(protocol='http',
host=g.arangohost,
port='8529',
username='******',
password=g.arangopw1,
enable_logging=True)
db = client.database('skills')
return db
def create_db(username: str, password: str, ip: str) -> None:
host = HOST.format(ip)
client = arango.ArangoClient(hosts=host)
sys_db = client.db('_system',
username=username,
password=password,
auth_method="basic")
if not sys_db.has_database(DB):
sys_db.create_database(DB)
def create_guest_user(username: str, password: str, ip: str) -> None:
host = HOST.format(ip)
client = arango.ArangoClient(hosts=host)
sys_db = client.db('_system', username=username, password=password, auth_method="basic")
if not sys_db.has_user(GUEST):
sys_db.create_user(username=GUEST, password=GUEST)
sys_db.update_permission(GUEST, 'ro', DB)
logging.info(sys_db.permissions(GUEST))
def __init__(self, arangoexe: Path, arangojs: Path,
root_temp_dir: Path) -> None:
'''
Create and start a new ArangoDB database. An unused port will be selected for the server.
:param arangoexe: The path to the ArangoDB server executable (e.g. arangod) to run.
:param arangojs: The path to the ArangoDB javascript files
(e.g. --javascript.startup-directory).
:param root_temp_dir: A temporary directory in which to store ArangoDB data and log files.
The files will be stored inside a child directory that is unique per invocation.
'''
arangoexe = Path(os.path.expanduser(arangoexe))
arangojs = Path(os.path.expanduser(arangojs))
if not arangoexe or not os.access(arangoexe, os.X_OK):
raise TestException(
'arangodb executable path {} does not exist or is not executable.'
.format(arangoexe))
if not arangojs or not os.path.isdir(arangojs):
raise TestException(
'arangodb javascript path {} does not exist or is not a directory.'
.format(arangoexe))
if not root_temp_dir:
raise ValueError('root_temp_dir is None')
# make temp dirs
root_temp_dir = root_temp_dir.absolute()
os.makedirs(root_temp_dir, exist_ok=True)
self.temp_dir = Path(
tempfile.mkdtemp(prefix='ArangoController-',
dir=str(root_temp_dir)))
data_dir = self.temp_dir.joinpath('data')
os.makedirs(data_dir)
self.port = find_free_port()
command = [
str(arangoexe), '--server.endpoint',
f'tcp://localhost:{self.port}', '--configuration', 'none',
'--database.directory',
str(data_dir), '--javascript.startup-directory',
str(arangojs), '--javascript.app-path',
str(data_dir / 'apps'), '--log.file',
str(self.temp_dir / 'arango.log')
]
self._outfile = open(self.temp_dir.joinpath('arango.out'), 'w')
self._proc = subprocess.Popen(command,
stdout=self._outfile,
stderr=subprocess.STDOUT)
time.sleep(3) # wait for server to start up
self.client = arango.ArangoClient(
hosts=f'http://localhost:{self.port}')
self.client.db(
verify=True) # connect to the _system db with default creds
def network_import(network_import: str, username: str, password: str, ip: str) -> None:
# TODO what is a good network file format... Now it is home made...
with open(network_import, 'r') as fd:
network = json.load(fd)
print(network)
host = HOST.format(ip)
client = arango.ArangoClient(hosts=host)
db = client.db(DB, username=username, password=password, auth_method="basic")
# TODO faster to arangoimport?
for node in network['nodes'].keys():
#TODO here
pass
def __init__(self, database_name='ipe'):
self.__database_name = database_name
client_kwargs = {
'protocol': 'http',
'host': CONFIG.get('arango_host'),
'port': CONFIG.get('arango_port'),
'username': CONFIG.get('arango_username'),
'password': CONFIG.get('arango_password'),
'enable_logging': True,
}
self.__client = arango.ArangoClient(**client_kwargs)
self.__graph = None
def get_arango_conn():
if ARANGODB_USER is None:
LOGGER.error('Please set env variable "ARANGODB_USER"')
sys.exit(1)
a_client = arango.ArangoClient(protocol='http',
host=ARANGODB_HOST,
port=8529)
LOGGER.info(f'Connecting to database {DB_NAME} on host {ARANGODB_HOST} '
f'as {ARANGODB_USER}')
db_conn = a_client.db(DB_NAME, ARANGODB_USER, ARANGODB_PASSWORD)
res = db_conn.ping()
if res != 200:
raise IOError(f'Unable to connect to {DB_NAME}: {res}')
return db_conn
def get_client(url=None, port=None, username=None, password=None):
"""Get arango client db handle"""
url = boltons.iterutils.first(
[url, settings.ARANGO_URL,
"http://localhost:8529"] # DevSkim: ignore DS137138
) # DevSkim: ignore DS137138
(username, password) = get_user_credentials(username, password)
try:
client = arango.ArangoClient(hosts=url)
client.db(verify=True)
return client
except Exception:
logger.error(f"Cannot access arangodb at {url}")
return None
def get_graph_traversal(starting_points: List[str], collection_name: str,
username: str, ip: str,
password: str) -> Dict[str, Dict[str, int]]:
client = arango.ArangoClient(hosts=f"http://{ip}:8529")
db = client.db(DB,
username=username,
password=password,
auth_method="basic")
graph = db.graph(GRAPH)
data = {}
for starting_point in starting_points:
query = ID_QUERY.format(collection_name, starting_point,
starting_point)
assert db.aql.validate(query), query
cursor = db.aql.execute(query)
start_vertex = set()
for document in cursor:
start_vertex = document["_id"]
# TODO Should get only one document
logging.info(f"{collection_name} {starting_point} {start_vertex}")
try:
values = graph.traverse(
start_vertex=start_vertex,
direction="ANY",
strategy='bfs',
edge_uniqueness='global',
vertex_uniqueness='global',
# TODO Max depth number of edges? (More results are
# provided when it is higher, what does that meann?
max_depth=len(EDGE_KEYS))
except TypeError as e:
logging.error(e)
values = {}
data[starting_point] = values
logging.info(','.join(map(str, map(len, values.values()))))
with open(f'tmp_{starting_point}.json', 'w') as fd:
json.dump(values, fd, indent=2)
# TODO what to return
return data
def get_client(host=None, port=None, username=None, password=None, enable_logging=True):
"""Get arango client and edgestore db handle"""
host = utils.first_true([host, config["bel_api"]["servers"]["arangodb_host"], "localhost"])
port = utils.first_true([port, config["bel_api"]["servers"]["arangodb_port"], 8529])
username = utils.first_true([username, config["bel_api"]["servers"]["arangodb_username"], ""])
password = utils.first_true(
[
password,
config.get("secrets", config["secrets"]["bel_api"]["servers"].get("arangodb_password")),
"",
]
)
arango_url = f"http://{host}:{port}"
try:
client = arango.ArangoClient(hosts=arango_url)
client.db(verify=True)
return client
except Exception as e:
log.error(f"Cannot access arangodb at {arango_url}")
return None
def main():
web_site_urls_file = pathlib.Path(WEB_SITE_URLS_FILE_NAME)
with web_site_urls_file.open() as fh:
web_site_info = yaml.load(fh)
a_client = arango.ArangoClient()
q_db = a_client.db('quaerere', os.getenv('ARANGODB_USER'),
os.getenv('ARANGODB_PASSWD'))
if not q_db.has_collection(WEB_SITE_DB_COLLECTION):
LOGGER.info(f'Creating website collection.',
extra={'website_collection': WEB_SITE_DB_COLLECTION})
q_db.create_collection(WEB_SITE_DB_COLLECTION)
web_sites = q_db.collection(WEB_SITE_DB_COLLECTION)
for web_site in web_site_info['web_sites']:
db_cur = web_sites.find({'url': web_site['url']})
if db_cur.count() == 0:
LOGGER.info('Inserting web site.', extra={'web_site': web_site})
web_sites.insert(web_site)
else:
LOGGER.info('Found web site in DB, skipping',
extra={'web_site': web_site})
import arango
from arango.database import StandardCollection, StandardDatabase
from loguru import logger
from settings import ARANGO_ENDPOINT, DATABASE_NAME, COLLECTION_NAME, ARANGO_PASSWORD
logger.info(
f"connecting to {ARANGO_ENDPOINT}, db: {DATABASE_NAME}, coll: {COLLECTION_NAME}"
)
client = arango.ArangoClient(hosts=ARANGO_ENDPOINT)
_sys_db: StandardDatabase = client.db(password=ARANGO_PASSWORD)
if not _sys_db.has_database(DATABASE_NAME):
_sys_db.create_database(DATABASE_NAME)
db = client.db(DATABASE_NAME, password=ARANGO_PASSWORD)
if db.has_collection(COLLECTION_NAME):
cases_collection: StandardCollection = db.collection(COLLECTION_NAME)
else:
cases_collection: StandardCollection = db.create_collection(
COLLECTION_NAME)
cases_collection.add_hash_index(fields=["id"], unique=False)
logger.info("cases collection ready")
def build_samples(
config: Dict[str, str]) -> Tuple[Samples, KBaseUserLookup, List[str]]:
'''
Build the sample service instance from the SDK server provided parameters.
:param cfg: The SDK generated configuration.
:returns: A samples instance.
'''
if not config:
raise ValueError('config is empty, cannot start service')
arango_url = _check_string_req(config.get('arango-url'),
'config param arango-url')
arango_db = _check_string_req(config.get('arango-db'),
'config param arango-db')
arango_user = _check_string_req(config.get('arango-user'),
'config param arango-user')
arango_pwd = _check_string_req(config.get('arango-pwd'),
'config param arango-pwd')
col_sample = _check_string_req(config.get('sample-collection'),
'config param sample-collection')
col_version = _check_string_req(config.get('version-collection'),
'config param version-collection')
col_ver_edge = _check_string_req(config.get('version-edge-collection'),
'config param version-edge-collection')
col_node = _check_string_req(config.get('node-collection'),
'config param node-collection')
col_node_edge = _check_string_req(config.get('node-edge-collection'),
'config param node-edge-collection')
col_data_link = _check_string_req(config.get('data-link-collection'),
'config param data-link-collection')
col_ws_obj_ver = _check_string_req(
config.get('workspace-object-version-shadow-collection'),
'config param workspace-object-version-shadow-collection')
col_schema = _check_string_req(config.get('schema-collection'),
'config param schema-collection')
auth_root_url = _check_string_req(config.get('auth-root-url'),
'config param auth-root-url')
auth_token = _check_string_req(config.get('auth-token'),
'config param auth-token')
full_roles = split_value(config, 'auth-full-admin-roles')
read_roles = split_value(config, 'auth-read-admin-roles')
read_exempt_roles = split_value(config, 'auth-read-exempt-roles')
ws_url = _check_string_req(config.get('workspace-url'),
'config param workspace-url')
ws_token = _check_string_req(config.get('workspace-read-admin-token'),
'config param workspace-read-admin-token')
kafka_servers = _check_string(config.get('kafka-bootstrap-servers'),
'config param kafka-bootstrap-servers',
optional=True)
kafka_topic = None
if kafka_servers: # have to start the server twice to test no kafka scenario
kafka_topic = _check_string(config.get('kafka-topic'),
'config param kafka-topic')
metaval_url = _check_string(config.get('metadata-validator-config-url'),
'config param metadata-validator-config-url',
optional=True)
# meta params may have info that shouldn't be logged so don't log any for now.
# Add code to deal with this later if needed
print(f'''
Starting server with config:
arango-url: {arango_url}
arango-db: {arango_db}
arango-user: {arango_user}
arango-pwd: [REDACTED FOR YOUR SAFETY AND COMFORT]
sample-collection: {col_sample}
version-collection: {col_version}
version-edge-collection: {col_ver_edge}
node-collection: {col_node}
node-edge-collection: {col_node_edge}
data-link-collection: {col_data_link}
workspace-object-version-shadow-collection: {col_ws_obj_ver}
schema-collection: {col_schema}
auth-root-url: {auth_root_url}
auth-token: [REDACTED FOR YOUR CONVENIENCE AND ENJOYMENT]
auth-full-admin-roles: {', '.join(full_roles)}
auth-read-admin-roles: {', '.join(read_roles)}
auth-read-exempt-roles: {', '.join(read_exempt_roles)}
workspace-url: {ws_url}
workspace-read-admin-token: [REDACTED FOR YOUR ULTIMATE PLEASURE]
kafka-bootstrap-servers: {kafka_servers}
kafka-topic: {kafka_topic}
metadata-validators-config-url: {metaval_url}
''')
# build the validators before trying to connect to arango
metaval = get_validators(
metaval_url) if metaval_url else MetadataValidatorSet()
arangoclient = _arango.ArangoClient(hosts=arango_url)
arango_db = arangoclient.db(arango_db,
username=arango_user,
password=arango_pwd,
verify=True)
storage = _ArangoSampleStorage(
arango_db,
col_sample,
col_version,
col_ver_edge,
col_node,
col_node_edge,
col_ws_obj_ver,
col_data_link,
col_schema,
)
storage.start_consistency_checker()
kafka = _KafkaNotifer(kafka_servers, _cast(
str, kafka_topic)) if kafka_servers else None
user_lookup = KBaseUserLookup(auth_root_url, auth_token, full_roles,
read_roles)
ws = _WS(_Workspace(ws_url, token=ws_token))
return Samples(storage, user_lookup, metaval, ws,
kafka), user_lookup, read_exempt_roles
def _get_client(self):
return arango.ArangoClient(hosts=self._get_db_uri())
import json
import arango
import re
client = arango.ArangoClient(protocol='http',
host='localhost',
port='6969',
username='******',
password='******',
enable_logging=True)
def get_graph():
# create or get the database
print(client.databases())
# get the user
try:
client.user('admin')
except:
client.create_user('admin', 'password')
print("Creating user...")
dbs = client.databases()
print("Databases: ", dbs)
if 'skills' not in dbs:
db = client.create_database('skills',
username='******',
password='******')
client.grant_user_access('admin', 'skills')
else:
db = client.database('skills')
def connect():
client = arango.ArangoClient(protocol='http',
host='localhost',
port=8529)
test_db = client.db('books', username='******', password='******')
return arango_orm.Database(test_db)
from translate.storage.po import pofile
import lxml.html
import arango
import subprocess
import pathlib
import json
import regex
storage = pathlib.Path('./storage')
if not storage.exists():
client = arango.ArangoClient(protocol='http', host='localhost', port=8529)
db = client.db('suttacentral', username='******', password='******')
subprocess.run(['docker', 'cp', 'sc-flask:/opt/sc/storage/',
str(storage)],
check=True)
def make_clean_html(uid):
markup_path = storage / f'{uid}_ms.html'
strings_path = storage / f'{uid}_sujato.json'
markup_string = markup_path.open().read()
strings = json.load(strings_path.open())
def repl_fn(m):
s = strings.get(m[1], '')
if s:
return s + ' '
return s
def create():
client = arango.ArangoClient(protocol='http',
host='localhost',
port=8529)
sys_db = client.db('_system', username='******', password='******')
sys_db.create_database('books')
import logging
import arango
import pm.settings as settings
import xxhash
log = logging.getLogger()
username = "******"
password = ""
client = arango.ArangoClient(hosts=settings.ARANGO_URL)
sys_db = client.db("_system", username=username, password=password)
# Create a new database
if not sys_db.has_database(settings.PUBMED_DB_NAME):
sys_db.create_database(
name=settings.PUBMED_DB_NAME,
users=[{
"username": username,
"password": password,
"active": True
}],
)
pubmed_db = client.db(settings.PUBMED_DB_NAME,
username=username,
password=password)
# xml collection
