def create_db(self):
client = ArangoClient(hosts='http://localhost:8529')
# Connect to "_system" database as root user.
# This returns an API wrapper for "_system" database.
sys_db = client.db('_system',\
username='******',
password='******')
# Create a new arangopipe database if it does not exist.
if not sys_db.has_database('ITSM_db'):
sys_db.create_database('ITSM_db')
if not sys_db.has_user('ITSM_db_admin'):
sys_db.create_user(username = '******',\
password = '******')
sys_db.update_permission(username = '******',\
database = 'ITSM_db', permission = "rw")
# Connect to arangopipe database as administrative user.
#This returns an API wrapper for "test" database.
db = client.db('ITSM_db',\
username='******',\
password='******')
self.db = db
return
def verify(block):
print('BRIGHTID')
db = ArangoClient(hosts=config.ARANGO_SERVER).db('_system')
verifieds = db.aql.execute('''
FOR v in verifications
FILTER v.name == 'SeedConnected'
AND v.rank > 0
AND v.block == @block
RETURN v.user
''',
bind_vars={'block': block})
batch_db = db.begin_batch_execution(return_result=True)
verifications = batch_db.collection('verifications')
for i, verified in enumerate(verifieds):
verifications.insert({
'name': 'BrightID',
'user': verified,
'block': block,
'timestamp': int(time.time() * 1000),
'hash': utils.hash('BrightID', verified)
})
if i % 1000 == 0:
batch_db.commit()
batch_db = db.begin_batch_execution(return_result=True)
verifications = batch_db.collection('verifications')
batch_db.commit()
verifiedCount = db['verifications'].find({
'name': 'BrightID',
'block': block
}).count()
print(f'verifieds: {verifiedCount}\n')
def all_games():
client = ArangoClient(hosts=os.getenv('DB_URI'))
db = client.db(os.getenv('DB_NAME'), username=os.getenv('DB_USER'), password=os.getenv('DB_PASSWORD'))
result = []
for name in get_all_games(db):
result.append(json.dumps(read(db, name), cls=MarugotoEncoder))
return result, 200
def __init__(self,
database,
protocol='http',
host='localhost',
port=8529,
username='******',
password=''):
"""Initialize an ArangoDB Handler to handle all arangodb operations.
:param database: (str) The database name to use
:param protocol: (str) The protocol
:param host: (str) The host
:param port: (int) The port
:param username: (str)
:param password:
"""
self._database = database
self.username = username
self.password = password
self.client = ArangoClient(protocol=protocol, host=host, port=port)
self._system_db = self.client.db('_system',
username=self.username,
password=self.password)
if not self._system_db.has_database(self._database):
self._system_db.create_database(self._database)
self._database = self.client.db(self._database,
username=self.username,
password=self.password)
self.collections = {}
def startup():
global arangodb_client
arangodb_client = ArangoClient(host=settings.ARANGODB_HOST)
arangodb_client = arangodb_client.db(settings.ARANGODB_DATABASE,
username=settings.ARANGODB_USERNAME,
password=str(
settings.ARANGODB_PASSWORD))
def register_model(self, mi, user_id = "authorized_user",\
project = "Wine-Quality-Regression-Modelling"):
""" Register a model. The operation requires specifying a user id. If the user id is permitted to register a model, then the registration proceeds, otherwise an unauthorized operation is indicated. """
models = self.emlg.vertex_collection("models")
model_reg = models.insert(mi)
client = ArangoClient(protocol='http',\
host=self.cfg['arangodb']['host'],\
port=self.cfg['arangodb']['port'])
db = client.db(name=self.cfg['arangodb']['arangopipe_dbname'],\
username=self.cfg['arangodb']['arangopipe_admin_username'],\
password=self.cfg['arangodb']['arangopipe_admin_password'])
# Execute the query
cursor = db.aql.execute(
'FOR doc IN project FILTER doc.name == @value RETURN doc',
bind_vars={'value': project})
project_keys = [doc for doc in cursor]
the_project_info = project_keys[0]
project_model_edge = self.emlg.edge_collection("project_models")
project_model_key = the_project_info["_key"] + "-" + model_reg["_key"]
a_project_model_edge = {"_key": project_model_key,\
"_from": "project/" + the_project_info["_key"],\
"_to": "models/" + model_reg["_key"]}
pm_reg = project_model_edge.insert(a_project_model_edge)
logger.info("Recording project model link " + str(pm_reg))
return model_reg
def log_serving_perf(self, sp, dep_tag, userid = "authorized user"):
""" Log serving performance against a deployed model. The user making the request needs to be authorized to log this performance update. A serving performance vertex is created and is linked with its deployment vertex"""
servingperf = self.emlg.vertex_collection("servingperf")
sp_reg = servingperf.insert(sp)
# Locate the deployment record
client = ArangoClient(protocol='http',\
host=self.cfg['arangodb']['host'],\
port=self.cfg['arangodb']['port'])
db = client.db(name=self.cfg['arangodb']['arangopipe_dbname'],\
username=self.cfg['arangodb']['arangopipe_admin_username'],\
password=self.cfg['arangodb']['arangopipe_admin_password'])
# Execute the query
cursor = db.aql.execute(
'FOR doc IN deployment FILTER doc.tag == @value RETURN doc',
bind_vars={'value': dep_tag})
dep_docs = [doc for doc in cursor]
the_dep_doc = dep_docs[0]
# Link the service performance record with the deployment record
dep_servingperf_edge = self.emlg.edge_collection("deployment_servingperf")
dep_servingperf_key = the_dep_doc["_key"] + "-" + sp_reg["_key"]
the_dep_servingperf_edge = { "_key": dep_servingperf_key,\
"_from": the_dep_doc["_id"],\
"_to": sp_reg["_id"]}
dep_servingperf_reg = dep_servingperf_edge.insert(the_dep_servingperf_edge)
return dep_servingperf_reg
def ArangoV():
client = ArangoClient(protocol='http',
host='localhost',
port=8529,
username='******',
password='',
enable_logging=True)
db = client.db('graph')
graphPersons = db.graph('graphPersons')
id = 1
record = []
cpu = []
mem = []
disk = []
startTime = time.time()
while id <= nodeNum:
a = []
a.append('persons' + getRamdomNo())
a.append('persons' + getRamdomNo())
try:
db.aql.execute(
"for v,e in outbound shortest_path '%s' to '%s' graph 'graphPersons' return [v._key,e._key]"
% tuple(a))
except:
id = writeRecord(id, record, cpu, mem, disk, startTime)
continue
id = writeRecord(id, record, cpu, mem, disk, startTime)
return record, cpu, mem, disk
def init_graph(self):
""" Initialize a graph when an instance of ArangoPipe is provisioned. """
client = ArangoClient(protocol='http',\
host=self.cfg['arangodb']['host'],\
port=self.cfg['arangodb']['port'])
# Connect to "_system" database as root user.
# This returns an API wrapper for "_system" database.
sys_db = client.db('_system',\
username=self.cfg['arangodb']['root_user'],\
password=self.cfg['arangodb']['root_user_password'])
if not sys_db.has_database(self.cfg['arangodb']['arangopipe_dbname']):
logger.error("arangopipe database has not been created.")
raise AttributeError("arangopipe database has not been created!")
else:
db = client.db(name=self.cfg['arangodb']['arangopipe_dbname'],\
username=self.cfg['arangodb']['arangopipe_admin_username'],\
password=self.cfg['arangodb']['arangopipe_admin_password'])
if db.has_graph(self.cfg['mlgraph']['graphname']):
self.emlg = db.graph(self.cfg['mlgraph']['graphname'])
else:
logger.error("ML tracking graph was not created. ")
raise AttributeError("ML tracking graph has not been created")
logger.info("Arango Pipe ML Graph initialized")
return
def main():
a = parse_args()
url = urlparse(a.arango_url)
client = ArangoClient(protocol=url.scheme,
host=url.hostname,
port=url.port)
if a.user:
if a.pwd_file:
with open(a.pwd_file) as pwd_file:
pwd = pwd_file.read().strip()
else:
pwd = getpass.getpass()
db = client.db(a.database, a.user, pwd, verify=True)
else:
db = client.db(a.database, verify=True)
attdb = ArangoBatchTimeTravellingDB(
db,
a.load_registry_collection,
a.node_collection,
default_edge_collection=a.edge_collection)
with open(a.file) as in1, open(a.file) as in2:
nodeprov = GTDBNodeProvider(in1)
edgeprov = GTDBEdgeProvider(in2)
load_graph_delta(_LOAD_NAMESPACE, nodeprov, edgeprov, attdb,
a.load_timestamp, a.release_timestamp, a.load_version)
def main():
# Override csv default 128k field size
csv.field_size_limit(int(ctypes.c_ulong(-1).value // 2))
# Initialize the client for ArangoDB.
client = ArangoClient(hosts='http://localhost:8529')
# Upload settings
filename = 'C:\\Working\\iresearch-toolkit\\iresearch\\iresearch.deps\\benchmark_resources\\benchmark.data' # data file
collection = 'wikipedia' # target collection
database = '_system' # target database
line_limit = 10000000 # how many documents to upload
batch_size = 10000 # batch size for inserting into Arango
db = client.db(database)
wikipedia = db.collection(collection)
f = open(filname, mode ='r', encoding='utf-8', errors='replace')
reader = csv.reader(f, delimiter='\t')
data = []
total = 0
for row in reader:
data.append({'title': row[0], 'body': row[2]})
if len(data) > batch_size:
wikipedia.insert_many(data)
data.clear()
total = total + 1
if total >= line_limit:
break
if len(data) > 0:
wikipedia.insert_many(data)
f.close()
def map_datapath_single_by_key(basepath_key, matchpath_key, author, weight, annotation):
basepath = fetch_datapath(basepath_key)
if not basepath:
raise Exception('Specified base DataPath not found!')
matchpath = fetch_datapath(matchpath_key)
if not matchpath:
raise Exception('Specified matching DataPath not found!')
client = ArangoClient(hosts='http://dbms:{}'.format(ARANGO_PORT))
db = client.db('tdm', username='******', password='******')
datapath_matches = db.collection('DataPathMatch')
datapath_match_exist = datapath_matches.find({'_from': basepath['_id'], '_to': matchpath['_id']})
if datapath_match_exist.count() != 0:
raise Exception('Mapping already exists!')
datapath_match_exist = datapath_matches.find({'_to': basepath['_id'], '_from': matchpath['_id']})
if datapath_match_exist.count() != 0:
raise Exception('Mapping already exists!')
app.logger.debug('Mapping %s <-> %s', basepath['_id'], matchpath['_id'])
datapath_matches.insert(
{
'_from': basepath['_id'],
'_to': matchpath['_id'],
'timestamp': time.time(),
'author': author,
'validated': False,
'weight': weight,
'annotation': annotation,
'needs_human': False if not annotation or weight == 100 else True
}
)
def main():
a = parse_args()
client = ArangoClient(hosts=a.arango_url)
if a.user:
if a.pwd_file:
with open(a.pwd_file) as pwd_file:
pwd = pwd_file.read().strip()
else:
pwd = getpass.getpass()
db = client.db(a.database, a.user, pwd, verify=True)
else:
db = client.db(a.database, verify=True)
attdb = ArangoBatchTimeTravellingDB(
db,
a.load_registry_collection,
a.node_collection,
default_edge_collection=a.edge_collection,
merge_collection=a.merge_edge_collection)
with open(a.file) as f:
obograph = json.loads(f.read())
loader = OBOGraphLoader(obograph, a.onto_id_prefix, graph_id=a.graph_id)
load_graph_delta(a.load_namespace,
loader.get_node_provider(),
loader.get_edge_provider(),
attdb,
a.load_timestamp,
a.release_timestamp,
a.load_version,
merge_source=loader.get_merge_provider())
def check_collection_fields(collection,
fields,
value,
return_single=True,
db=None):
if db is None:
client = ArangoClient(protocol='http', host='dbms')
db = client.db('tdm', username='******', password='******')
collection_ref = db.collection(collection)
return_val = None
exception_messages = []
for field in fields:
check = collection_ref.find({field: value})
if return_single and check.count() > 1:
exception_messages.append(
'More than one document exists for (%s: %s)!' % (field, value))
elif check.count() > 0:
return_val = check
break
else:
exception_messages.append('Unable to find (%s: %s)!' %
(field, value))
if return_val is None and exception_messages:
raise Exception(' '.join(exception_messages))
return return_val
def create_connection(self):
"""Creates ArangoDB connection
"""
# Extract host and port from URL
host, port = self.get_host_port(self.url)
# Extract Arango username and password from environment variable
arango_username = os.environ.get('USER_ARANGO')
arango_password = os.environ.get('PASSWD_ARANGO')
if not arango_username and not arango_password:
# Create Arrango connection
arango_connection = ArangoClient(host=host, port=port)
self.verify_connection(arango_connection)
elif (not arango_username and arango_password) or \
(arango_username and not arango_password):
raise Exception("Invalid credentials, ArangoDB username/"
"password can't be blank")
elif arango_username and arango_password:
# Create Arrango connection
arango_connection = ArangoClient(host=host,
port=port,
username=arango_username,
password=arango_password)
self.verify_connection(arango_connection)
return arango_connection
def validate_token(token):
"""
validate if token is valid and not expired
:param token: JWT token
:return: nothing or dict(mail, uid), 500, 401 or 200
"""
logger.debug(f'validate token request for {token}')
client = ArangoClient(hosts=os.getenv('DB_URI'))
db = client.db(os.getenv('DB_NAME'),
username=os.getenv('DB_USER'),
password=os.getenv('DB_PASSWORD'))
player = validate(db, token)
if player:
try:
jwt.decode(token,
os.getenv('SECRET_KEY'),
algorithms=[os.getenv('TOKEN_ALGO')])
except ExpiredSignatureError:
logger.info(f'token {token} expired for {player.email}')
remove_token(db, player.email, token)
return NoContent, 401
return dict(sub=player.email, uid=player.id), 200
else:
logger.warning(f'unable to validate {token}')
return NoContent, 500
def run_query(self, query, user):
client = ArangoClient(hosts='{}:{}'.format(
self.configuration["host"], self.configuration.get("port", 8529)))
db = client.db(self.configuration["dbname"],
username=self.configuration["user"],
password=self.configuration["password"])
try:
cursor = db.aql.execute(query,
max_runtime=self.configuration.get(
"timeout", 0.0))
result = [i for i in cursor]
column_tuples = [(i, TYPE_STRING) for i in result[0].keys()]
columns = self.fetch_columns(column_tuples)
data = {
"columns": columns,
"rows": result,
}
json_data = json_dumps(data, ignore_nan=True)
error = None
except Exception:
raise
return json_data, error
def main():
a = parse_args()
nodes = os.path.join(a.dir, NODES_IN_FILE)
names = os.path.join(a.dir, NAMES_IN_FILE)
merged = os.path.join(a.dir, MERGED_IN_FILE)
client = ArangoClient(hosts=a.arango_url)
if a.user:
if a.pwd_file:
with open(a.pwd_file) as pwd_file:
pwd = pwd_file.read().strip()
else:
pwd = getpass.getpass()
db = client.db(a.database, a.user, pwd, verify=True)
else:
db = client.db(a.database, verify=True)
attdb = ArangoBatchTimeTravellingDB(
db,
a.load_registry_collection,
a.node_collection,
default_edge_collection=a.edge_collection,
merge_collection=a.merge_edge_collection)
with open(nodes) as in1, open(names) as namesfile, open(nodes) as in2, open(merged) as merge:
nodeprov = NCBINodeProvider(namesfile, in1)
edgeprov = NCBIEdgeProvider(in2)
merge = NCBIMergeProvider(merge)
load_graph_delta(_LOAD_NAMESPACE, nodeprov, edgeprov, attdb,
a.load_timestamp, a.release_timestamp, a.load_version, merge_source=merge)
def __enter__(self) -> "IDataSource":
# create connection to arango
logging.info(
f"host={self.host} port={self.port} username={self.username} database={self.database_name}"
)
self.client = ArangoClient(protocol='http',
host=self.host,
port=self.port)
self.sys_db = self.client.db('_system',
username=self.username,
password=self.password)
# create or open a database
self.database = self.fetch_or_create_database(self.database_name)
# let the user have the permission over the database
self.let_user_access_database(username=self.username,
db_name=self.database_name)
# create the default ks001 to collection id collection if not existing
self.ks001_to_collection = self.fetch_or_create_collection(
ArangoDB.KS001_TO_COLLECTIONID_NAME)
self.add_hash_index_on_field(self.ks001_to_collection,
fields=["ks001", "index"],
unique=True)
return self
def load():
client = ArangoClient()
db = client.db(DB_NAME, username=DB_USER, password=DB_PASS)
users = db.collection('users')
groups = db.collection('groups')
connections = db.collection('connections')
usersInGroups = db.collection('usersInGroups')
user_groups = {}
seed_groups = set(
[group['_key'] for group in groups if group.get('seed', None)])
for user_group in usersInGroups:
if not user_group['_from'].replace('users/', '') in user_groups:
user_groups[user_group['_from'].replace('users/', '')] = set()
user_groups[user_group['_from'].replace('users/', '')].add(
user_group['_to'].replace('groups/', ''))
users_dic = {}
for user in users:
cur_user_groups = user_groups.get(user['_key'])
user_type = 'Seed' if cur_user_groups and (cur_user_groups
& seed_groups) else 'Honest'
users_dic[user['_key']] = graphs.node.Node(user['_key'], user_type,
cur_user_groups,
user['score'])
edges = [(users_dic[connection['_from'].replace('users/', '')],
users_dic[connection['_to'].replace('users/', '')])
for connection in connections]
graph = nx.Graph()
graph.add_edges_from([(edge[0], edge[1]) for edge in edges])
return graph
class Arango:
"""
Client to work with ArangoDB
"""
def __init__(self, database="domain", collection="listings"):
"""
Class constructor
@param database: arango database name
@param collection: arango collection name
"""
self.__client = ArangoClient(hosts=os.getenv("ARANGO_HOST"))
sys_db = self.__client.db("_system", username="", password="")
if "domain" not in sys_db.databases():
sys_db.create_database(database)
print("Creating database")
self.db = self.__client.db("domain", username="", password="")
if "listings" not in [c["name"] for c in self.db.collections()]:
self.db.create_collection(collection)
print("Creating listings collection")
self.collection = self.db.collection(collection)
def __del__(self):
"""
Class destructor
@return: None
"""
self.__client.close()
print("Closing DB Collection")
def ArangoE():
client = ArangoClient(protocol='http',
host='localhost',
port=8529,
username='******',
password='',
enable_logging=True)
db = client.db('graph')
graphPersons = db.graph('graphPersons')
e = graphPersons.edge_collection('know')
id = 1
record = []
cpu = []
mem = []
disk = []
startTime = time.time()
while id <= relationNum:
try:
e.insert(
json.loads(
json.dumps({
'_from': 'persons/' + '%s',
'_to': 'persons/' + '%s'
}) % getRelationTuple(getSingleRelation(id))))
except:
id = writeRecord(id, record, cpu, mem, disk, startTime)
continue
id = writeRecord(id, record, cpu, mem, disk, startTime)
return record, cpu, mem, disk
def login(player):
"""
login user or service
:param player: dict containing mail and password
:return: nothing or token, 500, 200 or 401
"""
if 'mail' not in player or 'password' not in player:
return 'not all parameters supplies', 500
mail = player['mail']
password = player['password']
logger.debug(f'login request for {mail}')
if 'username' in session:
return f'You are already logged in {mail}', 500
else:
client = ArangoClient(hosts=os.getenv('DB_URI'))
db = client.db(os.getenv('DB_NAME'),
username=os.getenv('DB_USER'),
password=os.getenv('DB_PASSWORD'))
player = authenticate(db, mail, password, os.getenv('SECRET_KEY'))
if player:
token = generate_token(mail)
session['username'] = mail
session['uid'] = player.id
session['token'] = token
return token, 200
return NoContent, 401
def delete_users():
cfg = read_data()
mscp = ManagedServiceConnParam()
print("Deleting users before test !")
pl = ['_system', 'root', 'rajiv', 'node2vec_db_admin', 'susr']
protocol = cfg['arangodb'][mscp.DB_CONN_PROTOCOL]
srv_host = cfg['arangodb'][mscp.DB_SERVICE_HOST]
port = cfg['arangodb'][mscp.DB_SERVICE_PORT]
try:
root_user = cfg['arangodb'][mscp.DB_ROOT_USER]
root_user_password = cfg['arangodb'][mscp.DB_ROOT_USER_PASSWORD]
except KeyError as k:
msg = "Root credentials are unvailable, try again " + \
"with a new connection and credentials for root provided"
print(msg)
print("Credential information that is missing : " + k.args[0])
raise Exception("Key error associated with missing " + k.args[0])
host_connection = protocol + "://" + srv_host + ":" + str(port)
# sys_user_name = cfg['arangodb'][mscp.DB_ROOT_USER]
# sys_passwd = cfg['arangodb'][mscp.DB_ROOT_USER_PASSWORD]
client = ArangoClient(hosts= host_connection,\
http_client=CustomHTTPClient(username = root_user,\
password = root_user_password))
sys_db = client.db('_system',\
username=root_user,\
password=root_user_password)
ul = sys_db.users()
unl = [tu['username'] for tu in ul]
for u in unl:
if not u in pl:
sys_db.delete_user(u)
return
async def amain() -> None:
"""
Aynchronous main method of command-line program. Async so it can call async
methods without messing around with event loops.
:return: nothing
"""
config = get_config()
adb_conf = config["arangodb"]["events"]
client = ArangoClient(**adb_conf["client"])
adb = client.db(**adb_conf["database"])
collection = adb.collection("metar")
airport = "KITH"
async with ClientSession() as session:
try:
metar = await get_metar(session, airport)
except ClientConnectorError:
(_, exception, _) = exc_info()
inner_exception = exception.os_error
if isinstance(inner_exception, gaierror) and "Temporary" in str(inner_exception):
return
raise
try:
LOGGER.debug(metar)
collection.insert(metar, silent=True)
except DocumentInsertError:
pass
def get_arangodb_client(protocol, ip_addr, port, database, cred_name,
cred_pass):
hosts = f"{protocol}://{ip_addr}:{port}"
client = ArangoClient(hosts=hosts)
sys_db = client.db(database, username=cred_name, password=cred_pass)
return sys_db
def set_db_connection(self):
client = ArangoClient(hosts='http://localhost:8529')
db = client.db('ITSM_db',\
username='******',\
password='******')
self.db = db
self.emlg = self.db.graph('ITSMg')
def main():
if len(sys.argv) < 6:
print(
"Usage: host database collection template_file number_of_copies\n Example: python BySampleLoader.py 'http://localhost:8529' _system persons person.json 10000000"
)
return
client = ArangoClient(hosts=sys.argv[1])
db = client.db(sys.argv[2])
collection = db.collection(sys.argv[3])
line_limit = int(sys.argv[5]) # how many documents to upload
batch_size = 20000 # batch size for inserting into Arango
data = []
total = 0
template = ''
with open(sys.argv[4]) as json_file:
template = json.load(json_file)
id_template = template['id']
while total < line_limit:
record = template.copy()
record['id'] = id_template + str(total)
record['_key'] = record['id']
data.append(record)
if len(data) > batch_size:
collection.insert_many(data)
data.clear()
print('Loaded ' + str(total) + ' ' +
str(round((total / line_limit) * 100, 2)) + '% \n')
total = total + 1
if len(data) > 0:
collection.insert_many(data)
def ArangoV():
client = ArangoClient(protocol='http',
host='localhost',
port=8529,
username='******',
password='',
enable_logging=True)
db = client.db('graph')
graphPersons = db.graph('graphPersons')
persons = graphPersons.vertex_collection("persons")
id = 1
record = []
cpu = []
mem = []
disk = []
startTime = time.time()
while id <= nodeNum:
persons.insert(
json.loads(
json.dumps({
'_key': '%s',
'name': '%s'
}) % getPersonTuple(getSingleInfo(id))))
id = writeRecord(id, record, cpu, mem, disk, startTime)
return record, cpu, mem, disk
def init_app(self, app):
url = furl(app.config['ARANGODB_DATABASE_URI'])
host = furl(scheme=url.scheme, host=url.host, port=url.port)
db_name = str(url.path).strip('/')
self.client = ArangoClient(hosts=host.url)
self.db = self.client.db(name=db_name, username=url.username, password=url.password)
try:
self.db.status()
except exceptions.ServerStatusError:
sys_db = self.client.db(name="_system", username=url.username, password=url.password)
sys_db.create_database(db_name)
try:
self.db.collection('User').properties()
except exceptions.CollectionPropertiesError:
self.db.create_collection('User')
try:
self.db.collection('Following').properties()
except exceptions.CollectionPropertiesError:
self.db.create_collection('Following', edge=True)
try:
self.db.graph('users-following').properties()
except exceptions.GraphPropertiesError:
self.db.create_graph(
'users-following',
[{
'edge_collection': 'Following',
'from_vertex_collections': ['User'],
'to_vertex_collections': ['User'],
}],
)
def map_datapath_calculation_single(name, description, equation, author,
InCalculation, CalculationResult):
in_calculation_ids = set()
calculation_result_ids = set()
check_fields = ['machine_id', 'human_id']
for in_calc in InCalculation:
in_calc_doc = check_collection_fields('DataPath', check_fields,
in_calc)
if in_calc_doc is not None:
in_calculation_ids.add(in_calc_doc.next()['_id'])
for calc_result in CalculationResult:
calc_result_doc = check_collection_fields('DataPath', check_fields,
calc_result)
if calc_result_doc is not None:
calculation_result_ids.add(calc_result_doc.next()['_id'])
client = ArangoClient(protocol='http', host='dbms')
db = client.db('tdm', username='******', password='******')
calculation_collection = db.collection('Calculation')
calculation_exists = calculation_collection.find({'name': name})
if calculation_exists.count() > 0:
raise Exception('Calculation of name %s already exists!' % name)
app.logger.debug('Adding calculation %s', name)
calc_doc = calculation_collection.insert({
'name': name,
'description': description,
'equation': equation,
'author': author
})
for dp_id in in_calculation_ids:
add_mapping('InCalculation', dp_id, calc_doc['_id'])
for dp_id in calculation_result_ids:
add_mapping('CalculationResult', calc_doc['_id'], dp_id)
def connect_to_adb(config):
"""
Connect to arango database (duplicated all over I know)
:param config:
:return: connection to database
"""
from arango import ArangoClient
client = ArangoClient(**config["arangodb"]["events"]["client"])
return client.db(**config["arangodb"]["events"]["database"])
def connect_to_adb(config):
"""
Connect to arango database
:param config:
:return:
"""
from arango import ArangoClient
client = ArangoClient(**config["arangodb"]["events"]["client"])
return client.db(**config["arangodb"]["events"]["database"])
def connect():
client = ArangoClient(
host=os.getenv('ARANGO_HOST'),
port=os.getenv('ARANGO_PORT')
)
# arango client >= 4 thows only when verify is set to True
client.db(
'_system',
username=os.getenv('ARANGO_USER'),
password=os.getenv('ARANGO_ROOT_PASSWORD'),
verify=True
)
def main() -> None:
"""
main() method of command line program
:return: None
"""
config = get_config()
adb_conf = config["arangodb"]["events"]
client = ArangoClient(**adb_conf["client"])
adb = client.db(**adb_conf["database"])
collection = adb.collection("metar")
collection.add_skiplist_index(["station", "time"])
def _connection(self):
self.username = app.config['ARANGO_USERNAME']
self.password = app.config['ARANGO_PASSWORD']
self.arango_protocol = app.config['ARANGO_PROTOCOL']
self.arango_host = app.config['ARANGO_HOST']
self.arango_port = app.config['ARANGO_PORT']
self._conn = ArangoClient(
protocol=self.arango_protocol,
host=self.arango_host,
port=self.arango_port,
username=self.username,
password=self.password,
enable_logging=True
)
AsyncExecuteError,
AsyncJobCancelError,
AsyncJobClearError,
AsyncJobResultError,
AsyncJobStatusError,
AsyncJobListError,
AQLQueryExecuteError
)
from arango.graph import Graph
from .utils import (
generate_db_name,
generate_col_name
)
arango_client = ArangoClient()
db_name = generate_db_name(arango_client)
db = arango_client.create_database(db_name)
col_name = generate_col_name(db)
col = db.create_collection(col_name)
col.add_fulltext_index(fields=['val'])
def teardown_module(*_):
arango_client.delete_database(db_name, ignore_missing=True)
def setup_function(*_):
col.truncate()
AQLFunctionListError,
AQLFunctionCreateError,
AQLFunctionDeleteError,
AQLCacheClearError,
AQLCacheConfigureError,
AQLCachePropertiesError,
)
from .utils import (
generate_db_name,
generate_col_name,
generate_user_name
)
arango_client = ArangoClient()
db_name = generate_db_name(arango_client)
db = arango_client.create_database(db_name)
col_name = generate_col_name(db)
db.create_collection(col_name)
username = generate_user_name(arango_client)
user = arango_client.create_user(username, 'password')
func_name = ''
func_body = ''
def teardown_module(*_):
arango_client.delete_database(db_name, ignore_missing=True)
arango_client.delete_user(username, ignore_missing=True)
def pytest_configure(config):
client = ArangoClient(
host=config.getoption('host'),
port=config.getoption('port')
)
sys_db = client.db(
name='_system',
username='******',
password=config.getoption('passwd')
)
# Create a user and non-system database for testing.
username = generate_username()
password = generate_string()
tst_db_name = generate_db_name()
bad_db_name = generate_db_name()
sys_db.create_database(
name=tst_db_name,
users=[{
'active': True,
'username': username,
'password': password,
}]
)
sys_db.update_permission(
username=username,
permission='rw',
database=tst_db_name
)
tst_db = client.db(tst_db_name, username, password)
bad_db = client.db(bad_db_name, username, password)
# Create a standard collection for testing.
col_name = generate_col_name()
tst_col = tst_db.create_collection(col_name, edge=False)
tst_col.add_skiplist_index(['val'])
tst_col.add_fulltext_index(['text'])
geo_index = tst_col.add_geo_index(['loc'])
# Create a legacy edge collection for testing.
lecol_name = generate_col_name()
tst_db.create_collection(lecol_name, edge=True)
# Create test vertex & edge collections and graph.
graph_name = generate_graph_name()
ecol_name = generate_col_name()
fvcol_name = generate_col_name()
tvcol_name = generate_col_name()
tst_graph = tst_db.create_graph(graph_name)
tst_graph.create_vertex_collection(fvcol_name)
tst_graph.create_vertex_collection(tvcol_name)
tst_graph.create_edge_definition(
edge_collection=ecol_name,
from_vertex_collections=[fvcol_name],
to_vertex_collections=[tvcol_name]
)
global_data.update({
'client': client,
'username': username,
'password': password,
'sys_db': sys_db,
'tst_db': tst_db,
'bad_db': bad_db,
'geo_index': geo_index,
'col_name': col_name,
'lecol_name': lecol_name,
'graph_name': graph_name,
'ecol_name': ecol_name,
'fvcol_name': fvcol_name,
'tvcol_name': tvcol_name,
})
from __future__ import absolute_import, unicode_literals
from uuid import UUID
import pytest
from arango import ArangoClient
from arango.aql import AQL
from arango.collections import Collection
from arango.exceptions import DocumentRevisionError, DocumentInsertError, BatchExecuteError
from arango.graph import Graph
from .utils import generate_db_name, generate_col_name
arango_client = ArangoClient()
db_name = generate_db_name(arango_client)
db = arango_client.create_database(db_name)
bad_db_name = generate_db_name(arango_client)
bad_db = arango_client.db(bad_db_name)
col_name = generate_col_name(db)
col = db.create_collection(col_name)
def teardown_module(*_):
arango_client.delete_database(db_name, ignore_missing=True)
def setup_function(*_):
col.truncate()
from __future__ import absolute_import, unicode_literals
import pytest
from arango import ArangoClient
from arango.exceptions import (
WALConfigureError,
WALFlushError,
WALPropertiesError,
WALTransactionListError
)
from .utils import generate_user_name
arango_client = ArangoClient()
wal = arango_client.wal
username = generate_user_name(arango_client)
user = arango_client.create_user(username, 'password')
def teardown_module(*_):
arango_client.delete_user(username, ignore_missing=True)
@pytest.mark.order1
def test_wal_properties():
properties = wal.properties()
assert 'ArangoDB write-ahead log' in repr(wal)
assert 'oversized_ops' in properties
assert 'log_size' in properties
assert 'historic_logs' in properties
class DB(object):
"""DB"""
collection = None
database = None
graph = None
conn = None
edge = None
def __init__(self):
self._connection()
def _connection(self):
self.username = app.config['ARANGO_USERNAME']
self.password = app.config['ARANGO_PASSWORD']
self.arango_protocol = app.config['ARANGO_PROTOCOL']
self.arango_host = app.config['ARANGO_HOST']
self.arango_port = app.config['ARANGO_PORT']
self._conn = ArangoClient(
protocol=self.arango_protocol,
host=self.arango_host,
port=self.arango_port,
username=self.username,
password=self.password,
enable_logging=True
)
############
# DATABASE #
############
def has_database(self, name=''):
"""Return True if there database"""
try:
database = self._conn.database(name)
database.properties()
except exceptions.DatabasePropertiesError:
return False
else:
return True
def get_database(self, name=''):
"""Return database"""
if not name:
name = app.config['ARANGO_DB']
if self.has_database(name):
self.database = self._conn.database(name)
return self.database
else:
msg = db_err.get(1228).format(name)
raise gmap_exceptions.DatabaseNotExist(msg)
def create_database(self, name=''):
"""Create DB"""
try:
self.database = self._conn.create_database(name)
return self.database
except exceptions.DatabaseCreateError as err:
if err.error_code == 1207:
msg = db_err.get(1207).format(name)
raise gmap_exceptions.DatabaseAlreadyExist(msg)
else:
msg = db_err.get(0).format(name, err.message)
raise gmap_exceptions.DatabaseException(msg)
except Exception as err:
msg = db_err.get(0).format(name, err.message)
raise gmap_exceptions.DatabaseException(msg)
def delete_database(self, name=''):
"""Delete DB"""
try:
self._conn.delete_database(name)
self.database = None
return True
except exceptions.DatabaseDeleteError as err:
if err.error_code == 1228:
msg = db_err.get(1228).format(name)
raise gmap_exceptions.DatabaseNotExist(msg)
else:
msg = db_err.get(0).format(name, err.message)
raise gmap_exceptions.DatabaseException(msg)
except Exception as err:
msg = db_err.get(0).format(name, err.message)
raise gmap_exceptions.DatabaseException(msg)
def search_in_database(self, collection, field, value, offset=0, count=10):
"""Search Document"""
# TODO: To use a better way to search
try:
if field and value:
where = 'FILTER LOWER(doc.`{}`) like "%{}%"'.format(
field, value.lower())
else:
where = ''
cursor = self.database.aql.execute('''
FOR doc IN {} {} LIMIT {}, {} RETURN doc'''.format(
collection, where, offset, count),
count=True,
batch_size=1,
ttl=10,
optimizer_rules=['+all']
)
return cursor
except Exception as err:
msg = db_err.get(1).format(err.message)
raise gmap_exceptions.DatabaseException(msg)
###############
# COLLECTIONS #
###############
def _has_collection(self, name='', edge=False):
"""Return True if there collection/edge"""
try:
collection = self.database.collection(name)
res = collection.properties().get('edge') is edge
# import pdb; pdb.Pdb().set_trace()
# if res is edge:
return res
except exceptions.CollectionPropertiesError:
return False
def has_collection(self, name=''):
"""Return True if there collection"""
return self._has_collection(name, False)
def get_collection(self, name=''):
"""Return Collection"""
if self.has_collection(name):
self.collection = self.database.collection(name)
return self.collection
else:
msg = coll_err.get(1228).format(name)
raise gmap_exceptions.CollectionNotExist(msg)
def create_collection(self, name='', edge=False):
"""Create Collection"""
try:
self.collection = self.database.create_collection(
name=name, edge=False)
return self.collection
except exceptions.CollectionCreateError as err:
if err.error_code == 1207:
msg = coll_err.get(1207).format(name)
raise gmap_exceptions.CollectionAlreadyExist(msg)
else:
msg = coll_err.get(0).format(name, err.message)
raise gmap_exceptions.CollectionException(msg)
except Exception as err:
msg = coll_err.get(0).format(name, err.message)
raise gmap_exceptions.CollectionException(msg)
else:
return True
def delete_collection(self, name=''):
"""Delete Collection """
try:
self.database.delete_collection(name=name)
self.collection = None
return True
except exceptions.CollectionDeleteError as err:
if err.error_code == 1203:
msg = coll_err.get(1228).format(name)
raise gmap_exceptions.CollectionNotExist(msg)
else:
msg = coll_err.get(0).format(name, err.message)
raise gmap_exceptions.CollectionException(msg)
except Exception as err:
msg = coll_err.get(0).format(name, err.message)
raise gmap_exceptions.CollectionException(msg)
def has_edge(self, name=''):
"""Return True if there edge"""
return self._has_collection(name, True)
def get_edge(self, name='', ):
"""Return Edge"""
if self.has_edge(name):
self.edge = self.database.collection(name)
return self.edge
else:
msg = edge_err.get(1228).format(name)
raise gmap_exceptions.EdgeNotExist(msg)
def create_edge(self, name=''):
"""Create Edge"""
try:
self.edge = self.database.create_collection(
name=name, edge=True)
return self.edge
except exceptions.CollectionCreateError as err:
if err.error_code == 1207:
msg = edge_err.get(1207).format(name)
raise gmap_exceptions.EdgeAlreadyExist(msg)
else:
msg = edge_err.get(0).format(name, err.message)
raise gmap_exceptions.EdgeException(msg)
except Exception as err:
msg = edge_err.get(0).format(name, err.message)
raise gmap_exceptions.EdgeException(msg)
def delete_edge(self, name=''):
"""Delete Edge """
try:
self.database.delete_collection(name=name)
self.edge = None
return True
except exceptions.CollectionDeleteError as err:
if err.error_code == 1203:
msg = edge_err.get(1228).format(name)
raise gmap_exceptions.EdgeNotExist(msg)
else:
msg = edge_err.get(0).format(name, err.message)
raise gmap_exceptions.EdgeException(msg)
except Exception as err:
msg = edge_err.get(0).format(name, err.message)
raise gmap_exceptions.EdgeException(msg)
else:
return True
##########
# GRAPHS #
##########
def has_graph(self, name=''):
"""Return True if there graph"""
try:
graph = self.database.graph(name)
graph.properties()
return True
except exceptions.GraphPropertiesError:
return False
def get_graph(self, name=''):
"""Return graph"""
if self.has_graph(name):
self.graph = self.database.graph(name)
return self.graph
else:
msg = gph_err.get(1924).format(name)
raise gmap_exceptions.GraphNotExist(msg)
def create_graph(self, name='', edge_definitions=None):
"""Create Graph"""
try:
self.graph = self.database.create_graph(name)
except exceptions.GraphCreateError as err:
if err.error_code == 1925:
msg = gph_err.get(1925).format(name, err)
raise gmap_exceptions.GraphAlreadyExist(msg)
else:
msg = gph_err.get(0).format(name, err.message)
raise gmap_exceptions.GraphException(msg)
except Exception as err:
msg = gph_err.get(0).format(name, err.message)
raise gmap_exceptions.GraphException(msg)
else:
if edge_definitions:
for edge in edge_definitions:
try:
self.graph.create_edge_definition(
name=edge.get('edge'),
from_collections=edge.get('from_collections'),
to_collections=edge.get('to_collections')
)
except Exception as err:
self.database.delete_graph(name)
msg = gph_err.get(1).format(name)
raise gmap_exceptions.GraphException(msg)
return self.graph
def delete_graph(self, name=''):
"""Delete Graph"""
try:
self.database.delete_graph(name)
return True
except exceptions.GraphDeleteError as err:
if err.error_code == 1924:
msg = gph_err.get(1924).format(name)
raise gmap_exceptions.GraphNotExist(msg)
else:
msg = gph_err.get(0).format(name, err.message)
raise gmap_exceptions.GraphException(msg)
except Exception as err:
msg = gph_err.get(0).format(name, err.message)
raise gmap_exceptions.GraphException(msg)
