Пример #1
0
def map_datapath_calculation_single(name, description, equation, author, InCalculation, CalculationResult):
    in_calculation_ids = set()
    calculation_result_ids = set()
    check_fields = ['machine_id', 'human_id']
    for in_calc in InCalculation:
        in_calc_doc = check_collection_fields('DataPath', check_fields, in_calc)
        if in_calc_doc is not None:
            in_calculation_ids.add(in_calc_doc.next()['_id'])
    for calc_result in CalculationResult:
        calc_result_doc = check_collection_fields('DataPath', check_fields, calc_result)
        if calc_result_doc is not None:
            calculation_result_ids.add(calc_result_doc.next()['_id'])
    client = ArangoClient(hosts='http://dbms:{}'.format(ARANGO_PORT))
    db = client.db('tdm', username='******', password='******')
    calculation_collection = db.collection('Calculation')
    calculation_exists = calculation_collection.find({'name': name})
    if calculation_exists.count() > 0:
        raise Exception('Calculation of name %s already exists!' % name)
    app.logger.debug('Adding calculation %s', name)
    calc_doc = calculation_collection.insert(
        {
            'name': name,
            'description': description,
            'equation': equation,
            'author': author
        }
    )
    for dp_id in in_calculation_ids:
        add_mapping('InCalculation', dp_id, calc_doc['_id'])
    for dp_id in calculation_result_ids:
        add_mapping('CalculationResult', calc_doc['_id'], dp_id)
Пример #2
0
def main():
  # Override csv default 128k field size
  csv.field_size_limit(int(ctypes.c_ulong(-1).value // 2))

  # Initialize the client for ArangoDB.
  client = ArangoClient(hosts='http://localhost:8529')
  
  # Upload settings
  filename = 'C:\\Working\\iresearch-toolkit\\iresearch\\iresearch.deps\\benchmark_resources\\benchmark.data' # data file
  collection = 'wikipedia' # target collection
  database = '_system'  # target database
  line_limit = 10000000 # how many documents to upload
  batch_size = 10000    # batch size for inserting into Arango

  db = client.db(database)
  wikipedia = db.collection(collection)
  f = open(filname, mode ='r', encoding='utf-8', errors='replace')
  reader = csv.reader(f, delimiter='\t')
  data = []
  total = 0
  for row in reader:
    data.append({'title': row[0], 'body': row[2]})
    if len(data) > batch_size:
      wikipedia.insert_many(data)
      data.clear()
    total = total + 1
    if total >= line_limit:
      break
  if len(data) > 0:
    wikipedia.insert_many(data)
  f.close()
Пример #3
0
def main():
    print("Start")

    client = ArangoClient(hosts=HOSTS)
    db = client.db("shylock_test", username=USERNAME, password=PASSWORD)
    configure(ShylockPythonArangoBackend.create(db, "shylock"))
    lock_name = "test-lock"

    test_lock = Lock(lock_name)
    try:
        with Lock(lock_name):
            print("Got lock")
            print("Testing re-lock")
            assert not test_lock.acquire(False)
            raise ValueError()
    except ValueError:
        print("Caught exception, lock should be released")

    assert test_lock.acquire(False)
    test_lock.release()

    print("Testing automatic release, this might take a while.")

    # Test automatic release
    start = time()
    with test_lock:
        lock2 = Lock(lock_name)
        try:
            lock2.acquire()
            released = time() - start
        finally:
            lock2.release()
    print(f"Lock automatically released after {released:.3f}s")
Пример #4
0
def map_datapath_single_by_key(basepath_key, matchpath_key, author, weight, annotation):
    basepath = fetch_datapath(basepath_key)
    if not basepath:
        raise Exception('Specified base DataPath not found!')
    matchpath = fetch_datapath(matchpath_key)
    if not matchpath:
        raise Exception('Specified matching DataPath not found!')
    client = ArangoClient(hosts='http://dbms:{}'.format(ARANGO_PORT))
    db = client.db('tdm', username='******', password='******')
    datapath_matches = db.collection('DataPathMatch')
    datapath_match_exist = datapath_matches.find({'_from': basepath['_id'], '_to': matchpath['_id']})
    if datapath_match_exist.count() != 0:
        raise Exception('Mapping already exists!')
    datapath_match_exist = datapath_matches.find({'_to': basepath['_id'], '_from': matchpath['_id']})
    if datapath_match_exist.count() != 0:
        raise Exception('Mapping already exists!')
    app.logger.debug('Mapping %s <-> %s', basepath['_id'], matchpath['_id'])
    datapath_matches.insert(
        {
            '_from': basepath['_id'],
            '_to': matchpath['_id'],
            'timestamp': time.time(),
            'author': author,
            'validated': False,
            'weight': weight,
            'annotation': annotation,
            'needs_human': False if not annotation or weight == 100 else True
        }
    )
Пример #5
0
    def create_db(self):
        client = ArangoClient(protocol='http',\
                              host=self.cfg['arangodb']['host'],\
                              port=self.cfg['arangodb']['port'])

        # Connect to "_system" database as root user.
        # This returns an API wrapper for "_system" database.
        sys_db = client.db('_system',\
                           username=self.cfg['arangodb']['root_user'],
                           password=self.cfg['arangodb']['root_user_password'])

        # Create a new arangopipe database if it does not exist.
        if not sys_db.has_database(self.cfg['arangodb']['arangopipe_dbname']):
            sys_db.create_database(self.cfg['arangodb']['arangopipe_dbname'])

        if not sys_db.has_user(
                self.cfg['arangodb']['arangopipe_admin_username']):
            sys_db.create_user(username = self.cfg['arangodb']['arangopipe_admin_username'],\
                               password = self.cfg['arangodb']['arangopipe_admin_password'])

        sys_db.update_permission(username = self.cfg['arangodb']['arangopipe_admin_username'],\
                                 database = self.cfg['arangodb']['arangopipe_dbname'], permission = "rw")

        # Connect to arangopipe database as administrative user.
        #This returns an API wrapper for "test" database.
        db = client.db(self.cfg['arangodb']['arangopipe_dbname'],\
                       username=self.cfg['arangodb']['arangopipe_admin_username'],\
                       password=self.cfg['arangodb']['arangopipe_admin_password'])
        self.db = db

        return
def main():
    a = parse_args()
    client = ArangoClient(hosts=a.arango_url)
    if a.user:
        if a.pwd_file:
            with open(a.pwd_file) as pwd_file:
                pwd = pwd_file.read().strip()
        else:
            pwd = getpass.getpass()
        db = client.db(a.database, a.user, pwd, verify=True)
    else:
        db = client.db(a.database, verify=True)
    attdb = ArangoBatchTimeTravellingDB(
        db,
        a.load_registry_collection,
        a.node_collection,
        default_edge_collection=a.edge_collection)

    TaxNode.parse_taxfile(a.input_dir)
    SeqNode.parse_fastas(a.input_dir)
    nodeprov = SILVANodeProvider()
    edgeprov = SILVAEdgeProvider()
    print('got node/edge providers')

    load_graph_delta(_LOAD_NAMESPACE, nodeprov, edgeprov, attdb,
                     a.load_timestamp, a.release_timestamp, a.load_version)
Пример #7
0
    def delete_arangopipe_db(self):
        print("Deleting users before test !")

        host_connection = self.test_cfg['arangodb'][self.mscp.DB_CONN_PROTOCOL] + "://" + \
        self.test_cfg['arangodb'][self.mscp.DB_SERVICE_HOST] + ":" + \
        str(self.test_cfg['arangodb'][self.mscp.DB_SERVICE_PORT])
        try:
            root_user = self.test_cfg['arangodb'][self.mscp.DB_ROOT_USER]
            root_user_password = self.test_cfg['arangodb'][
                self.mscp.DB_ROOT_USER_PASSWORD]
        except KeyError as k:
            msg = "Root credentials are unvailable, try again " + \
                     "with a new connection and credentials for root provided"
            print(msg)
            print("Credential information that is missing : " + k.args[0])
            raise Exception("Key error associated with missing " + k.args[0])
        client = ArangoClient(hosts= host_connection,\
                            http_client=CustomHTTPClient(username=root_user,\
                                                         password= root_user_password))
        sys_db = client.db('_system',\
                           username=self.test_cfg['arangodb'][self.mscp.DB_ROOT_USER],\
                           password=self.test_cfg['arangodb'][self.mscp.DB_ROOT_USER_PASSWORD] )
        try:
            if sys_db.has_database("arangopipe"):
                print(
                    "Before starting the test, cleaning up arangopipe instances..."
                )
                sys_db.delete_database("arangopipe")
            else:
                print("Test Prep: The database arangopipe does not exist !")

        except DatabaseListError as err:
            print.error(err)
            print("Error code: " + str(err.error_code) + " received !")
            print("Error Message: " + str(err.error_message))
Пример #8
0
def load():
    client = ArangoClient()
    db = client.db(DB_NAME, username=DB_USER, password=DB_PASS)
    users = db.collection('users')
    groups = db.collection('groups')
    connections = db.collection('connections')
    usersInGroups = db.collection('usersInGroups')
    user_groups = {}
    seed_groups = set(
        [group['_key'] for group in groups if group.get('seed', None)])
    for user_group in usersInGroups:
        if not user_group['_from'].replace('users/', '') in user_groups:
            user_groups[user_group['_from'].replace('users/', '')] = set()
        user_groups[user_group['_from'].replace('users/', '')].add(
            user_group['_to'].replace('groups/', ''))
    users_dic = {}
    for user in users:
        cur_user_groups = user_groups.get(user['_key'])
        user_type = 'Seed' if cur_user_groups and (cur_user_groups
                                                   & seed_groups) else 'Honest'
        users_dic[user['_key']] = graphs.node.Node(user['_key'], user_type,
                                                   cur_user_groups,
                                                   user['score'])
    edges = [(users_dic[connection['_from'].replace('users/', '')],
              users_dic[connection['_to'].replace('users/', '')])
             for connection in connections]
    graph = nx.Graph()
    graph.add_edges_from([(edge[0], edge[1]) for edge in edges])
    return graph
Пример #9
0
def get_db():
    if common_config.ARANGODB_CLUSTER:
        hosts = []
        for protocol, host, port in common_config.ARANGODB_HOST_POOL:
            hosts.append(ArangoClient(protocol=protocol, host=host, port=port))
        return ConnectionPool(hosts,
                              dbname=common_config.ARANGODB_DATABASE,
                              password=common_config.ARANGODB_PASSWORD,
                              username=common_config.ARANGODB_USER)
    else:
        protocol, host, port = common_config.ARANGODB_HOST
        client = ArangoClient(protocol=protocol, host=host, port=port)
        return Database(
            client.db(name=common_config.ARANGODB_DATABASE,
                      username=common_config.ARANGODB_USER,
                      password=common_config.ARANGODB_PASSWORD))
Пример #10
0
def create_clean_db(wait_for_api):
    client = ArangoClient(hosts=os.getenv('DB_URI'))
    sys_db = client.db('_system', username=os.getenv('DB_USER'), password=os.getenv('DB_PASSWORD'))
    if sys_db.has_database(os.getenv('DB_NAME')):
        sys_db.delete_database(os.getenv('DB_NAME'))
    sys_db.create_database(os.getenv('DB_NAME'))
    return client.db(os.getenv('DB_NAME'), username=os.getenv('DB_USER'), password=os.getenv('DB_PASSWORD'))
Пример #11
0
def startup():
    global arangodb_client
    arangodb_client = ArangoClient(host=settings.ARANGODB_HOST)
    arangodb_client = arangodb_client.db(settings.ARANGODB_DATABASE,
                                         username=settings.ARANGODB_USERNAME,
                                         password=str(
                                             settings.ARANGODB_PASSWORD))
Пример #12
0
def all_games():
    client = ArangoClient(hosts=os.getenv('DB_URI'))
    db = client.db(os.getenv('DB_NAME'), username=os.getenv('DB_USER'), password=os.getenv('DB_PASSWORD'))
    result = []
    for name in get_all_games(db):
        result.append(json.dumps(read(db, name), cls=MarugotoEncoder))
    return result, 200
Пример #13
0
 def set_db_connection(self):
     client = ArangoClient(hosts='http://localhost:8529')
     db = client.db('ITSM_db',\
                    username='******',\
                    password='******')
     self.db = db
     self.emlg = self.db.graph('ITSMg')
def main():
    a = parse_args()
    url = urlparse(a.arango_url)
    client = ArangoClient(protocol=url.scheme,
                          host=url.hostname,
                          port=url.port)
    if a.user:
        if a.pwd_file:
            with open(a.pwd_file) as pwd_file:
                pwd = pwd_file.read().strip()
        else:
            pwd = getpass.getpass()
        db = client.db(a.database, a.user, pwd, verify=True)
    else:
        db = client.db(a.database, verify=True)
    attdb = ArangoBatchTimeTravellingDB(
        db,
        a.load_registry_collection,
        a.node_collection,
        default_edge_collection=a.edge_collection)

    with open(a.file) as in1, open(a.file) as in2:
        nodeprov = GTDBNodeProvider(in1)
        edgeprov = GTDBEdgeProvider(in2)

        load_graph_delta(_LOAD_NAMESPACE, nodeprov, edgeprov, attdb,
                         a.load_timestamp, a.release_timestamp, a.load_version)
Пример #15
0
 def register_model(self, mi, user_id = "authorized_user",\
                    project = "Wine-Quality-Regression-Modelling"):
     """ Register a model. The operation requires specifying a user id. If the user id is permitted to register a model, then the registration proceeds, otherwise an unauthorized operation is indicated. """
     models = self.emlg.vertex_collection("models")
     model_reg = models.insert(mi)
     
     client = ArangoClient(protocol='http',\
                           host=self.cfg['arangodb']['host'],\
                           port=self.cfg['arangodb']['port'])
     db = client.db(name=self.cfg['arangodb']['arangopipe_dbname'],\
                    username=self.cfg['arangodb']['arangopipe_admin_username'],\
                    password=self.cfg['arangodb']['arangopipe_admin_password'])
     # Execute the query
     cursor = db.aql.execute(
             'FOR doc IN project FILTER doc.name == @value RETURN doc',
             bind_vars={'value': project})
     project_keys = [doc for doc in cursor]
     the_project_info = project_keys[0]
     
     project_model_edge = self.emlg.edge_collection("project_models")
     project_model_key = the_project_info["_key"] + "-" + model_reg["_key"]
     
     a_project_model_edge = {"_key": project_model_key,\
                     "_from": "project/" + the_project_info["_key"],\
                     "_to": "models/" + model_reg["_key"]}
     pm_reg = project_model_edge.insert(a_project_model_edge)
     logger.info("Recording project model link " + str(pm_reg))
     
     
     return model_reg
Пример #16
0
 def log_serving_perf(self, sp, dep_tag, userid = "authorized user"):
 
     """ Log serving performance against a deployed model. The user making the request needs to be authorized to log this performance update. A serving performance vertex is created and is linked with its deployment vertex""" 
     servingperf = self.emlg.vertex_collection("servingperf")
     sp_reg = servingperf.insert(sp)
     # Locate the deployment record
     client = ArangoClient(protocol='http',\
                           host=self.cfg['arangodb']['host'],\
                           port=self.cfg['arangodb']['port'])
     db = client.db(name=self.cfg['arangodb']['arangopipe_dbname'],\
                    username=self.cfg['arangodb']['arangopipe_admin_username'],\
                    password=self.cfg['arangodb']['arangopipe_admin_password'])
     # Execute the query
     cursor = db.aql.execute(
             'FOR doc IN deployment FILTER doc.tag == @value RETURN doc',
             bind_vars={'value': dep_tag})
     dep_docs = [doc for doc in cursor]
     the_dep_doc = dep_docs[0]
     # Link the service performance record with the deployment record
     dep_servingperf_edge = self.emlg.edge_collection("deployment_servingperf")
     dep_servingperf_key = the_dep_doc["_key"] + "-" + sp_reg["_key"]
     the_dep_servingperf_edge = { "_key": dep_servingperf_key,\
                             "_from": the_dep_doc["_id"],\
                             "_to": sp_reg["_id"]}
     
     dep_servingperf_reg = dep_servingperf_edge.insert(the_dep_servingperf_edge)
     return dep_servingperf_reg
Пример #17
0
    def delete_users(self):
        print("Deleting users before test !")
        pl = ['_system', 'root', 'rajiv', 'node2vec_db_admin', 'susr']
        host_connection = self.test_cfg['arangodb'][self.mscp.DB_CONN_PROTOCOL] + "://" + \
        self.test_cfg['arangodb'][self.mscp.DB_SERVICE_HOST] + ":" + \
        str(self.test_cfg['arangodb'][self.mscp.DB_SERVICE_PORT])
        try:
            root_user = self.test_cfg['arangodb'][self.mscp.DB_ROOT_USER]
            root_user_password = self.test_cfg['arangodb'][
                self.mscp.DB_ROOT_USER_PASSWORD]
        except KeyError as k:
            msg = "Root credentials are unvailable, try again " + \
                     "with a new connection and credentials for root provided"
            print(msg)
            print("Credential information that is missing : " + k.args[0])
            raise Exception("Key error associated with missing " + k.args[0])

        client = ArangoClient(hosts= host_connection,\
                            http_client=CustomHTTPClient(username=root_user,\
                                                         password= root_user_password))
        sys_db = client.db('_system',\
                           username=self.test_cfg['arangodb'][self.mscp.DB_ROOT_USER],\
                           password=self.test_cfg['arangodb'][self.mscp.DB_ROOT_USER_PASSWORD])
        ul = sys_db.users()
        unl = [tu['username'] for tu in ul]
        for u in unl:
            if not u in pl:
                sys_db.delete_user(u)

        return
Пример #18
0
 def init_graph(self):
     """ Initialize a graph when an instance of ArangoPipe is provisioned. """
     client = ArangoClient(protocol='http',\
                           host=self.cfg['arangodb']['host'],\
                           port=self.cfg['arangodb']['port'])
 
     # Connect to "_system" database as root user.
     # This returns an API wrapper for "_system" database.
     sys_db = client.db('_system',\
                        username=self.cfg['arangodb']['root_user'],\
                        password=self.cfg['arangodb']['root_user_password'])
     
     if not sys_db.has_database(self.cfg['arangodb']['arangopipe_dbname']):
         logger.error("arangopipe database has not been created.")
         raise AttributeError("arangopipe database has not been created!")
     else:
         db = client.db(name=self.cfg['arangodb']['arangopipe_dbname'],\
                        username=self.cfg['arangodb']['arangopipe_admin_username'],\
                        password=self.cfg['arangodb']['arangopipe_admin_password'])
         if db.has_graph(self.cfg['mlgraph']['graphname']):
             self.emlg = db.graph(self.cfg['mlgraph']['graphname'])
         else:
             logger.error("ML tracking graph was not created. ")
             raise AttributeError("ML tracking graph has not been created")
     logger.info("Arango Pipe ML Graph initialized")
     return
Пример #19
0
def verify(fname):
    print('DOLLAR FOR EVERYONE')
    db = ArangoClient().db('_system')
    for admin in db['users'].find({'dfeAdmin': True}):
        conns1 = [
            c['_to'] for c in db['connections'].find({'_from': admin['_id']})
            if c['timestamp'] > 1564600000000
        ]
        conns2 = [
            c['_from'] for c in db['connections'].find({'_to': admin['_id']})
            if c['timestamp'] > 1564600000000
        ]
        for u in conns1 + conns2:
            u = u.replace('users/', '')
            verifications = set(
                [v['name'] for v in db['verifications'].find({'user': u})])
            if 'DollarForEveryone' not in verifications:
                db['verifications'].insert({
                    'name': 'DollarForEveryone',
                    'user': u,
                    'timestamp': int(time.time() * 1000)
                })
                print('user: {}\tverification: DollarForEveryone'.format(u))
    verifiedCount = db['verifications'].find({
        'name': 'DollarForEveryone'
    }).count()
    print('verifieds: {}\n'.format(verifiedCount))
Пример #20
0
def ArangoV():
    client = ArangoClient(protocol='http',
                          host='localhost',
                          port=8529,
                          username='******',
                          password='',
                          enable_logging=True)
    db = client.db('graph')
    graphPersons = db.graph('graphPersons')
    id = 1
    record = []
    cpu = []
    mem = []
    disk = []
    startTime = time.time()
    while id <= nodeNum:
        a = []
        a.append('persons' + getRamdomNo())
        a.append('persons' + getRamdomNo())
        try:
            db.aql.execute(
                "for v,e in outbound shortest_path '%s' to '%s' graph 'graphPersons' return [v._key,e._key]"
                % tuple(a))
        except:
            id = writeRecord(id, record, cpu, mem, disk, startTime)
            continue
        id = writeRecord(id, record, cpu, mem, disk, startTime)
    return record, cpu, mem, disk
Пример #21
0
    def create_db(self):
        client = ArangoClient(hosts='http://localhost:8529')

        # Connect to "_system" database as root user.
        # This returns an API wrapper for "_system" database.
        sys_db = client.db('_system',\
                       username='******',
                       password='******')

        # Create a new arangopipe database if it does not exist.
        if not sys_db.has_database('ITSM_db'):
            sys_db.create_database('ITSM_db')

        if not sys_db.has_user('ITSM_db_admin'):
            sys_db.create_user(username = '******',\
                               password = '******')

        sys_db.update_permission(username = '******',\
                                 database = 'ITSM_db', permission = "rw")

        # Connect to arangopipe database as administrative user.
        #This returns an API wrapper for "test" database.
        db = client.db('ITSM_db',\
                       username='******',\
                       password='******')
        self.db = db

        return
Пример #22
0
 def test_credentials_database(self):
     client = ArangoClient(hosts="http://localhost:8529")
     sys_db = client.db("_system", username="******", password="******")
     # first try to delete the test database
     sys_db.delete_database('test', ignore_missing=True)
     print("create DB")
     self.assertTrue(create_database(sys_db, dbname='test'))
     db = client.db("test", username="******", password="******")
     # test collection creation
     print("create collections")
     self.assertTrue(create_collections(db))
     # test create user
     print("add credential")
     credentialid = "abcd"
     credentialdict = {"a": ['b'], "c": 1}
     status, result = add_credential(db, credentialid, credentialdict)
     self.assertTrue(status)
     self.assertEqual(credentialid, result["_key"])
     print("get credential")
     status, result = get_credential(db, credentialid)
     self.assertTrue(status)
     self.assertEqual(credentialid, result["_key"])
     self.assertEqual(credentialdict["a"], result["a"])
     print("delete credential")
     status = delete_credential(db, credentialid)
     self.assertTrue(status)
     print("check missing credential")
     status, result = get_credential(db, credentialid)
     self.assertFalse(status)
     # close sessions
     client.close()
Пример #23
0
    def __init__(self,
                 database,
                 protocol='http',
                 host='localhost',
                 port=8529,
                 username='******',
                 password=''):
        """Initialize an ArangoDB Handler to handle all arangodb operations.

        :param database: (str) The database name to use
        :param protocol: (str) The protocol
        :param host: (str) The host
        :param port: (int) The port
        :param username: (str)
        :param password:
        """
        self._database = database
        self.username = username
        self.password = password
        self.client = ArangoClient(protocol=protocol, host=host, port=port)
        self._system_db = self.client.db('_system',
                                         username=self.username,
                                         password=self.password)

        if not self._system_db.has_database(self._database):
            self._system_db.create_database(self._database)

        self._database = self.client.db(self._database,
                                        username=self.username,
                                        password=self.password)
        self.collections = {}
Пример #24
0
def pytest_configure(config):
    client = ArangoClient(host=config.getoption('host'),
                          port=config.getoption('port'))
    sys_db = client.db(name='_system',
                       username='******',
                       password=config.getoption('passwd'))

    # Create a user and non-system database for testing.
    username = generate_username()
    password = generate_string()
    tst_db_name = generate_db_name()
    bad_db_name = generate_db_name()
    sys_db.create_database(name=tst_db_name,
                           users=[{
                               'active': True,
                               'username': username,
                               'password': password,
                           }])
    tst_db = client.db(tst_db_name, username, password)
    bad_db = client.db(bad_db_name, username, password)

    # Create a standard collection for testing.
    col_name = generate_col_name()
    tst_col = tst_db.create_collection(col_name, edge=False)
    tst_col.add_skiplist_index(['val'])
    tst_col.add_fulltext_index(['text'])
    geo_index = tst_col.add_geo_index(['loc'])

    # Create a legacy edge collection for testing.
    lecol_name = generate_col_name()
    tst_db.create_collection(lecol_name, edge=True)

    # Create test vertex & edge collections and graph.
    graph_name = generate_graph_name()
    ecol_name = generate_col_name()
    fvcol_name = generate_col_name()
    tvcol_name = generate_col_name()
    tst_graph = tst_db.create_graph(graph_name)
    tst_graph.create_vertex_collection(fvcol_name)
    tst_graph.create_vertex_collection(tvcol_name)
    tst_graph.create_edge_definition(edge_collection=ecol_name,
                                     from_vertex_collections=[fvcol_name],
                                     to_vertex_collections=[tvcol_name])

    global_data.update({
        'client': client,
        'username': username,
        'password': password,
        'sys_db': sys_db,
        'tst_db': tst_db,
        'bad_db': bad_db,
        'geo_index': geo_index,
        'col_name': col_name,
        'lecol_name': lecol_name,
        'graph_name': graph_name,
        'ecol_name': ecol_name,
        'fvcol_name': fvcol_name,
        'tvcol_name': tvcol_name,
    })
def main():

    parser = argparse.ArgumentParser()
    parser.add_argument("-d",
                        "--database",
                        default="ugent",
                        help="ArangoDB database")
    parser.add_argument("-c",
                        "--collection",
                        default="import",
                        help="ArangoDB collection")
    parser.add_argument(
        "-url",
        "--connection-url",
        default="http://localhost:8529",
        help="ArangoDB connection URL",
    )
    parser.add_argument("-u",
                        "--username",
                        default="root",
                        help="ArangoDB username")
    parser.add_argument("-p",
                        "--password",
                        default="",
                        help="ArangoDB password")
    args = parser.parse_args()
    print(f"connection={args.connection_url}")
    print(f"username={args.username}")
    print(f"database={args.database}")
    print(f"collection={args.collection}")

    f = furl.furl(args.connection_url)
    client = ArangoClient(protocol=f.scheme, host=f.host, port=8529)

    db = client.db(args.database,
                   username=args.username,
                   password=args.password)

    query = """
        FOR doc in import
           FILTER doc._type in ['Document', 'Vacancy', 'News Item', 'Event', 'PhdDefense', 'LibraryDocument']
           return doc
    """

    print('Fetching data')
    result = db.aql.execute(query)
    print('Got data')
    for i, r in enumerate(result):
        if i % 1000 == 0:
            print(i)

        for name in ('text', 'toptext', 'bottomtext'):
            if name in r:
                text = r[name]
                soup = bs4.BeautifulSoup(text, 'html.parser')
                try:
                    str(soup)
                except Exception as e:
                    print(f"HTML error in {r['_path']}, field={name} ({e})")
Пример #26
0
    def __init__(self):
        self.__client = ArangoClient(protocol='http',
                                     host='localhost',
                                     port=8529)

        self.db = self.__client.db('proverbs_net',
                                   username='******',
                                   password='******')
Пример #27
0
def test_verify():
    assert arango_client.verify() is True
    with pytest.raises(ServerConnectionError):
        ArangoClient(
            username='******',
            password='******',
            verify=True
        )
Пример #28
0
def connect():
    client = ArangoClient(
        host=os.getenv('ARANGO_HOST'),
        port=os.getenv('ARANGO_PORT'),
        username=os.getenv('ARANGO_USER'),
        password=os.getenv('ARANGO_ROOT_PASSWORD'),
    )
    client.databases()
Пример #29
0
 def connect_to_db_v2(self, db_hostname):
     conn = ArangoClient(hosts=self.db_param[db_hostname]['url'])
     db = conn.db(
         name=self.db_param[db_hostname]['name'],
         username=self.db_param[db_hostname]['username'], 
         password=self.db_param[db_hostname]['password']
         )
     return db
Пример #30
0
    def init_db():
        arangodb_client = ArangoClient(protocol='http', host='172.17.0.1', port=8529)

        sys_db = arangodb_client.db('_system', username='******', password='******')
        try:
            sys_db.create_database('int20h_test')
        except:
            pass