def db(self):
conn = Connection(username=self.config.user,
password=self.config.password)
name = self.config.database
if not conn.hasDatabase(name):
return conn.createDatabase(name)
return conn[name]
def _get_db():
"""Return DB & collection
Returns:
(db, collection)
"""
conn = Connection(arangoURL='http://localhost:8529',
username='******',
password='******')
if not conn.hasDatabase(DB):
db = conn.createDatabase(DB)
else:
db = conn[DB]
if not db.hasCollection(COLLECTION):
collection = db.createCollection(name=COLLECTION)
else:
collection = db.collections[COLLECTION]
return db, collection
class ArangoDbPersister(Persister):
"""
A basic ArangoDB persister.
>>> from py2store.persisters._arangodb_in_progress import ArangoDbPersister
>>> s = ArangoDbPersister()
>>> k = {'key': '777'} # Each collection will happily accept user-defined _key values.
>>> v = {'val': 'bar'}
>>> for _key in s:
... del s[_key]
...
>>> k in s
False
>>> len(s)
0
>>> s[k] = v
>>> len(s)
1
>>> s[k]
{'val': 'bar'}
>>> s.get(k)
{'val': 'bar'}
>>> s.get({'not': 'a key'}, {'default': 'val'}) # testing s.get with default
{'default': 'val'}
>>> list(s.values())
[{'val': 'bar'}]
>>> k in s # testing __contains__ again
True
>>> del s[k]
>>> len(s)
0
>>> s = ArangoDbPersister(db_name='py2store', key_fields=('name',))
>>> for _key in s:
... del s[_key]
...
>>> s[{'name': 'guido'}] = {'yob': 1956, 'proj': 'python', 'bdfl': False}
>>> s[{'name': 'guido'}]
{'yob': 1956, 'proj': 'python', 'bdfl': False}
>>> s[{'name': 'vitalik'}] = {'yob': 1994, 'proj': 'ethereum', 'bdfl': True}
>>> s[{'name': 'vitalik'}]
{'yob': 1994, 'proj': 'ethereum', 'bdfl': True}
>>> for key, val in s.items():
... print(f"{key}: {val}")
{'name': 'guido'}: {'yob': 1956, 'proj': 'python', 'bdfl': False}
{'name': 'vitalik'}: {'yob': 1994, 'proj': 'ethereum', 'bdfl': True}
"""
# reserved by the database fields
_reserved = {"_key", "_id", "_rev"}
def __init__(
self,
user='******',
password='******',
url='http://127.0.0.1:8529',
db_name='py2store',
collection_name='test',
key_fields=('key', ) # _id, _key and _rev are reserved by db
):
self._connection = Connection(arangoURL=url,
username=user,
password=password)
self._db_name = db_name
self._collection_name = collection_name
# if db not created
if not self._connection.hasDatabase(self._db_name):
self._connection.createDatabase(self._db_name)
self._adb = self._connection[self._db_name]
# if collection not created
if not self._adb.hasCollection(self._collection_name):
self._collection = self._adb.createCollection(
name=self._collection_name)
self._collection = self._adb[self._collection_name]
if isinstance(key_fields, str):
key_fields = (key_fields, )
self._key_fields = key_fields
def __fetchitem__(self, k):
f = self._collection.fetchFirstExample(k)
if f is not None and len(f) == 1:
return f[0]
return None
def __getitem__(self, k):
f = self.__fetchitem__(k)
if f is not None:
d = f.getStore()
# exclude reserved keys and corresponded values
d = {
x: d[x]
for x in d
if x not in self._reserved and x not in self._key_fields
}
return d
else:
raise KeyError(f"No document found for query: {k}")
def __setitem__(self, k, v):
doc = self._collection.createDocument(dict(k, **v))
doc.save()
def __delitem__(self, k):
if len(k) > 0:
f = self.__fetchitem__(k)
if f is not None:
return f.delete()
raise KeyError(f"You can't removed that key: {k}")
def __iter__(self):
docs = self._collection.fetchAll()
yield from [{x: d[x]
for x in d.getStore() if x in self._key_fields}
for d in docs]
def __len__(self):
return self._collection.count()
class dbConnector:
"""Connects with ONLY ONE db and perform operation on it."""
def __init__(self,
db_name,
auth=False,
password='',
user='******',
arangoURL='http://127.0.0.1:8529'):
"""Init the database conection and set atributes db atribute."""
if auth:
self.conn = Connection(username=user,
password=password,
arangoURL=arangoURL)
else:
self.conn = Connection(username=user, arangoURL=arangoURL)
if self.conn.hasDatabase(db_name):
# conectartla
self.db = pyArango.database.Database(self.conn, db_name)
pass
else:
self.db = self.conn.createDatabase(name=db_name)
# Crearla
def retrieve_collection(self, coll_name): # doc, bd, coll?
# Posiblemente innecesario
"""Return the collection in a list form."""
self.db.reload()
coll = self.db.collections[coll_name]
document_list = []
for document in coll.fetchAll():
document_list.append(document._store)
return document_list
def save_document(self, doc, coll_name): # doc, bd, coll?
"""
Save the document in the database.
It is saved in the collection with the name specified.
Doc, is in a python dic form.
"""
self.db.reload()
if self.db.hasCollection(coll_name):
coll = self.db.collections[coll_name]
document = coll.createDocument()
document._store = doc
document.save()
else:
print('There is no collection with that name')
def retrieve_document(self, coll_name, doc_name): # doc, bd, coll?
"""Return the document in a python dic form."""
self.db.reload()
# FIXME: Modo cutre de encontrar documentos sin clave
# Usar AQL quizas
document_list = self.retrieve_collection(coll_name)
for doc in document_list:
if doc[doc_name]:
return doc
# doc._store
pass
def create_collection(self, coll_name): # doc, bd, coll?
"""Create and return the collection."""
self.db.reload()
if self.db.hasCollection(coll_name):
print('The database already has a collection with that name')
else:
self.db.createCollection(name=coll_name)
class ArangoDbPersister(Persister):
"""
A basic ArangoDB persister.
>>> from py2store.persisters.arangodb_w_pyarango import ArangoDbPersister
>>> s = ArangoDbPersister()
>>> k = {'key': '777'} # Each collection will happily accept user-defined _key values.
>>> v = {'val': 'bar'}
>>> for _key in s:
... del s[_key]
...
>>> k in s
False
>>> len(s)
0
>>> s[k] = v
>>> len(s)
1
>>> s[k]
{'val': 'bar'}
>>> s.get(k)
{'val': 'bar'}
>>> s.get({'not': 'a key'}, {'default': 'val'}) # testing s.get with default
{'default': 'val'}
>>> list(s.values())
[{'val': 'bar'}]
>>> k in s # testing __contains__ again
True
>>> del s[k]
>>> len(s)
0
>>> s = ArangoDbPersister(db_name='py2store', key_fields=('name',))
>>> for _key in s:
... del s[_key]
...
>>> s[{'name': 'guido'}] = {'yob': 1956, 'proj': 'python', 'bdfl': False}
>>> s[{'name': 'guido'}]
{'yob': 1956, 'proj': 'python', 'bdfl': False}
>>> s[{'name': 'vitalik'}] = {'yob': 1994, 'proj': 'ethereum', 'bdfl': True}
>>> s[{'name': 'vitalik'}]
{'yob': 1994, 'proj': 'ethereum', 'bdfl': True}
>>> for key, val in s.items():
... print(f"{key}: {val}")
{'name': 'guido'}: {'yob': 1956, 'proj': 'python', 'bdfl': False}
{'name': 'vitalik'}: {'yob': 1994, 'proj': 'ethereum', 'bdfl': True}
"""
# reserved by the database fields
_reserved = {"_key", "_id", "_rev"}
def __init__(
self,
user='******',
password='******',
url='http://127.0.0.1:8529',
db_name='py2store',
collection_name='test',
key_fields=('key', ), # _id, _key and _rev are reserved by db
key_fields_separator='::',
):
self._connection = Connection(
arangoURL=url,
username=user,
password=password,
)
self._db_name = db_name
self._collection_name = collection_name
# If DB not created:
if not self._connection.hasDatabase(self._db_name):
self._connection.createDatabase(self._db_name)
self._adb = self._connection[self._db_name]
# If collection not created:
if not self._adb.hasCollection(self._collection_name):
self._collection = self._adb.createCollection(
name=self._collection_name)
self._collection = self._adb[self._collection_name]
if isinstance(key_fields, str):
key_fields = (key_fields, )
self._key_fields = key_fields
self._key_fields_separator = key_fields_separator
def _make_key(self, keys_dict):
"""
Convert a dict of keys into a real key-string by joining dict values in a predefined order.
DB requirements for the key:
The key must be a string value.
Keys are case-sensitive.
Numeric keys are not allowed.
The key must be from 1 byte to 254 bytes long.
It must consist of:
- letters a-z (lower or upper case),
- digits 0-9
- any of the following characters: _ - : . @ ( ) + , = ; $ ! * ' %
Any other characters cannot be used inside key values.
"""
key_values = [keys_dict[key_label] for key_label in self._key_fields]
key_str = self._key_fields_separator.join(key_values)
return key_str
def _split_key(self, joined_key_str):
"""
Convert a key-string used by DB internally
into a user-friendly dict of key labels and values.
"""
key_values = joined_key_str.split(self._key_fields_separator)
keys_dict = dict(zip(self._key_fields, key_values))
return keys_dict
def __fetchitem__(self, keys_dict):
key = self._make_key(keys_dict)
try:
return self._collection[key]
except DocumentNotFoundError:
raise KeyError(f"No document found for query: {keys_dict}")
def __getitem__(self, keys_dict):
item = self.__fetchitem__(keys_dict)
doc = item.getStore()
# todo (Mike): maybe move this cleanup to a base Arango Store?
# exclude reserved keys and corresponded values
data = {
key: doc[key]
for key in doc
if key not in self._reserved and key not in self._key_fields
}
return data
def __setitem__(self, keys_dict, values_dict):
try:
doc = self.__fetchitem__(keys_dict)
except KeyError:
doc = self._collection.createDocument()
doc._key = self._make_key(keys_dict)
for k, v in values_dict.items():
doc[k] = v
doc.save()
def __delitem__(self, keys_dict):
doc = self.__fetchitem__(keys_dict)
doc.delete()
def __iter__(self):
docs = self._collection.fetchAll()
yield from ({key_name: doc[key_name]
for key_name in self._key_fields} for doc in docs)
def __len__(self):
return self._collection.count()
SCOPE = [
'https://spreadsheets.google.com/feeds',
'https://www.googleapis.com/auth/drive'
]
CREDENTIALS = ServiceAccountCredentials.from_json_keyfile_name(
f'{DIR_PATH}/credentials.json', SCOPE)
CLIENT = gspread.authorize(CREDENTIALS)
SHEET = CLIENT.open_by_key(CONFIG['google_sheet']['key'])
REVIEW_FIRST_DAY = date(2018, 5, 3)
CONN = Connection(username=CONFIG['db']['username'],
password=CONFIG['db']['password'])
if not CONN.hasDatabase('utopian'):
CONN.createDatabase('utopian')
DB = CONN['utopian']
POSTS_COLLECTION = 'posts'
def connect_collection(db, col_name):
if not db.hasCollection(name=col_name):
db.createCollection(name=col_name)
return db[col_name]
postCol = connect_collection(DB, POSTS_COLLECTION)
class ArangoDB: # pylint: disable = R0902
"""Handle creation of all required Documents."""
def __init__(self) -> None:
self.conn = Connection(arangoURL=config.db_host,
username=config.db_username,
password=config.db_password)
self.db = self._get_db(config.db_name)
self.groups: Chats = self._get_collection('Chats')
self.ab_bio_blacklist: AutobahnBioBlacklist = self._get_collection(
'AutobahnBioBlacklist')
self.ab_string_blacklist: AutobahnStringBlacklist = self._get_collection(
'AutobahnStringBlacklist')
self.ab_filename_blacklist: AutobahnFilenameBlacklist = self._get_collection(
'AutobahnFilenameBlacklist')
self.ab_channel_blacklist: AutobahnChannelBlacklist = self._get_collection(
'AutobahnChannelBlacklist')
self.ab_domain_blacklist: AutobahnDomainBlacklist = self._get_collection(
'AutobahnDomainBlacklist')
self.ab_collection_map = {
'0x0': self.ab_bio_blacklist,
'0x1': self.ab_string_blacklist,
'0x2': self.ab_filename_blacklist,
'0x3': self.ab_channel_blacklist,
'0x4': self.ab_domain_blacklist
}
self.banlist: BanList = self._get_collection('BanList')
def query(self,
query: str,
batch_size: int = 100,
raw_results: bool = False,
bind_vars: Dict = None,
options: Dict = None,
count: bool = False,
full_count: bool = False,
json_encoder: bool = None,
**kwargs: Any) -> AQLQuery: # pylint: disable = R0913
"""Wrapper around the pyArango AQLQuery to avoid having to do `db.db.AQLQuery`."""
bind_vars = bind_vars or {}
options = options or {}
return self.db.AQLQuery(query,
rawResults=raw_results,
batchSize=batch_size,
bindVars=bind_vars,
options=options,
count=count,
fullCount=full_count,
json_encoder=json_encoder,
**kwargs)
def _get_db(self, db: str) -> Database:
"""Return a database. Create it if it doesn't exist yet.
Args:
db: The name of the Database
Returns: The Database object
"""
if self.conn.hasDatabase(db):
return self.conn[db]
else:
return self.conn.createDatabase(db)
def _get_collection(self, collection: str) -> Collection:
"""Return a collection of create it if it doesn't exist yet.
Args:
collection: The name of the collection
Returns: The Collection object
"""
if self.db.hasCollection(collection):
return self.db[collection]
else:
return self.db.createCollection(collection)
