class Flats:
"""
Interface to operate with Flats data base
"""
def __init__(self):
""" connect to a db """
self.db = TinyDB(FLATS_DB_NAME)
def is_new(self, flat: Flat) -> bool:
old_record = self._get_flat(flat.id)
if old_record:
if self._is_flats_identical(old_record, flat):
return False
self._upsert_flat(flat)
return True
def _get_flat(self, flat_id: FlatId) -> Optional[Flat]:
""" db get query """
flat_q = Query()
result = self.db.get(flat_q.id == flat_id)
return Flat(**result) if result else None
def _is_flats_identical(self, old_record: Flat, flat: Flat) -> bool:
""" db compare instances """
return old_record == flat
def _upsert_flat(self, flat: Flat) -> None:
""" db upsert given flat """
flat_q = Query()
self.db.upsert(asdict(flat), flat_q.id == flat.id)
class Database:
db: TinyDB
guilds: Table
def __init__(self):
self.db = TinyDB('db.json')
self.guilds = self.db.table('guilds')
def get_guild(self, id):
guild = self.guilds.get(where('id') == str(id))
return guild and guild['data'] or None
def set_guild(self, id, data):
self.guilds.upsert({
'id': str(id),
'data': data
},
where('id') == str(id))
def get_hacks(self):
hacks = self.db.get(where('key') == 'hacks')
return hacks and hacks['value'] or []
def set_hacks(self, data):
self.db.upsert({
'key': 'hacks',
'value': data
},
where('key') == 'hacks')
class TinyDBPasteStorage(PasteStorage):
def __init__(self, db_file):
if not os.path.isfile(db_file):
with io.open(db_file, "w", encoding="utf-8") as file_handler:
file_handler.write("")
self.db = TinyDB(db_file)
def save_paste(self, paste_model: Paste):
self.db.insert({
"author": paste_model.author,
"title": paste_model.title,
"content": paste_model.content,
"date": str(paste_model.date),
})
def get_last_date(self):
db_query = Query()
query_result = self.db.search(db_query.type == "config")
if query_result:
return parser.parse(query_result[0].get("last_date"))
def save_last_date(self, date):
db_query = Query()
self.db.upsert({
"last_date": str(date),
"type": "config"
}, db_query.type == "config")
class GenericTinyRepo(Generic[EntryType]):
'''repository used to manage objects in a database'''
def __init__(self, dbfile: str, table: str, idx_name: str,
entry_type: Type[EntryType]):
'''create repository using a specific database file'''
self.table = TinyDB(dbfile).table(table)
self.entry_type = entry_type
self.idx_name = idx_name
def put(self, obj: EntryType):
'''put new or update an entry'''
self.table.upsert(obj._asdict(), where(self.idx_name) == obj.id)
def get(self, idx) -> EntryType:
'''get an entry by id'''
result = self.table.search(where(self.idx_name) == idx)
return self.entry_type(**result[0]) if result else None
def delete(self, idx):
'''remove an entry from the table'''
self.table.remove(where(self.idx_name) == idx)
def list(self) -> List[EntryType]:
'''return all entries, for debugging purposes usually'''
return [self.entry_type(**row) for row in self.table.all()]
def _query(self, key, value) -> list:
'''return music album directories'''
return [
self.entry_type(**row)
for row in self.table.search(where(key) == value)
]
def upsert_raw_entry(dataset, filename, frame_nr, body):
db = TinyDB("db/" + dataset + '-raw.json')
query = Query()
existing_entries = db.search(query.filename == filename)
if(len(existing_entries) > 1):
print('Found duplicate entry for file ' + filename)
db.upsert({'filename': filename, 'frame_nr': frame_nr, 'body': body.tolist()}, query.filename == filename)
def import_kb(source_dir):
index = client.get_index(KB_INDEX_UID)
db = TinyDB(storage=MemoryStorage)
KB = Query()
source_dir = Path(source_dir)
for filename in source_dir.glob("**/*.json"):
with open(filename, "r") as f:
data = json.load(f)
source_id = data["metadata"]["source_id"]
collection_normalized = [
normalize(v, KB_SCHEMA) for v in data["collection"]
]
for _item in collection_normalized:
item = db.get(KB.uid == _item["uid"])
item = item if item is not None else _item
if "sources" in item:
item["sources"] = list(set(item["sources"]) | set([source_id]))
else:
item.update({"sources": [source_id]})
db.upsert(item, KB.uid == item["uid"])
resp = index.add_documents(db.all())
logger.info(resp)
def parse(self, response):
db=TinyDB('DBFiles/UndefeatedDB.json')
Q = Query()
items = response.css('div.product-grid-item ')
if items:
for item in items:
dirtyimage = item.css('img::attr(src)').extract()
namecolor = item.css('a::text').extract()
if sepNameColor(namecolor[0]):
combo = sepNameColor(namecolor[0])
else:
combo = [namecolor[0],"N/A"]
name = combo[0]
colors = combo[1]
image = cleanLink(dirtyimage[0])
ahref = item.css('a::attr(href)').extract()[0]
price = item.css('span.money::text').extract()[0]
db.upsert({"Site":"Undefeated","catagory":self.urls[0][1], 'title':name,'price':price, 'colors':colors, 'image':image, 'href':'undefeated.com'+str(ahref)}, Q.href == 'undefeated.com'+str(ahref))
self.x += 1
yield scrapy.Request(url=self.urls[self.CurURL][0]+str(self.x), callback=self.parse)
else:
self.x = 1
self.CurURL += 1
if self.CurURL < len(self.urls):
self.site=self.urls[self.CurURL][0]
self.catagory = self.urls[self.CurURL][1]
yield scrapy.Request(url=self.urls[self.CurURL][0]+str(self.x), callback=self.parse)
def upsert_body_entry(dataset, filename, frame_nr, label, angles):
db = TinyDB("db/" + dataset + '.json')
query = Query()
existing_entries = db.search(query.filename == filename)
if(len(existing_entries) > 1):
print('Found duplicate entry for file ' + filename)
db.upsert({'filename': filename, 'frame_nr': frame_nr, 'label': label, 'angles': angles.tolist()}, query.filename == filename)
class Widget:
def __init__(self):
self.db = TinyDB('./app/db/data.json')
def get_widgets(self):
widgets = self.db.all()
return jsonify(widgets)
def get_widget(self, name):
widget = self.db.search(where('name') == name)
return jsonify(widget)
def create_widget(self, data):
self.db.insert(data)
return self.get_widget(data['name'])
def upsert_widget(self, name, data):
Widget = Query()
self.db.upsert(data, Widget.name == name)
return self.get_widget(data['name'])
def delete_widget(self, name):
self.db.remove(where('name') == name)
return 'Removed Widget with name %s' % name
def purge(self):
self.db.purge()
return 'Purged database'
def addUpdatesToDatabase(channel: TextChannel, server: str):
success = True
if (server == 'en'):
db = TinyDB(enEventUpdatesDB)
elif (server == 'jp'):
db = TinyDB(jpEventUpdatesDB)
try:
db.upsert(
{
'name': channel.name,
'guild': channel.guild.id,
'guildName': channel.guild.name,
'id': channel.id
},
where('id') == channel.id)
except Exception as e:
print(e)
success = False
if success:
text = "Channel " + channel.name + " will receive %s event updates." % server.upper(
)
else:
text = "Failed adding " + channel.name + " to the %s event updates list." % server.upper(
)
return text
def addChannelToNewsDatabase(channel: TextChannel, server: str):
success = True
if (server == 'en'):
db = TinyDB(bestdoriENNewsDB)
elif (server == 'jp'):
db = TinyDB(bestdoriJPNewsDB)
elif (server == 'cn'):
db = TinyDB(bestdoriCNNewsDB)
else:
db = TinyDB(bestdoriAllNewsDB)
try:
db.upsert(
{
'name': channel.name,
'guild': channel.guild.id,
'guildName': channel.guild.name,
'id': channel.id
},
where('id') == channel.id)
except Exception as e:
print(e)
success = False
if success:
text = "Channel " + channel.name + " will receive %s server patch updates from Bestdori." % server.upper(
)
else:
text = "Failed adding " + channel.name + " to the %s server patch updates list." % server.upper(
)
return text
def add_channel_to_cutoff_db(channel: TextChannel, interval: int):
success = True
if (interval == 1):
db = TinyDB(one_minute_cutoff_updates_db)
interval = '1 minute'
elif (interval == 60):
db = TinyDB(one_hour_cutoff_updates_db)
interval = '1 hour'
try:
db.upsert(
{
'name': channel.name,
'guild': channel.guild.id,
'guildName': channel.guild.name,
'id': channel.id
},
where('id') == channel.id)
except Exception as e:
print(e)
success = False
if success:
text = f"Channel {channel.name} will receive {interval} updates"
else:
text = f"Failed adding {channel.name} to the {interval} tracking list"
return text
def test_upsert_by_id(db: TinyDB):
assert len(db) == 3
# Single document existing
extant_doc = Document({'char': 'v'}, doc_id=1)
assert db.upsert(extant_doc) == [1]
doc = db.get(where('char') == 'v')
assert doc is not None
assert doc.doc_id == 1
assert len(db) == 3
# Single document missing
missing_doc = Document({'int': 5, 'char': 'w'}, doc_id=5)
assert db.upsert(missing_doc) == [5]
doc = db.get(where('char') == 'w')
assert doc is not None
assert doc.doc_id == 5
assert len(db) == 4
# Missing doc_id and condition
with pytest.raises(ValueError, match=r"(?=.*\bdoc_id\b)(?=.*\bquery\b)"):
db.upsert({'no_Document': 'no_query'})
# Make sure we didn't break anything
assert db.insert({'check': '_next_id'}) == 6
def addChannelToDatabase(channel: TextChannel, interval: int, server: str):
success = True
if server == 'en':
if(interval == 2):
db = TinyDB(eventCheckDb2min)
interval = '2 minute'
if(interval == 1):
db = TinyDB(eventCheckDb1min)
interval = '1 minute'
if(interval == 3600):
db = TinyDB(eventCheckDb1hr)
interval = '1 hour'
else:
if(interval == 2):
db = TinyDB(jp2MinuteTracking)
interval = '2 minute'
if(interval == 3600):
db = TinyDB(jp1HourTracking)
interval = '1 hour'
try:
db.upsert({'name': channel.name,
'guild': channel.guild.id,
'guildName': channel.guild.name,
'id': channel.id
}, where('id') == channel.id)
except Exception as e:
print(e)
success = False
if success:
text = "Channel " + channel.name + " will receive %s updates for the %s server" %(interval,server)
else:
text = "Failed adding " + channel.name + " to the %s %s tracking list" %(server, interval)
return text
class StateManager():
def __init__(self, state_db=None):
if state_db:
self.state_db = state_db
else:
self.state_db = TinyDB('state_db.json')
def get_user_state(self, user):
try:
UserState = Query()
result = self.state_db.search(UserState.slack_user == user)
if len(result) > 0:
return result[0].get("state", {})
return {}
except Exception as e:
print(f"ERROR getting state for user {user}")
print(str(e))
return {}
def save_user_state(self, user, state):
try:
UserState = Query()
self.state_db.upsert({
'slack_user': user,
'state': state
}, UserState.slack_user == user)
except Exception as e:
print("ERROR SAVING STATE for user {user} : {state} ")
print(str(e))
def get_key_value(self, key):
try:
UserState = Query()
result = self.state_db.search(UserState.key == key)
if len(result) > 0:
return result[0].get("value", {})
return {}
except Exception as e:
print("ERROR getting state for key {key}")
print(str(e))
return {}
def save_key_value(self, key, value):
try:
UserState = Query()
self.state_db.upsert({
'key': key,
'value': value
}, UserState.key == key)
except Exception as e:
print("ERROR SAVING STATE for key {key} : {value} ")
print(str(e))
def get_all_keys(self):
try:
return self.state_db.all()
except Exception as e:
print("ERROR SAVING STATE for key {key} : {value} ")
print(str(e))
def inventory_site_add(site):
db = TinyDB(tinydb_site_facts)
q = Query()
# set defaults
if "ise" not in site:
site["ise"] = {"present": False}
db.upsert(site, q.siteid == site["siteid"])
return True
def inventory_device_add(device):
db = TinyDB(tinydb_host_facts)
q = Query()
# set defaults
if "ise" not in device:
device["ise"] = {"present": False}
db.upsert(device, q.hostname == device["hostname"])
return True
class GNGraphDBConfigModel:
query = Query()
def __init__(self, db_path):
dbpath = os.path.join(db_path, 'gngraph_pgres_dbcreds.json')
self._db = TinyDB(dbpath)
def req_fields_json(self, dict_result):
req_items = [
'serverIP', 'serverPort', 'username', 'password', 'dbname'
]
return {
key: value
for key, value in dict_result.items() if key in req_items
}
def search_op(self, req_dict):
return True if self.search_res(req_dict['serverIP']) else False
def search_res(self):
return self._db.search(doc_id == 1)
def insert_op(self, req_dict):
if not self.search_op(req_dict):
self._db.insert(req_dict)
return self._db.all()
return "None_Insert"
def upsert_op(self, req_dict):
self._db.upsert(
req_dict,
GNGraphDBConfigModel.query.serverIP == req_dict['serverIP'])
def delete_op(self, req_dict):
if self.search_op(req_dict):
self._db.remove(where('serverIP') == req_dict['serverIP'])
return self._db.all()
return "None_Delete"
def get_op(self):
return self._db.get(doc_id=1)
def update_op(self, old_srv_IP, req_dict):
if not self.search_res(old_srv_IP):
return False
self._db.update(
{
'serverIP': req_dict['serverIP'],
'serverPort': req_dict['serverPort'],
'username': req_dict['username'],
'password': req_dict['password']
}, GNGraphDBConfigModel.query.serverIP == old_srv_IP)
return self._db.all()
def stop_db(self):
self._db.close()
class TinydbREST(object):
class Request(object):
def __init__(self, status_code):
self.status_code = status_code
def __init__(self, path, table, pk=None):
self.table = table
self.db = TinyDB(path, storage=_tdbserialization).table(table)
self.pk = pk
def _getpk(self):
r = self.db.get(Query().name==self.pk)
if r is None:
# todo: return 404 not found error. on GET and _('pipe').post()
raise IOError('{} Not found'.format(self.pk))
elif 'data' in r.keys():
return r['data']
else:
return r
def get(self):
if self.pk is not None:
return self.Request(200), self._getpk()
else:
return self.Request(200), self.db.all()
def post(self, request_body):
if self.pk is None:
name = request_body['name']
if name in ['pipes','remotes']:
raise APIError('{} is reserved, use another name'.format(name))
if self.db.contains(Query().name == name):
raise APIError('entry {} already exists, needs to be unique'.format(name))
self.db.insert(request_body)
self.pk = name
return self.Request(201), self._getpk()
else:
name = self.pk
self.db.upsert({'name': name, 'data': request_body}, Query().name == self.pk)
return self.Request(200), self._getpk()
def put(self, request_body):
if self.pk is not None:
self.db.upsert(request_body, Query().name == self.pk)
return self.Request(200), self._getpk()
def patch(self, request_body):
if self.pk is not None:
dict_ = self._getpk()
dict_.update(request_body)
return self.put(dict_)
def delete(self):
if self.pk is not None:
self.db.remove(Query().name == self.pk)
return self.Request(204),not self.db.contains(Query().name == self.pk)
class RobCog(commands.Cog, Server):
def __init__(self, client):
self.client = client
self.events = TinyDB('database/events.json')
self.commands = TinyDB('database/commands.json')
Server.__init__(self)
async def isRob(ctx):
if (ctx.message.author.id == ctx.message.guild.owner_id
): #Server Owner
await ctx.message.delete() #Delete command usage
return True
return False
@commands.command()
@commands.check(isRob)
async def addcom(self, ctx, use, *, args):
self.commands.upsert({'use': use, 'resp': args}, where('use') == use)
await ctx.message.channel.send("Added command `" + use +
"`\n**Response:** " + args)
@commands.command()
@commands.check(isRob)
async def delcom(self, ctx, use):
removed = self.commands.remove(where('use') == use)
if (len(removed) > 0):
await ctx.message.channel.send("Deleted command `" + use + "`")
else:
await ctx.message.channel.send("No command found `" + use + "`")
@commands.command()
@commands.check(isRob)
async def raffle(self, ctx):
message = ctx.message
#Get active users
self.activeUsers = self.events.get(
where('name') == 'coolguy')['activeUsers']
#Raffle
found = False
while (not found):
selection = choice(self.activeUsers)
if message.guild.get_member(int(selection)) != None:
found = True
winner = message.guild.get_member(int(selection))
await winner.add_roles(message.guild.get_role(self.coolGuyRole))
general = message.guild.get_channel(self.generalChannel)
await general.send("<@" + selection + "> won the cool guy raffle! ")
def test_upsert(db: TinyDB):
assert len(db) == 3
# Document existing
db.upsert({'int': 5}, where('char') == 'a')
assert db.count(where('int') == 5) == 1
# Document missing
assert db.upsert({'int': 9, 'char': 'x'}, where('char') == 'x') == [4]
assert db.count(where('int') == 9) == 1
def share_my_addresses(fname):
db = TinyDB(fname)
Addresses = Query()
addresses = getaddressesbyaccount("")
count = len(addresses)
print("Adding " + str(count) + " addresses.")
count = 0
for address in addresses:
if count < 10:
db.upsert({'address': address}, Addresses.address == address)
count = count + 1
class MongoFileStorage(AbstractStorage):
def __init__(self):
self.storage = TinyDB('db.json')
def upsert(self, key, value):
self.storage.upsert({"key": key, "value": value}, Query().key == key)
def get(self, key):
stored_object = self.storage.get(Query().key == key)
if stored_object is None:
return None
return stored_object["value"]
def DatabaseCleanup(self, response):
db = TinyDB('DBFiles/KithDB.json')
dbQ = TinyDB('DBFiles/DatabaseQueue.json')
selected = dbQ.get(doc_id=self.storage)
items = response.xpath('descendant-or-self::span').extract()
Q = Query()
for item in items:
if 'data-product-price' in item:
selected['price'] = cleanhtml(item)
db.upsert(selected, Q.href == selected['href'])
#self.log("adding "+str(selected)+" into main db")
self.storage += 1
if self.storage <= len(dbQ) and dbQ.get(doc_id=self.storage) != None and dbQ.get(doc_id=self.storage)['href'] != None:
yield scrapy.Request(url=str("https://"+dbQ.get(doc_id=self.storage)['href']), callback=self.DatabaseCleanup)
def update(self, citizen):
db = TinyDB(self.db_filename)
query = Query()
if citizen.name != None:
if citizen.score > 100:
citizen.score = 100
if citizen.score < 0:
citizen.score = 0
# Updated if exists, inserted if not
db.upsert(citizen.toDict(), query.name == citizen.name)
db.close()
return True
db.close()
return False
def add_prefix_to_database(guild: Guild, prefix: str):
success = True
try:
db = TinyDB(prefix_db)
db.upsert({'id': guild.id, 'prefix': prefix}, where('id') == guild.id)
except Exception as e:
print(e)
success = False
if success:
text = "Prefix " + prefix + " set for server " + str(guild.name)
else:
text = "Failed adding " + prefix + " to the prefix list"
return text
def AddReactToDatabase(msgID: int, data: dict):
success = True
db = TinyDB('data/databases/tinydb/reactbasedroles.json')
try:
db.upsert({'msgID': msgID, 'reactList': data}, where('msgID') == msgID)
except Exception as e:
print(e)
success = False
if success:
text = f"Updated values for valid roles and emoji for message {msgID}."
else:
text = f"Failed to update/enable message {msgID} for reaction based role assignment."
return text
class TinyDBCookbook(object):
def __init__(self, db_path):
self._db = TinyDB(db_path)
def upsert(self, recipe):
query = Query()
self._db.upsert(recipe.to_dict(), query.name == recipe.name)
def retrieve(self, recipe_name):
query = Query()
return self._db.get(query.name == recipe_name)
@contextmanager
def open(self):
yield self
self._db.close()
def update_db(new_results: List[Dict], db: TinyDB, ids: List[Dict],
article: Query) -> None:
for result in new_results:
# if sub present, update it, else insert it
db.upsert(result, article.sub_id == result["sub_id"])
# copy all db
all_db = db.all()
logger.warning('DB original length: %s', len(all_db))
# remove old db entry not present in the last search
filtered_db = [x for x in all_db if x in ids]
db.purge()
for db_id in filtered_db:
# reinitialize db
db.insert(db_id)
logger.warning('DB final length: %s', len(db.all()))
class ArticleStore(object):
def __init__(self, json_path: Path):
self.db = TinyDB(json_path)
def get_article_id(self, file_path: Path):
Article = Query()
results = self.db.search(Article.folder == section_name, Article.filename == article_filename)
if not len(results):
return None
article_id = results[0]['zen_id']
return article_id
def save_article_id(self, section_name, article_filename, article_id):
Article = Query()
self.db.upsert({'section': section_name, 'filename': article_filename, 'zen_id': article_id},
Article.section == section_name, Article.filename == article_filename)
