class Flats: """ Interface to operate with Flats data base """ def __init__(self): """ connect to a db """ self.db = TinyDB(FLATS_DB_NAME) def is_new(self, flat: Flat) -> bool: old_record = self._get_flat(flat.id) if old_record: if self._is_flats_identical(old_record, flat): return False self._upsert_flat(flat) return True def _get_flat(self, flat_id: FlatId) -> Optional[Flat]: """ db get query """ flat_q = Query() result = self.db.get(flat_q.id == flat_id) return Flat(**result) if result else None def _is_flats_identical(self, old_record: Flat, flat: Flat) -> bool: """ db compare instances """ return old_record == flat def _upsert_flat(self, flat: Flat) -> None: """ db upsert given flat """ flat_q = Query() self.db.upsert(asdict(flat), flat_q.id == flat.id)
class Database: db: TinyDB guilds: Table def __init__(self): self.db = TinyDB('db.json') self.guilds = self.db.table('guilds') def get_guild(self, id): guild = self.guilds.get(where('id') == str(id)) return guild and guild['data'] or None def set_guild(self, id, data): self.guilds.upsert({ 'id': str(id), 'data': data }, where('id') == str(id)) def get_hacks(self): hacks = self.db.get(where('key') == 'hacks') return hacks and hacks['value'] or [] def set_hacks(self, data): self.db.upsert({ 'key': 'hacks', 'value': data }, where('key') == 'hacks')
class TinyDBPasteStorage(PasteStorage): def __init__(self, db_file): if not os.path.isfile(db_file): with io.open(db_file, "w", encoding="utf-8") as file_handler: file_handler.write("") self.db = TinyDB(db_file) def save_paste(self, paste_model: Paste): self.db.insert({ "author": paste_model.author, "title": paste_model.title, "content": paste_model.content, "date": str(paste_model.date), }) def get_last_date(self): db_query = Query() query_result = self.db.search(db_query.type == "config") if query_result: return parser.parse(query_result[0].get("last_date")) def save_last_date(self, date): db_query = Query() self.db.upsert({ "last_date": str(date), "type": "config" }, db_query.type == "config")
class GenericTinyRepo(Generic[EntryType]): '''repository used to manage objects in a database''' def __init__(self, dbfile: str, table: str, idx_name: str, entry_type: Type[EntryType]): '''create repository using a specific database file''' self.table = TinyDB(dbfile).table(table) self.entry_type = entry_type self.idx_name = idx_name def put(self, obj: EntryType): '''put new or update an entry''' self.table.upsert(obj._asdict(), where(self.idx_name) == obj.id) def get(self, idx) -> EntryType: '''get an entry by id''' result = self.table.search(where(self.idx_name) == idx) return self.entry_type(**result[0]) if result else None def delete(self, idx): '''remove an entry from the table''' self.table.remove(where(self.idx_name) == idx) def list(self) -> List[EntryType]: '''return all entries, for debugging purposes usually''' return [self.entry_type(**row) for row in self.table.all()] def _query(self, key, value) -> list: '''return music album directories''' return [ self.entry_type(**row) for row in self.table.search(where(key) == value) ]
def upsert_raw_entry(dataset, filename, frame_nr, body): db = TinyDB("db/" + dataset + '-raw.json') query = Query() existing_entries = db.search(query.filename == filename) if(len(existing_entries) > 1): print('Found duplicate entry for file ' + filename) db.upsert({'filename': filename, 'frame_nr': frame_nr, 'body': body.tolist()}, query.filename == filename)
def import_kb(source_dir): index = client.get_index(KB_INDEX_UID) db = TinyDB(storage=MemoryStorage) KB = Query() source_dir = Path(source_dir) for filename in source_dir.glob("**/*.json"): with open(filename, "r") as f: data = json.load(f) source_id = data["metadata"]["source_id"] collection_normalized = [ normalize(v, KB_SCHEMA) for v in data["collection"] ] for _item in collection_normalized: item = db.get(KB.uid == _item["uid"]) item = item if item is not None else _item if "sources" in item: item["sources"] = list(set(item["sources"]) | set([source_id])) else: item.update({"sources": [source_id]}) db.upsert(item, KB.uid == item["uid"]) resp = index.add_documents(db.all()) logger.info(resp)
def parse(self, response): db=TinyDB('DBFiles/UndefeatedDB.json') Q = Query() items = response.css('div.product-grid-item ') if items: for item in items: dirtyimage = item.css('img::attr(src)').extract() namecolor = item.css('a::text').extract() if sepNameColor(namecolor[0]): combo = sepNameColor(namecolor[0]) else: combo = [namecolor[0],"N/A"] name = combo[0] colors = combo[1] image = cleanLink(dirtyimage[0]) ahref = item.css('a::attr(href)').extract()[0] price = item.css('span.money::text').extract()[0] db.upsert({"Site":"Undefeated","catagory":self.urls[0][1], 'title':name,'price':price, 'colors':colors, 'image':image, 'href':'undefeated.com'+str(ahref)}, Q.href == 'undefeated.com'+str(ahref)) self.x += 1 yield scrapy.Request(url=self.urls[self.CurURL][0]+str(self.x), callback=self.parse) else: self.x = 1 self.CurURL += 1 if self.CurURL < len(self.urls): self.site=self.urls[self.CurURL][0] self.catagory = self.urls[self.CurURL][1] yield scrapy.Request(url=self.urls[self.CurURL][0]+str(self.x), callback=self.parse)
def upsert_body_entry(dataset, filename, frame_nr, label, angles): db = TinyDB("db/" + dataset + '.json') query = Query() existing_entries = db.search(query.filename == filename) if(len(existing_entries) > 1): print('Found duplicate entry for file ' + filename) db.upsert({'filename': filename, 'frame_nr': frame_nr, 'label': label, 'angles': angles.tolist()}, query.filename == filename)
class Widget: def __init__(self): self.db = TinyDB('./app/db/data.json') def get_widgets(self): widgets = self.db.all() return jsonify(widgets) def get_widget(self, name): widget = self.db.search(where('name') == name) return jsonify(widget) def create_widget(self, data): self.db.insert(data) return self.get_widget(data['name']) def upsert_widget(self, name, data): Widget = Query() self.db.upsert(data, Widget.name == name) return self.get_widget(data['name']) def delete_widget(self, name): self.db.remove(where('name') == name) return 'Removed Widget with name %s' % name def purge(self): self.db.purge() return 'Purged database'
def addUpdatesToDatabase(channel: TextChannel, server: str): success = True if (server == 'en'): db = TinyDB(enEventUpdatesDB) elif (server == 'jp'): db = TinyDB(jpEventUpdatesDB) try: db.upsert( { 'name': channel.name, 'guild': channel.guild.id, 'guildName': channel.guild.name, 'id': channel.id }, where('id') == channel.id) except Exception as e: print(e) success = False if success: text = "Channel " + channel.name + " will receive %s event updates." % server.upper( ) else: text = "Failed adding " + channel.name + " to the %s event updates list." % server.upper( ) return text
def addChannelToNewsDatabase(channel: TextChannel, server: str): success = True if (server == 'en'): db = TinyDB(bestdoriENNewsDB) elif (server == 'jp'): db = TinyDB(bestdoriJPNewsDB) elif (server == 'cn'): db = TinyDB(bestdoriCNNewsDB) else: db = TinyDB(bestdoriAllNewsDB) try: db.upsert( { 'name': channel.name, 'guild': channel.guild.id, 'guildName': channel.guild.name, 'id': channel.id }, where('id') == channel.id) except Exception as e: print(e) success = False if success: text = "Channel " + channel.name + " will receive %s server patch updates from Bestdori." % server.upper( ) else: text = "Failed adding " + channel.name + " to the %s server patch updates list." % server.upper( ) return text
def add_channel_to_cutoff_db(channel: TextChannel, interval: int): success = True if (interval == 1): db = TinyDB(one_minute_cutoff_updates_db) interval = '1 minute' elif (interval == 60): db = TinyDB(one_hour_cutoff_updates_db) interval = '1 hour' try: db.upsert( { 'name': channel.name, 'guild': channel.guild.id, 'guildName': channel.guild.name, 'id': channel.id }, where('id') == channel.id) except Exception as e: print(e) success = False if success: text = f"Channel {channel.name} will receive {interval} updates" else: text = f"Failed adding {channel.name} to the {interval} tracking list" return text
def test_upsert_by_id(db: TinyDB): assert len(db) == 3 # Single document existing extant_doc = Document({'char': 'v'}, doc_id=1) assert db.upsert(extant_doc) == [1] doc = db.get(where('char') == 'v') assert doc is not None assert doc.doc_id == 1 assert len(db) == 3 # Single document missing missing_doc = Document({'int': 5, 'char': 'w'}, doc_id=5) assert db.upsert(missing_doc) == [5] doc = db.get(where('char') == 'w') assert doc is not None assert doc.doc_id == 5 assert len(db) == 4 # Missing doc_id and condition with pytest.raises(ValueError, match=r"(?=.*\bdoc_id\b)(?=.*\bquery\b)"): db.upsert({'no_Document': 'no_query'}) # Make sure we didn't break anything assert db.insert({'check': '_next_id'}) == 6
def addChannelToDatabase(channel: TextChannel, interval: int, server: str): success = True if server == 'en': if(interval == 2): db = TinyDB(eventCheckDb2min) interval = '2 minute' if(interval == 1): db = TinyDB(eventCheckDb1min) interval = '1 minute' if(interval == 3600): db = TinyDB(eventCheckDb1hr) interval = '1 hour' else: if(interval == 2): db = TinyDB(jp2MinuteTracking) interval = '2 minute' if(interval == 3600): db = TinyDB(jp1HourTracking) interval = '1 hour' try: db.upsert({'name': channel.name, 'guild': channel.guild.id, 'guildName': channel.guild.name, 'id': channel.id }, where('id') == channel.id) except Exception as e: print(e) success = False if success: text = "Channel " + channel.name + " will receive %s updates for the %s server" %(interval,server) else: text = "Failed adding " + channel.name + " to the %s %s tracking list" %(server, interval) return text
class StateManager(): def __init__(self, state_db=None): if state_db: self.state_db = state_db else: self.state_db = TinyDB('state_db.json') def get_user_state(self, user): try: UserState = Query() result = self.state_db.search(UserState.slack_user == user) if len(result) > 0: return result[0].get("state", {}) return {} except Exception as e: print(f"ERROR getting state for user {user}") print(str(e)) return {} def save_user_state(self, user, state): try: UserState = Query() self.state_db.upsert({ 'slack_user': user, 'state': state }, UserState.slack_user == user) except Exception as e: print("ERROR SAVING STATE for user {user} : {state} ") print(str(e)) def get_key_value(self, key): try: UserState = Query() result = self.state_db.search(UserState.key == key) if len(result) > 0: return result[0].get("value", {}) return {} except Exception as e: print("ERROR getting state for key {key}") print(str(e)) return {} def save_key_value(self, key, value): try: UserState = Query() self.state_db.upsert({ 'key': key, 'value': value }, UserState.key == key) except Exception as e: print("ERROR SAVING STATE for key {key} : {value} ") print(str(e)) def get_all_keys(self): try: return self.state_db.all() except Exception as e: print("ERROR SAVING STATE for key {key} : {value} ") print(str(e))
def inventory_site_add(site): db = TinyDB(tinydb_site_facts) q = Query() # set defaults if "ise" not in site: site["ise"] = {"present": False} db.upsert(site, q.siteid == site["siteid"]) return True
def inventory_device_add(device): db = TinyDB(tinydb_host_facts) q = Query() # set defaults if "ise" not in device: device["ise"] = {"present": False} db.upsert(device, q.hostname == device["hostname"]) return True
class GNGraphDBConfigModel: query = Query() def __init__(self, db_path): dbpath = os.path.join(db_path, 'gngraph_pgres_dbcreds.json') self._db = TinyDB(dbpath) def req_fields_json(self, dict_result): req_items = [ 'serverIP', 'serverPort', 'username', 'password', 'dbname' ] return { key: value for key, value in dict_result.items() if key in req_items } def search_op(self, req_dict): return True if self.search_res(req_dict['serverIP']) else False def search_res(self): return self._db.search(doc_id == 1) def insert_op(self, req_dict): if not self.search_op(req_dict): self._db.insert(req_dict) return self._db.all() return "None_Insert" def upsert_op(self, req_dict): self._db.upsert( req_dict, GNGraphDBConfigModel.query.serverIP == req_dict['serverIP']) def delete_op(self, req_dict): if self.search_op(req_dict): self._db.remove(where('serverIP') == req_dict['serverIP']) return self._db.all() return "None_Delete" def get_op(self): return self._db.get(doc_id=1) def update_op(self, old_srv_IP, req_dict): if not self.search_res(old_srv_IP): return False self._db.update( { 'serverIP': req_dict['serverIP'], 'serverPort': req_dict['serverPort'], 'username': req_dict['username'], 'password': req_dict['password'] }, GNGraphDBConfigModel.query.serverIP == old_srv_IP) return self._db.all() def stop_db(self): self._db.close()
class TinydbREST(object): class Request(object): def __init__(self, status_code): self.status_code = status_code def __init__(self, path, table, pk=None): self.table = table self.db = TinyDB(path, storage=_tdbserialization).table(table) self.pk = pk def _getpk(self): r = self.db.get(Query().name==self.pk) if r is None: # todo: return 404 not found error. on GET and _('pipe').post() raise IOError('{} Not found'.format(self.pk)) elif 'data' in r.keys(): return r['data'] else: return r def get(self): if self.pk is not None: return self.Request(200), self._getpk() else: return self.Request(200), self.db.all() def post(self, request_body): if self.pk is None: name = request_body['name'] if name in ['pipes','remotes']: raise APIError('{} is reserved, use another name'.format(name)) if self.db.contains(Query().name == name): raise APIError('entry {} already exists, needs to be unique'.format(name)) self.db.insert(request_body) self.pk = name return self.Request(201), self._getpk() else: name = self.pk self.db.upsert({'name': name, 'data': request_body}, Query().name == self.pk) return self.Request(200), self._getpk() def put(self, request_body): if self.pk is not None: self.db.upsert(request_body, Query().name == self.pk) return self.Request(200), self._getpk() def patch(self, request_body): if self.pk is not None: dict_ = self._getpk() dict_.update(request_body) return self.put(dict_) def delete(self): if self.pk is not None: self.db.remove(Query().name == self.pk) return self.Request(204),not self.db.contains(Query().name == self.pk)
class RobCog(commands.Cog, Server): def __init__(self, client): self.client = client self.events = TinyDB('database/events.json') self.commands = TinyDB('database/commands.json') Server.__init__(self) async def isRob(ctx): if (ctx.message.author.id == ctx.message.guild.owner_id ): #Server Owner await ctx.message.delete() #Delete command usage return True return False @commands.command() @commands.check(isRob) async def addcom(self, ctx, use, *, args): self.commands.upsert({'use': use, 'resp': args}, where('use') == use) await ctx.message.channel.send("Added command `" + use + "`\n**Response:** " + args) @commands.command() @commands.check(isRob) async def delcom(self, ctx, use): removed = self.commands.remove(where('use') == use) if (len(removed) > 0): await ctx.message.channel.send("Deleted command `" + use + "`") else: await ctx.message.channel.send("No command found `" + use + "`") @commands.command() @commands.check(isRob) async def raffle(self, ctx): message = ctx.message #Get active users self.activeUsers = self.events.get( where('name') == 'coolguy')['activeUsers'] #Raffle found = False while (not found): selection = choice(self.activeUsers) if message.guild.get_member(int(selection)) != None: found = True winner = message.guild.get_member(int(selection)) await winner.add_roles(message.guild.get_role(self.coolGuyRole)) general = message.guild.get_channel(self.generalChannel) await general.send("<@" + selection + "> won the cool guy raffle! ")
def test_upsert(db: TinyDB): assert len(db) == 3 # Document existing db.upsert({'int': 5}, where('char') == 'a') assert db.count(where('int') == 5) == 1 # Document missing assert db.upsert({'int': 9, 'char': 'x'}, where('char') == 'x') == [4] assert db.count(where('int') == 9) == 1
def share_my_addresses(fname): db = TinyDB(fname) Addresses = Query() addresses = getaddressesbyaccount("") count = len(addresses) print("Adding " + str(count) + " addresses.") count = 0 for address in addresses: if count < 10: db.upsert({'address': address}, Addresses.address == address) count = count + 1
class MongoFileStorage(AbstractStorage): def __init__(self): self.storage = TinyDB('db.json') def upsert(self, key, value): self.storage.upsert({"key": key, "value": value}, Query().key == key) def get(self, key): stored_object = self.storage.get(Query().key == key) if stored_object is None: return None return stored_object["value"]
def DatabaseCleanup(self, response): db = TinyDB('DBFiles/KithDB.json') dbQ = TinyDB('DBFiles/DatabaseQueue.json') selected = dbQ.get(doc_id=self.storage) items = response.xpath('descendant-or-self::span').extract() Q = Query() for item in items: if 'data-product-price' in item: selected['price'] = cleanhtml(item) db.upsert(selected, Q.href == selected['href']) #self.log("adding "+str(selected)+" into main db") self.storage += 1 if self.storage <= len(dbQ) and dbQ.get(doc_id=self.storage) != None and dbQ.get(doc_id=self.storage)['href'] != None: yield scrapy.Request(url=str("https://"+dbQ.get(doc_id=self.storage)['href']), callback=self.DatabaseCleanup)
def update(self, citizen): db = TinyDB(self.db_filename) query = Query() if citizen.name != None: if citizen.score > 100: citizen.score = 100 if citizen.score < 0: citizen.score = 0 # Updated if exists, inserted if not db.upsert(citizen.toDict(), query.name == citizen.name) db.close() return True db.close() return False
def add_prefix_to_database(guild: Guild, prefix: str): success = True try: db = TinyDB(prefix_db) db.upsert({'id': guild.id, 'prefix': prefix}, where('id') == guild.id) except Exception as e: print(e) success = False if success: text = "Prefix " + prefix + " set for server " + str(guild.name) else: text = "Failed adding " + prefix + " to the prefix list" return text
def AddReactToDatabase(msgID: int, data: dict): success = True db = TinyDB('data/databases/tinydb/reactbasedroles.json') try: db.upsert({'msgID': msgID, 'reactList': data}, where('msgID') == msgID) except Exception as e: print(e) success = False if success: text = f"Updated values for valid roles and emoji for message {msgID}." else: text = f"Failed to update/enable message {msgID} for reaction based role assignment." return text
class TinyDBCookbook(object): def __init__(self, db_path): self._db = TinyDB(db_path) def upsert(self, recipe): query = Query() self._db.upsert(recipe.to_dict(), query.name == recipe.name) def retrieve(self, recipe_name): query = Query() return self._db.get(query.name == recipe_name) @contextmanager def open(self): yield self self._db.close()
def update_db(new_results: List[Dict], db: TinyDB, ids: List[Dict], article: Query) -> None: for result in new_results: # if sub present, update it, else insert it db.upsert(result, article.sub_id == result["sub_id"]) # copy all db all_db = db.all() logger.warning('DB original length: %s', len(all_db)) # remove old db entry not present in the last search filtered_db = [x for x in all_db if x in ids] db.purge() for db_id in filtered_db: # reinitialize db db.insert(db_id) logger.warning('DB final length: %s', len(db.all()))
class ArticleStore(object): def __init__(self, json_path: Path): self.db = TinyDB(json_path) def get_article_id(self, file_path: Path): Article = Query() results = self.db.search(Article.folder == section_name, Article.filename == article_filename) if not len(results): return None article_id = results[0]['zen_id'] return article_id def save_article_id(self, section_name, article_filename, article_id): Article = Query() self.db.upsert({'section': section_name, 'filename': article_filename, 'zen_id': article_id}, Article.section == section_name, Article.filename == article_filename)