class DBManager(): def __init__(self, store): self._db = SqliteDict(store, autocommit=True) self.commands_cache = {} def add(self, command): if self.get_command(command.get_name()) != None: return False self._db[command.get_name()] = command return True def update(self, command): self.add(command) def remove(self, command): del self._db[command.get_name()] def empty(self): self.commands_cache = {} self._db.clear() def count(self): return len(self._db) def get_command(self, name): if name not in self.commands_cache: try: self.commands_cache[name] = self._db[name] except KeyError: return None return self.commands_cache[name] def get_all_commands(self): list = [] for key, val in self._db.iteritems(): list.append(key) return list
class PrefOrderedSet: """ store/retrieve an ordered set of strings (like a list, but no duplicates) to/from a sqlite database """ def __init__(self, name: str, author: str, table: str): # DB stores values directly (not encoded as a pickle) self.sqlite_dict = SqliteDict(get_sqlite_path(name, author), table, encode=lambda x: x, decode=lambda x: x) def set(self, strings: list): """ set the list of strings :param strings: list of strings """ self.sqlite_dict.clear() # delete entire table # ordering is done by making the value in the key/value pair the index and our desired list "value" is the key for index, string in enumerate(strings): self.sqlite_dict[string] = index self.sqlite_dict.commit( ) # not using autocommit since we're updating (setting) multiple values in the above for loop def get(self) -> List[str]: """ returns the list of strings :return: list of strings """ return list(sorted(self.sqlite_dict, key=self.sqlite_dict.get))
class DBManager(): def __init__(self): self._db = SqliteDict('./matches.sqlite', autocommit=True) self.matches_cache = {} def add(self, match): self._db[match.get_code()] = match def update(self, match): self.add(match) def remove(self, match): del self._db[match.get_code()] def empty(self): self._db.clear() def count(self): return len(self._db) def get_match(self, code): if code not in self.matches_cache: try: self.matches_cache[code] = self._db[code] except KeyError: return None return self.matches_cache[code] def get_all_matches(self): list = [] for key, val in self._db.iteritems(): list.append(key) return list
class CDataBase(object): def __init__(self): try: self.close() except: pass self.mydict = SqliteDict('./DB/my_db.sqlite', autocommit=True) self.show() def set(self, key, value): self.mydict[key] = value def get(self, key): if key in self.mydict.keys(): ret = self.mydict[key] else: ret = None return ret def show(self, start_with=''): for key, value in self.mydict.items(): if key.find(start_with): print(key, '\t', value, '\n') def clear(self): self.mydict.clear() def close(self): self.mydict.close()
class ImageCache: def __init__(self, directory): db_dirname = os.path.join(directory, ".mikula") if not os.path.isdir(db_dirname): os.mkdir(db_dirname) db_filename = os.path.join(db_dirname, "images.cache") self.cache = SqliteDict(db_filename) self.recent_lookup_ = None def reset(self): self.cache.clear() self.recent_lookup_ = None def config_changed(self, config): if "config" not in self.cache.keys(): return True stored = self.cache["config"] return stored != config def update_config(self, config): self.cache["config"] = config self.cache.commit() def require_update(self, filename): if filename not in self.cache.keys(): self.recent_lookup_ = None return True timestamp, scaled, scaled_time, thumbnail, thumbnail_time = self.cache[ filename] if os.path.exists(scaled) and os.path.exists(thumbnail): if os.path.getmtime(filename) == timestamp and \ os.path.getmtime(scaled) == scaled_time and \ os.path.getmtime(thumbnail) == thumbnail_time: self.recent_lookup_ = (scaled, thumbnail) return False self.recent_lookup_ = None return True def get_filenames(self): return self.recent_lookup_ def update(self, filename, scaled, thumbnail): timestamp = os.path.getmtime(filename) scaled_time = os.path.getmtime(scaled) thumbnail_time = os.path.getmtime(thumbnail) self.cache[filename] = (timestamp, scaled, scaled_time, thumbnail, thumbnail_time) self.cache.commit()
def open_shelf(file_name: str, new: bool = False, raise_errors: bool = True, autocommit: bool = True) -> SqliteDict: try: sld = SqliteDict(file_name, autocommit=autocommit) if new: print('Deleting old shelf file and creating a new one.') sld.clear() except Exception as e: print(e) if raise_errors: raise e print( 'Error during shelf open action. Since raise_errors is false, starting from a new file.' ) sld = open_shelf(file_name, new=True, raise_errors=True) return sld
def reset(texts, index_dic=True, tfidf=True, hdp=False, lda=True, sim=False): total_start = timeit.default_timer() make_index_time = 0 make_dict_time = 0 make_lda_time = 0 make_tfidf_time = 0 sim_time = 0 hdptopicnum = 0 if index_dic: f = [i.split(',') for i in texts.readlines()] logging.info('Create id & ac_id list') ids = [f[i][1] for i in range(len(f))] ac_ids = [f[i][0] for i in range(len(f))] logging.info('Create contents list') contents = [] for i in range(len(f)): if len(f[i]) == 3: contents.append(f[i][2].strip().split(':')) else: contents.append([]) # make index logging.info('***********Now Make Index by sqlitedict***********') timer_start = timeit.default_timer() pos2paid = zip(range(len(f)), ac_ids) paid2pos_rel = {} for key, paid in groupby(sorted(pos2paid, key=itemgetter(1)), key=itemgetter(1)): paid2pos_rel.update({int(key): [i[0] for i in paid]}) id2pos_rel = dict(zip(ids, range(len(f)))) pos2id_rel = dict(zip(range(len(f)), ids)) id2pos = SqliteDict(filename=gl.res + '/resource/id2pos', autocommit=True) id2pos.clear() id2pos.update(id2pos_rel) id2pos.close() pos2id = SqliteDict(filename=gl.res + '/resource/pos2id', autocommit=True) pos2id.clear() pos2id.update(pos2id_rel) pos2id.close() paid2pos = SqliteDict(filename=gl.res + '/resource/paid2pos', autocommit=True) paid2pos.clear() paid2pos.update(paid2pos_rel) paid2pos.close() timer_end = timeit.default_timer() make_index_time = timer_end - timer_start # make dict logging.info('***********Now Make Dictionary***********') timer_start = timeit.default_timer() dic = corpora.Dictionary(contents) ############## optimized dictionary dic.filter_extremes(no_below=20, no_above=0.1, keep_n=None) ############## dic.save(gl.res + '/resource/dict') timer_end = timeit.default_timer() make_dict_time = timer_end - timer_start # make corpus logging.info('***********Now Make Corpus***********') temps = [] for i, t in enumerate(contents): temps.append(dic.doc2bow(t)) if i % 10000 == 0: logging.info('make corpus ' + str(i) + ' articles') corpus = temps corpora.MmCorpus.serialize(gl.res + '/resource/corpus', corpus) if tfidf: # do tfidf train logging.info('***********Now Training TF-IDF Model***********') timer_start = timeit.default_timer() corpus = corpora.MmCorpus(gl.res + '/resource/corpus') tfidf = models.TfidfModel(corpus) tfidf.save(gl.res + '/resource/tfidf') timer_end = timeit.default_timer() make_tfidf_time = timer_end - timer_start if hdp: gc.collect() corpus = corpora.MmCorpus(gl.res + '/resource/corpus') dic = corpora.Dictionary.load(gl.res + '/resource/dict') hdpmodel = models.hdpmodel.HdpModel(corpus, id2word=dic) hdptopicnum = len(hdpmodel.print_topics(topics=-1, topn=10)) logging.info('hdptopicnum is {}'.format(hdptopicnum)) if lda: # do lda train gc.collect() tfidf = models.TfidfModel.load(gl.res + '/resource/tfidf') corpus = corpora.MmCorpus(gl.res + '/resource/corpus') dic = corpora.Dictionary.load(gl.res + '/resource/dict') corpus_tfidf = tfidf[corpus] logging.info('***********Now Training LDA Model***********') timer_start = timeit.default_timer() if not hdptopicnum == 0: gl.topicCount = hdptopicnum lda = models.LdaMulticore(corpus_tfidf, id2word=dic, chunksize=gl.chunksize, num_topics=gl.topicCount, workers=gl.workers, passes=gl.lda_passes) # lda = models.LdaModel(corpus_tfidf, id2word=dic, chunksize=gl.chunksize, # num_topics=gl.topicCount, passes=gl.lda_passes, distributed=True) lda.save(gl.res + '/resource/lda') timer_end = timeit.default_timer() make_lda_time = timer_end - timer_start logging.info('lda training cost %.2f seconds' % make_lda_time) if sim: gc.collect() logging.info('***********Now Make Similarity Index***********') st = timeit.default_timer() corpus = corpora.MmCorpus(gl.res + '/resource/corpus') lda = models.LdaModel.load(gl.res + '/resource/lda') index = similarities.MatrixSimilarity(lda[corpus], num_features=gl.topicCount) index.save(gl.res + '/resource/simIndex') sim_time = timeit.default_timer() - st total_end = timeit.default_timer() total_time = total_end - total_start m = divmod(total_time, 60) h = divmod(m[0], 60) logging.info('\nReset LDA Model complete!!!\n' '***Using time*** \n' 'index training {:.2f}\n' 'dict training {:.2f}\n' 'tfidf training {:.2f}\n' 'lda training {:.2f}\n' 'sim training {:.2f}\n' 'Total time: {:d}h {:d}m {:.2f}s'.format(make_index_time, make_dict_time, make_tfidf_time, make_lda_time, sim_time, int(h[0]), int(h[1]), m[1])) basicConfig = open(gl.res + '/resource/basicConfig.txt', mode='w+') basicConfig.write('FileName: {}' '\nTopicNumber = {}' '\nestTopicNumber = {}' '\nldaPasses = {}' .format(os.path.basename(texts.name), gl.topicCount, hdptopicnum, gl.lda_passes)) basicConfig.close()
class SqliteSparseSequence(MutableSequence[Any]): def __init__(self, filename: Union[str, PathLike], read_only: bool = False): self.table = SqliteDict(filename, "sparse_sequence", flag="r" if read_only else "c") def __del__(self): self.close() def __getitem__(self, i: Union[int, slice]) -> Any: if isinstance(i, int): try: return self.table[str(i)] except KeyError: current_length = len(self) if i >= current_length or current_length <= 0: raise IndexError("list index out of range") elif i < 0 < current_length: return self.__getitem__(i % current_length) else: return None elif isinstance(i, slice): return SlicedSequence(self, i) else: raise TypeError(f"list indices must be integers or slices, not {i.__class__.__name__}") def __setitem__(self, i: Union[int, slice], value: Any): if isinstance(i, int): current_length = len(self) if i < 0: i %= current_length self.table[str(i)] = value self.table["_len"] = max(i, current_length) self.table.commit() else: raise TypeError(f"list indices must be integers, not {i.__class__.__name__}") def __delitem__(self, i: Union[int, slice]): current_length = len(self) if isinstance(i, int): if i < 0: i %= current_length if i >= current_length: raise IndexError("list assignment index out of range") for index in range(i + 1, current_length): self.table[str(index - 1)] = self.table.get(str(index)) del self.table[str(current_length - 1)] self.table["_len"] = current_length - 1 self.table.commit() elif isinstance(i, slice): # This isn't very efficient for continuous slices. for index in reversed(range(*i.indices(current_length))): del self[index] else: raise TypeError(f"list indices must be integers or slices, not {i.__class__.__name__}") def extend(self, values: Iterable[Any]) -> None: current_length = len(self) index = -1 for index, value in enumerate(values): self.table[str(index + current_length)] = value if index < 0: return self.table["_len"] = current_length + index + 1 self.table.commit() def insert(self, i: int, value: Any) -> None: current_length = len(self) for index in reversed(range(i, current_length)): self.table[str(index + 1)] = self.table.get(str(index)) self.table[str(i)] = value self.table["_len"] = current_length + 1 self.table.commit() def __len__(self) -> int: try: return self.table["_len"] except KeyError: return 0 def clear(self) -> None: self.table.clear() self.table.commit() def close(self) -> None: if self.table is not None: self.table.close() self.table = None def copy_to(self, target: Union[str, PathLike]): try: os.link(self.table.filename, target) except OSError as e: if e.errno == 18: # Cross-device link shutil.copy(self.table.filename, target) else: raise
class MyForm(QMainWindow): def __init__(self): super().__init__() self.ui = Ui_MainWindow() self.ui.setupUi(self) self.ui.button_load_data.clicked.connect(self.load) self.ui.edit_database_path.editingFinished.connect(self.load_from_lineEdit) self.ui.button_save.clicked.connect(self.save) self.ui.button_add.clicked.connect(self.add_entry) self.ui.button_plan.clicked.connect(self.gen_week_table) def closeEvent(self,event): close = QMessageBox() close.setText("Speichern ?") close.setStandardButtons(QMessageBox.Yes | QMessageBox.Cancel) close = close.exec() if close == QMessageBox.Yes: self.save() event.accept() else: event.accept() def load(self): # Load SQLdict #try: self.fname = QFileDialog.getOpenFileName(self, 'Open file','','Database *.sqlite') # gt file path self.ui.edit_database_path.setText('Loaded database: ' + self.fname[0]) # set path to LineEdit self.Kochbuch = SqliteDict(self.fname[0], autocommit=True) # Lade Haupt dictionary/ Gerichte self.create_content_table() #except: #print('Cannot load specified file!\nError in main.load()') #pass def load_from_lineEdit(self): # Load from LineEdit try: self.fname[0] = self.ui.edit_database_path.text()[17:] # get text without prefix self.Kochbuch = SqliteDict(self.fname[0], autocommit=True) # Lade Haupt dictionary/ Gerichte self.create_content_table() except: print('Cannot load specified file!\nError in main.load_from_lineEdit()') pass def save(self): # get items from content_table, update Kochbuch and commit it to database self.Kochbuch.clear() table = self.ui.content_table header_items = [table.model().headerData(i, Qt.Horizontal) for i in range(table.model().columnCount())] for row_index in range(self.ui.content_table.rowCount()): # Every row is one dish/gericht temp_dict = dict() for col_index, item in enumerate(header_items): temp_dict[item] = table.cellWidget(row_index,col_index).text() self.add_gericht(temp_dict) def add_entry(self): # Add empty entry to table row_cnt = self.ui.content_table.rowCount() col_cnt = self.ui.content_table.columnCount() self.ui.content_table.insertRow(row_cnt) for col_index in range(col_cnt): self.ui.content_table.setCellWidget(row_cnt, col_index, QLineEdit()) if col_index == col_cnt - 1: # Delete Option self.ui.content_table.setCellWidget(row_cnt, col_index, QPushButton('Delete')) self.ui.content_table.cellWidget(row_cnt, col_index).clicked.connect(self.remove_entry) def remove_entry(self): table = self.ui.content_table # --------------Remove Row------------ column = table.currentColumn() row = table.currentRow() table.removeRow(row) # -------------Remove dict entry-------- #name = table.cellWidget(row,0).text() #self.del_gericht(name) def create_content_table(self): # Creates the Widgets inside the Table table = self.ui.content_table table.setRowCount(len(self.Kochbuch)) header_items = [table.model().headerData(i, Qt.Horizontal) for i in range(table.model().columnCount())] row_label = [] col_cnt = table.model().columnCount() for row_index, val in enumerate(self.Kochbuch.items()): #row_label.append(str(row_index + 1) + ' ' + str(val[0])) for col_index in range(col_cnt): table.setCellWidget(row_index,col_index,QLineEdit()) if col_index == col_cnt - 1: # Add Delete Button table.setCellWidget(row_index, col_index, QPushButton('Delete')) table.cellWidget(row_index,col_index).clicked.connect(self.remove_entry) #self.ui.content_table.setVerticalHeaderLabels(row_label) self.set_text_to_table(header_items) def set_text_to_table(self,header_items): table = self.ui.content_table for row_index, val in enumerate(self.Kochbuch.items()): table.cellWidget(row_index, 0).setText(val[0]) # Name column/ set Name #print(val[1].values()) for col_index, item in enumerate(header_items[1:]): try: table.cellWidget(row_index, col_index + 1).setText(val[1][item]) except KeyError: if item == None: # Used, that the delete button text will not be overwritten pass else: # Set unfilled category empty table.cellWidget(row_index, col_index + 1).setText('') def add_gericht(self, entries:dict): # Olf func args: name: str, Fisch: bool, Nudeln: bool, Vortag: bool, SE: bool, WE: bool, WE_Wichtung: float, Wichtung: float # Gerichte werden gespeichert in dict() # Jedes Gericht wird dabei Kategorisiert in: # - Fisch: Bool # - Nudeln: Bool # - Vortag: Bool (Wenn Sonntags viel Bestellt wird das Essen vom Vortag machen) # - SE: Bool (Sonntagsessen) # - WE: Bool (Wochenendessen, wie z.B Holen/Bestellen) # - WE_Wichtung: Float (Jedes Gericht soll die chance haben am WE dran zu kommen, Holen/Bestellen oder z.B. Rolladen sollen bevorzugt werden) # - Wichtung: Float (Warkeit des Gerichtes ausgewählt zu werden, um doppelte zu vermeiden) 1.0 = Kommt dran; 0 = wird nicht dran kommen # etc. # ------------------------------------------------------------------------------- # Tortillas = dict() # Ein Gericht dict() # Tortillas['Fisch'] = False # Tortillas['Nudeln'] = False # Tortillas['Vortag'] = False # Tortillas['SE'] = False # Tortillas['WE'] = False # Tortillas['WE_Wichtung'] = 0.1 # Tortillas['Wichtung'] = 1.0 # Gerichte['Tortillas'] = Tortillas # ------------------------------------------------------------------------------- '''name_dict = dict() name_dict['Fisch'] = Fisch name_dict['Nudeln'] = Nudeln name_dict['Vortag'] = Vortag name_dict['SE'] = SE name_dict['WE'] = WE name_dict['WE_Wichtung'] = WE_Wichtung name_dict['Wichtung'] = Wichtung''' name = entries['Name'] self.Kochbuch[name] = entries def del_gericht(self,name): self.Kochbuch.pop(name) def update_kochbuch(self, Kochbuch: dict, name: str, kategorie: str, value: any): # Da hier mit dict im dict gearbeitet wird und dasa äußerste dict ein SQLdict ist, # müssen hier die Einträge ein bisschen umständlich verändert werden, um vom RAM in die SQL datei zu schreiben update = Kochbuch[name] update[kategorie] = value Kochbuch[name] = update def choose(self,dishes): choosed_dish = np.asarray(dishes) Wichtungen = choosed_dish[:,1].astype(np.float) #choosed_dish = np.sort(choosed_dish) # Find maximas in Wichtung column max_indizes = np.where(Wichtungen==np.amax(Wichtungen)) finds = [] for i in max_indizes[0]: finds.append(choosed_dish[i]) # Choose dish # If len of finds > 1 use random int to choose dish if len(finds) > 1: dish_index = random.randint(0,len(finds) - 1) return finds[dish_index] else: return finds #print('test','\n',choosed_dish) def gen_week_table(self): # generate table of dishes day wise dishes = [i for i in self.Kochbuch.items()] dishes_cnt = len(dishes) usable_dishes = [] possible_dishes_mon = [] possible_dishes_tue = [] possible_dishes_wed = [] possible_dishes_thu = [] possible_dishes_fri = [] possible_dishes_sat = [] possible_dishes_sun = [] saison = 'Winter' for index, dish in enumerate(dishes): dish = dish[1] # Perform standard check, to reduce dishes according to seasons and weigth #if float(dish['Wichtung']) > 0.7 and dish['Saison'] == saison: # Standard check if float(dish['Wichtung']) > 0.7 and (dish['Saison'] == saison or dish['Saison'] == 'None'): usable_dishes.append(dish) # -----------Monday------------- # ------------------------------ # Mondays should prefer Nudeln ---> Boni for Nudeln == True if dish['Fisch'] == 'False': if dish['Nudeln'] == 'True': possible_dishes_mon.append([dish['Name'], float(dish['Wichtung']) + 0.3]) else: possible_dishes_mon.append([dish['Name'], float(dish['Wichtung'])]) #----------------------------------------------------------------------------- # -----------Tuesday/Wednesday/Thursday------------- # -------------------------------------------------- # Days without preferations if dish['Fisch'] == 'False' and dish['Vortag'] == 'False': # Standard check possible_dishes_tue.append([dish['Name'], float(dish['Wichtung'])]) possible_dishes_wed.append([dish['Name'], float(dish['Wichtung'])]) possible_dishes_thu.append([dish['Name'], float(dish['Wichtung'])]) # ----------------------------------------------------------------------------- # -----------Friday------------- # ------------------------------ # Fish prefered if dish['WE'] == 'True' and dish['SE'] == 'False': if dish['Fisch'] == 'True': possible_dishes_fri.append([dish['Name'], float(dish['Wichtung']) + 0.3]) else: possible_dishes_fri.append([dish['Name'], float(dish['Wichtung'])]) # ----------------------------------------------------------------------------- # -----------Saturday------------- # -------------------------------- # WE category prefered if dish['Fisch'] == 'False' and dish['SE'] == 'False' and dish['Vortag'] == 'False': if dish['WE'] == 'True': possible_dishes_sat.append([dish['Name'], float(dish['Wichtung']) + 0.3]) else: possible_dishes_sat.append([dish['Name'], float(dish['Wichtung'])]) # ----------------------------------------------------------------------------- # -----------Sunday------------- # ------------------------------ # SE highly prefered if dish['Fisch'] == 'False' and dish['Vortag'] == 'False': if dish['SE'] == 'True': possible_dishes_sun.append([dish['Name'], float(dish['Wichtung']) + 0.5]) else: possible_dishes_sun.append([dish['Name'], float(dish['Wichtung'])]) print('============================================================================') print('=================================Wochenplan=================================') print('============================================================================') print('Monday: ', self.choose(possible_dishes_mon)[0]) print('----------------------------------------------------------------------------') print('Tuesday: ', self.choose(possible_dishes_tue)[0]) print('----------------------------------------------------------------------------') print('Wednesday: ', self.choose(possible_dishes_wed)[0]) print('----------------------------------------------------------------------------') print('Thurday: ', self.choose(possible_dishes_thu)[0]) print('----------------------------------------------------------------------------') print('Friday: ', self.choose(possible_dishes_fri)[0]) print('----------------------------------------------------------------------------') print('Saturday: ', self.choose(possible_dishes_sat)[0]) print('----------------------------------------------------------------------------') print('Sunday: ', self.choose(possible_dishes_sun)[0]) print('============================================================================') #print(self.choose(possible_dishes_mon)[0]) Speiseplan = collections.OrderedDict()
class SQLiteRepository(Repository): repository_name: str = 'sqlite' extension: str = '.db' def __init__(self, repository_path: str, commit_on_close: bool = True, verbosity: int = 0): super().__init__(repository_path, commit_on_close=commit_on_close, verbosity=verbosity) self.sqlite_repository = None self.table_name = None @contextmanager def connect(self, table_name: str) -> 'SQLiteRepository': yield self.open(table_name) self.close() def open(self, table_name: str): self.sqlite_repository = SqliteDict( self.repository_path, tablename=table_name, encode=json.dumps, decode=json.loads, flag='c') self.table_name = table_name return self def close(self): if self.sqlite_repository is not None: if self.commit_on_close: self.commit() self.sqlite_repository.close() self.sqlite_repository = None self.table_name = None def commit(self): self.sqlite_repository.commit() def keys(self) -> List[str]: return list(self.sqlite_repository.keys()) def update(self, key: str, update_obj: dict): self.sqlite_repository[key] = update_obj def upsert(self, key: str, obj: dict): self.sqlite_repository[key] = obj def get(self, key: str) -> dict: try: return self.sqlite_repository[key] except KeyError: raise InvalidEntryError(key) def get_multiple(self, keys: List[str]) -> Dict[str, dict]: values = {key: element for key, element in self.sqlite_repository.items() if key in keys} if len(set(keys)) != len(values): invalids = set(keys).difference(values.keys()) raise InvalidEntryError(', '.join(list(invalids))) return values def get_all(self) -> Dict[str, dict]: return {key: element for key, element in self.sqlite_repository.items()} def remove(self, key: str): try: del self.sqlite_repository[key] except KeyError: raise InvalidEntryError(key) def remove_multiple(self, keys: List[str]): for key in keys: self.remove(key) def clear(self): self.sqlite_repository.clear() def __repr__(self): return f"SQLiteRepository(file='{self.repository_path}', open={self.sqlite_repository is not None}, " \ f"table='{self.table_name}')"
""" sqlitedict支持Multi Thread Safe Write。也就是可以用多线程或是多个脚本对同一个 字典进行写操作。 """ from __future__ import print_function import os import time import string import random from multiprocessing.dummy import Pool from sqlitedict import SqliteDict file = "./sqlitedict.sqlite" mydict = SqliteDict(file, autocommit=True) mydict.clear() def multiple_script(): """多个脚本调用同一个字典。 """ s = "x" * 10000 st = time.clock() for i in range(10000): mydict[str(i)] = s elapsed = time.clock() - st print("elapsed %.6f seconds." % elapsed) assert len(mydict) == 10000
class Broker(object): def __init__(self, urls, id): self.id = id self.peer_urls = urls['peers'] self.queued_msgs = {} self.replicated_msgs = {} self.capacity = MAX_MSG_CAPACITY # self.consume_lock = threading.Lock() # self.consuming = False asyncio.set_event_loop(asyncio.new_event_loop()) # self.loop = ioloop.IOLoop.current() self.__init_metrics() self.__init_sockets(urls) self.__set_callback() self.loop = IOLoop.instance() self.rocks = SqliteDict( './remotedb-%d.sqlite' % (self.id), autocommit=True) # self.AAL = SqliteDict('./local_aal-%d.sqlite' % (self.id), autocommit=True) self.AAL={} # self.DAL = SqliteDict('./local_dal-%d.sqlite' % (self.id), autocommit=True) self.DAL={} self.rocks.clear() # self.AAL.clear() # self.DAL.clear() def __init_sockets(self, urls): context = zmq.Context() self.frontend_sock = context.socket(zmq.ROUTER) self.backend_sock = context.socket(zmq.ROUTER) self.peerend_sock = context.socket(zmq.ROUTER) self.frontend_sock.bind(urls["producer"]) self.backend_sock.bind(urls["consumer"]) self.peerend_sock.bind(urls["self"]) self.context = zmq.Context.instance() self.socket = context.socket(zmq.REQ) self.socket.identity = (u"Broker-%d" % (self.id)).encode('ascii') def __set_callback(self): self.frontend = ZMQStream(self.frontend_sock) self.backend = ZMQStream(self.backend_sock) self.peerend = ZMQStream(self.peerend_sock) self.frontend.on_recv(self.handle_frontend) self.backend.on_recv(self.handle_backend) self.peerend.on_recv(self.handle_peerend) def __init_metrics(self): pass def __lost_peer(self, url): pass def __new_peer_online(self, url): pass def __insert_add_ahead_log(self, prod_id, msg_id, dst): """ Implementation of add-ahead-log (AAL) """ if DEBUG == True: print('Thread', self.id, 'inserting AAL', prod_id, msg_id, dst) if prod_id not in self.AAL.keys(): self.AAL[prod_id] = {} self.AAL[prod_id][msg_id] = dst def __remove_add_ahead_log(self, prod_id, msg_id): if DEBUG == True: print("removing AAL", prod_id, msg_id) del self.AAL[prod_id][msg_id] def __insert_delete_ahead_log(self, prod_id, msg_id, dst): """ Implementation of delete-ahead-log (DAL) """ if DEBUG == True: print('Thread', self.id, 'inserting DAL', prod_id, msg_id, dst) if prod_id not in self.DAL.keys(): self.DAL[prod_id] = {} self.DAL[prod_id][msg_id] = dst def __remove_delete_ahead_log(self, prod_id, msg_id): if DEBUG == True: print("removing DAL", prod_id, msg_id) del self.DAL[prod_id][msg_id] def __send_replicates(self, prod_id, msg_id, payload): for url in self.peer_urls: self.socket.connect(url) self.__insert_add_ahead_log(prod_id, msg_id, url) self.socket.send_multipart([b'REPLICATE', prod_id, msg_id, payload]) reply = self.socket.recv() if reply == b'REPLICATED': self.__remove_add_ahead_log(prod_id, msg_id) self.socket.disconnect(url) def __delete_replicates(self, prod_id, msg_id): """ needs an efficient way to store metadata of all replicates """ for url in self.peer_urls: self.socket.connect(url) self.__insert_delete_ahead_log(prod_id, msg_id, url) self.socket.send_multipart([b'DELETE', prod_id, msg_id]) reply = self.socket.recv() if reply == b'DELETED': self.__remove_delete_ahead_log(prod_id, msg_id) self.socket.disconnect(url) def __persist_msgs(self, prod_id, msg_id, msg): if prod_id not in self.rocks.keys(): self.rocks[prod_id] = {} self.rocks[prod_id][msg_id] = msg def __retrieve_msgs(self, num=1): if len(list(self.rocks.keys())) == 0: return for _ in range(num): prod_id = random.choice(list(self.rocks.keys())) msg_id, msg = self.rocks[prod_id].popitem() self.queued_msgs[prod_id][msg_id] = msg def __get_size_of_queues(self, replica=False): return sum([len(queue) for queue in self.queued_msgs.values()]) def __enqueue_a_msg(self, prod_id, msg_id, payload, replica=False): if not replica: if self.__get_size_of_queues() <= self.capacity * PERSIST_THRESHOLD: if prod_id not in self.queued_msgs.keys(): self.queued_msgs[prod_id] = {} self.queued_msgs[prod_id][msg_id] = payload else: self.__persist_msgs(prod_id, msg_id, payload) # sorta redundant else: if prod_id not in self.replicated_msgs.keys(): self.replicated_msgs[prod_id] = {} self.replicated_msgs[prod_id][msg_id] = payload def __get_a_msg(self): try: prod_id = random.choice(list(self.queued_msgs.keys())) msg_id = random.choice(list(self.queued_msgs[prod_id].keys())) # msg_id, payload = self.queued_msgs[prod_id].popitem() return [prod_id, msg_id, self.queued_msgs[prod_id][msg_id]] except IndexError: return [None, None, None] def __dequeue_a_msg(self, prod_id, msg_id, replica=False): if not replica: if self.__get_size_of_queues() == self.capacity * PERSIST_THRESHOLD: self.__retrieve_msgs() if DEBUG == True: print(self.id, "dequing:", self.queued_msgs[prod_id]) try: return self.queued_msgs[prod_id].pop(msg_id) except IndexError: if DEBUG == True: print("Already consumed. (Delivered twice)") return None else: if DEBUG == True: print(self.id, "dequing replica:", self.replicated_msgs[prod_id]) try: return self.replicated_msgs[prod_id].pop(msg_id) except KeyError: return None def __consume_a_msg(self, cons_id): prod_id, msg_id, payload = self.__get_a_msg() if prod_id is None: next_timer = threading.Timer(NEXT_CONSUME_TIME, self.__consume_a_msg, [cons_id]) next_timer.start() return else: pass # print('get_a_msg', prod_id, msg_id, payload) self.backend.send_multipart([cons_id, prod_id, msg_id, payload]) def handle_frontend(self, msg): if DEBUG == True: print(self.id, 'handling frontend:', msg) start_time = time.time_ns() prod_id, msg_id, payload = msg[:3] self.__enqueue_a_msg(prod_id, msg_id, payload) self.__send_replicates(prod_id, msg_id, payload) # print("Time for frontend is", time.time_ns() - start_time) def handle_backend(self, msg): if DEBUG == True: print(self.id, 'handling backend:', msg) cons_id, command = msg[:2] start_time = time.time_ns() if command == b'CONSUME': start_time = time.time_ns() self.__consume_a_msg(cons_id) elif command == b'CONSUMED': prod_id, msg_id = msg[2:4] res = self.__dequeue_a_msg(prod_id, msg_id) if res is not None: self.__delete_replicates(prod_id, msg_id) # print("Time for backend is", time.time_ns() - start_time) def handle_peerend(self, msg): if DEBUG == True: print(self.id, 'handling peerend', msg) broker_id, _, command = msg[:3] start_time = None if command == b'REPLICATE': start_time = time.time_ns() prod_id, msg_id, payload = msg[3:6] self.__enqueue_a_msg(prod_id, msg_id, payload, replica=True) # self.peerend.send_multipart([broker_id, b'', b'REPLICATED', prod_id, msg_id]) self.peerend.send_multipart([broker_id, b'', b'REPLICATED']) # print("Time for replication is", time.time_ns() - start_time) elif command == b'DELETE': start_time = time.time_ns() prod_id, msg_id = msg[3:5] self.__dequeue_a_msg(prod_id, msg_id, replica=True) # self.peerend.send_multipart([broker_id, b'', b'DELETED', prod_id, msg_id]) self.peerend.send_multipart([broker_id, b'', b'DELETED']) # print("Time for deletion is", time.time_ns() - start_time) # elif command == b"REPLICATED": # self.__remove_add_ahead_log(prod_id, msg_id) # elif command == b"DELETED": # self.__remove_delete_ahead_log(prod_id, msg_id) def run(self): self.loop.start() def stop(self): self.loop.stop()