class DBManager():
def __init__(self, store):
self._db = SqliteDict(store, autocommit=True)
self.commands_cache = {}
def add(self, command):
if self.get_command(command.get_name()) != None:
return False
self._db[command.get_name()] = command
return True
def update(self, command):
self.add(command)
def remove(self, command):
del self._db[command.get_name()]
def empty(self):
self.commands_cache = {}
self._db.clear()
def count(self):
return len(self._db)
def get_command(self, name):
if name not in self.commands_cache:
try:
self.commands_cache[name] = self._db[name]
except KeyError:
return None
return self.commands_cache[name]
def get_all_commands(self):
list = []
for key, val in self._db.iteritems():
list.append(key)
return list
class PrefOrderedSet:
"""
store/retrieve an ordered set of strings (like a list, but no duplicates) to/from a sqlite database
"""
def __init__(self, name: str, author: str, table: str):
# DB stores values directly (not encoded as a pickle)
self.sqlite_dict = SqliteDict(get_sqlite_path(name, author),
table,
encode=lambda x: x,
decode=lambda x: x)
def set(self, strings: list):
"""
set the list of strings
:param strings: list of strings
"""
self.sqlite_dict.clear() # delete entire table
# ordering is done by making the value in the key/value pair the index and our desired list "value" is the key
for index, string in enumerate(strings):
self.sqlite_dict[string] = index
self.sqlite_dict.commit(
) # not using autocommit since we're updating (setting) multiple values in the above for loop
def get(self) -> List[str]:
"""
returns the list of strings
:return: list of strings
"""
return list(sorted(self.sqlite_dict, key=self.sqlite_dict.get))
class DBManager():
def __init__(self):
self._db = SqliteDict('./matches.sqlite', autocommit=True)
self.matches_cache = {}
def add(self, match):
self._db[match.get_code()] = match
def update(self, match):
self.add(match)
def remove(self, match):
del self._db[match.get_code()]
def empty(self):
self._db.clear()
def count(self):
return len(self._db)
def get_match(self, code):
if code not in self.matches_cache:
try:
self.matches_cache[code] = self._db[code]
except KeyError:
return None
return self.matches_cache[code]
def get_all_matches(self):
list = []
for key, val in self._db.iteritems():
list.append(key)
return list
class CDataBase(object):
def __init__(self):
try:
self.close()
except:
pass
self.mydict = SqliteDict('./DB/my_db.sqlite', autocommit=True)
self.show()
def set(self, key, value):
self.mydict[key] = value
def get(self, key):
if key in self.mydict.keys():
ret = self.mydict[key]
else:
ret = None
return ret
def show(self, start_with=''):
for key, value in self.mydict.items():
if key.find(start_with):
print(key, '\t', value, '\n')
def clear(self):
self.mydict.clear()
def close(self):
self.mydict.close()
class ImageCache:
def __init__(self, directory):
db_dirname = os.path.join(directory, ".mikula")
if not os.path.isdir(db_dirname):
os.mkdir(db_dirname)
db_filename = os.path.join(db_dirname, "images.cache")
self.cache = SqliteDict(db_filename)
self.recent_lookup_ = None
def reset(self):
self.cache.clear()
self.recent_lookup_ = None
def config_changed(self, config):
if "config" not in self.cache.keys():
return True
stored = self.cache["config"]
return stored != config
def update_config(self, config):
self.cache["config"] = config
self.cache.commit()
def require_update(self, filename):
if filename not in self.cache.keys():
self.recent_lookup_ = None
return True
timestamp, scaled, scaled_time, thumbnail, thumbnail_time = self.cache[
filename]
if os.path.exists(scaled) and os.path.exists(thumbnail):
if os.path.getmtime(filename) == timestamp and \
os.path.getmtime(scaled) == scaled_time and \
os.path.getmtime(thumbnail) == thumbnail_time:
self.recent_lookup_ = (scaled, thumbnail)
return False
self.recent_lookup_ = None
return True
def get_filenames(self):
return self.recent_lookup_
def update(self, filename, scaled, thumbnail):
timestamp = os.path.getmtime(filename)
scaled_time = os.path.getmtime(scaled)
thumbnail_time = os.path.getmtime(thumbnail)
self.cache[filename] = (timestamp, scaled, scaled_time, thumbnail,
thumbnail_time)
self.cache.commit()
def open_shelf(file_name: str,
new: bool = False,
raise_errors: bool = True,
autocommit: bool = True) -> SqliteDict:
try:
sld = SqliteDict(file_name, autocommit=autocommit)
if new:
print('Deleting old shelf file and creating a new one.')
sld.clear()
except Exception as e:
print(e)
if raise_errors:
raise e
print(
'Error during shelf open action. Since raise_errors is false, starting from a new file.'
)
sld = open_shelf(file_name, new=True, raise_errors=True)
return sld
def reset(texts, index_dic=True, tfidf=True, hdp=False, lda=True, sim=False):
total_start = timeit.default_timer()
make_index_time = 0
make_dict_time = 0
make_lda_time = 0
make_tfidf_time = 0
sim_time = 0
hdptopicnum = 0
if index_dic:
f = [i.split(',') for i in texts.readlines()]
logging.info('Create id & ac_id list')
ids = [f[i][1] for i in range(len(f))]
ac_ids = [f[i][0] for i in range(len(f))]
logging.info('Create contents list')
contents = []
for i in range(len(f)):
if len(f[i]) == 3:
contents.append(f[i][2].strip().split(':'))
else:
contents.append([])
# make index
logging.info('***********Now Make Index by sqlitedict***********')
timer_start = timeit.default_timer()
pos2paid = zip(range(len(f)), ac_ids)
paid2pos_rel = {}
for key, paid in groupby(sorted(pos2paid, key=itemgetter(1)), key=itemgetter(1)):
paid2pos_rel.update({int(key): [i[0] for i in paid]})
id2pos_rel = dict(zip(ids, range(len(f))))
pos2id_rel = dict(zip(range(len(f)), ids))
id2pos = SqliteDict(filename=gl.res + '/resource/id2pos', autocommit=True)
id2pos.clear()
id2pos.update(id2pos_rel)
id2pos.close()
pos2id = SqliteDict(filename=gl.res + '/resource/pos2id', autocommit=True)
pos2id.clear()
pos2id.update(pos2id_rel)
pos2id.close()
paid2pos = SqliteDict(filename=gl.res + '/resource/paid2pos', autocommit=True)
paid2pos.clear()
paid2pos.update(paid2pos_rel)
paid2pos.close()
timer_end = timeit.default_timer()
make_index_time = timer_end - timer_start
# make dict
logging.info('***********Now Make Dictionary***********')
timer_start = timeit.default_timer()
dic = corpora.Dictionary(contents)
############## optimized dictionary
dic.filter_extremes(no_below=20, no_above=0.1, keep_n=None)
##############
dic.save(gl.res + '/resource/dict')
timer_end = timeit.default_timer()
make_dict_time = timer_end - timer_start
# make corpus
logging.info('***********Now Make Corpus***********')
temps = []
for i, t in enumerate(contents):
temps.append(dic.doc2bow(t))
if i % 10000 == 0:
logging.info('make corpus ' + str(i) + ' articles')
corpus = temps
corpora.MmCorpus.serialize(gl.res + '/resource/corpus', corpus)
if tfidf:
# do tfidf train
logging.info('***********Now Training TF-IDF Model***********')
timer_start = timeit.default_timer()
corpus = corpora.MmCorpus(gl.res + '/resource/corpus')
tfidf = models.TfidfModel(corpus)
tfidf.save(gl.res + '/resource/tfidf')
timer_end = timeit.default_timer()
make_tfidf_time = timer_end - timer_start
if hdp:
gc.collect()
corpus = corpora.MmCorpus(gl.res + '/resource/corpus')
dic = corpora.Dictionary.load(gl.res + '/resource/dict')
hdpmodel = models.hdpmodel.HdpModel(corpus, id2word=dic)
hdptopicnum = len(hdpmodel.print_topics(topics=-1, topn=10))
logging.info('hdptopicnum is {}'.format(hdptopicnum))
if lda:
# do lda train
gc.collect()
tfidf = models.TfidfModel.load(gl.res + '/resource/tfidf')
corpus = corpora.MmCorpus(gl.res + '/resource/corpus')
dic = corpora.Dictionary.load(gl.res + '/resource/dict')
corpus_tfidf = tfidf[corpus]
logging.info('***********Now Training LDA Model***********')
timer_start = timeit.default_timer()
if not hdptopicnum == 0:
gl.topicCount = hdptopicnum
lda = models.LdaMulticore(corpus_tfidf, id2word=dic, chunksize=gl.chunksize,
num_topics=gl.topicCount, workers=gl.workers, passes=gl.lda_passes)
# lda = models.LdaModel(corpus_tfidf, id2word=dic, chunksize=gl.chunksize,
# num_topics=gl.topicCount, passes=gl.lda_passes, distributed=True)
lda.save(gl.res + '/resource/lda')
timer_end = timeit.default_timer()
make_lda_time = timer_end - timer_start
logging.info('lda training cost %.2f seconds' % make_lda_time)
if sim:
gc.collect()
logging.info('***********Now Make Similarity Index***********')
st = timeit.default_timer()
corpus = corpora.MmCorpus(gl.res + '/resource/corpus')
lda = models.LdaModel.load(gl.res + '/resource/lda')
index = similarities.MatrixSimilarity(lda[corpus], num_features=gl.topicCount)
index.save(gl.res + '/resource/simIndex')
sim_time = timeit.default_timer() - st
total_end = timeit.default_timer()
total_time = total_end - total_start
m = divmod(total_time, 60)
h = divmod(m[0], 60)
logging.info('\nReset LDA Model complete!!!\n'
'***Using time*** \n'
'index training {:.2f}\n'
'dict training {:.2f}\n'
'tfidf training {:.2f}\n'
'lda training {:.2f}\n'
'sim training {:.2f}\n'
'Total time: {:d}h {:d}m {:.2f}s'.format(make_index_time, make_dict_time, make_tfidf_time,
make_lda_time, sim_time, int(h[0]), int(h[1]), m[1]))
basicConfig = open(gl.res + '/resource/basicConfig.txt', mode='w+')
basicConfig.write('FileName: {}'
'\nTopicNumber = {}'
'\nestTopicNumber = {}'
'\nldaPasses = {}'
.format(os.path.basename(texts.name), gl.topicCount, hdptopicnum, gl.lda_passes))
basicConfig.close()
class SqliteSparseSequence(MutableSequence[Any]):
def __init__(self, filename: Union[str, PathLike], read_only: bool = False):
self.table = SqliteDict(filename, "sparse_sequence", flag="r" if read_only else "c")
def __del__(self):
self.close()
def __getitem__(self, i: Union[int, slice]) -> Any:
if isinstance(i, int):
try:
return self.table[str(i)]
except KeyError:
current_length = len(self)
if i >= current_length or current_length <= 0:
raise IndexError("list index out of range")
elif i < 0 < current_length:
return self.__getitem__(i % current_length)
else:
return None
elif isinstance(i, slice):
return SlicedSequence(self, i)
else:
raise TypeError(f"list indices must be integers or slices, not {i.__class__.__name__}")
def __setitem__(self, i: Union[int, slice], value: Any):
if isinstance(i, int):
current_length = len(self)
if i < 0:
i %= current_length
self.table[str(i)] = value
self.table["_len"] = max(i, current_length)
self.table.commit()
else:
raise TypeError(f"list indices must be integers, not {i.__class__.__name__}")
def __delitem__(self, i: Union[int, slice]):
current_length = len(self)
if isinstance(i, int):
if i < 0:
i %= current_length
if i >= current_length:
raise IndexError("list assignment index out of range")
for index in range(i + 1, current_length):
self.table[str(index - 1)] = self.table.get(str(index))
del self.table[str(current_length - 1)]
self.table["_len"] = current_length - 1
self.table.commit()
elif isinstance(i, slice):
# This isn't very efficient for continuous slices.
for index in reversed(range(*i.indices(current_length))):
del self[index]
else:
raise TypeError(f"list indices must be integers or slices, not {i.__class__.__name__}")
def extend(self, values: Iterable[Any]) -> None:
current_length = len(self)
index = -1
for index, value in enumerate(values):
self.table[str(index + current_length)] = value
if index < 0:
return
self.table["_len"] = current_length + index + 1
self.table.commit()
def insert(self, i: int, value: Any) -> None:
current_length = len(self)
for index in reversed(range(i, current_length)):
self.table[str(index + 1)] = self.table.get(str(index))
self.table[str(i)] = value
self.table["_len"] = current_length + 1
self.table.commit()
def __len__(self) -> int:
try:
return self.table["_len"]
except KeyError:
return 0
def clear(self) -> None:
self.table.clear()
self.table.commit()
def close(self) -> None:
if self.table is not None:
self.table.close()
self.table = None
def copy_to(self, target: Union[str, PathLike]):
try:
os.link(self.table.filename, target)
except OSError as e:
if e.errno == 18: # Cross-device link
shutil.copy(self.table.filename, target)
else:
raise
class MyForm(QMainWindow):
def __init__(self):
super().__init__()
self.ui = Ui_MainWindow()
self.ui.setupUi(self)
self.ui.button_load_data.clicked.connect(self.load)
self.ui.edit_database_path.editingFinished.connect(self.load_from_lineEdit)
self.ui.button_save.clicked.connect(self.save)
self.ui.button_add.clicked.connect(self.add_entry)
self.ui.button_plan.clicked.connect(self.gen_week_table)
def closeEvent(self,event):
close = QMessageBox()
close.setText("Speichern ?")
close.setStandardButtons(QMessageBox.Yes | QMessageBox.Cancel)
close = close.exec()
if close == QMessageBox.Yes:
self.save()
event.accept()
else:
event.accept()
def load(self):
# Load SQLdict
#try:
self.fname = QFileDialog.getOpenFileName(self, 'Open file','','Database *.sqlite') # gt file path
self.ui.edit_database_path.setText('Loaded database: ' + self.fname[0]) # set path to LineEdit
self.Kochbuch = SqliteDict(self.fname[0], autocommit=True) # Lade Haupt dictionary/ Gerichte
self.create_content_table()
#except:
#print('Cannot load specified file!\nError in main.load()')
#pass
def load_from_lineEdit(self):
# Load from LineEdit
try:
self.fname[0] = self.ui.edit_database_path.text()[17:] # get text without prefix
self.Kochbuch = SqliteDict(self.fname[0], autocommit=True) # Lade Haupt dictionary/ Gerichte
self.create_content_table()
except:
print('Cannot load specified file!\nError in main.load_from_lineEdit()')
pass
def save(self):
# get items from content_table, update Kochbuch and commit it to database
self.Kochbuch.clear()
table = self.ui.content_table
header_items = [table.model().headerData(i, Qt.Horizontal) for i in range(table.model().columnCount())]
for row_index in range(self.ui.content_table.rowCount()): # Every row is one dish/gericht
temp_dict = dict()
for col_index, item in enumerate(header_items):
temp_dict[item] = table.cellWidget(row_index,col_index).text()
self.add_gericht(temp_dict)
def add_entry(self):
# Add empty entry to table
row_cnt = self.ui.content_table.rowCount()
col_cnt = self.ui.content_table.columnCount()
self.ui.content_table.insertRow(row_cnt)
for col_index in range(col_cnt):
self.ui.content_table.setCellWidget(row_cnt, col_index, QLineEdit())
if col_index == col_cnt - 1: # Delete Option
self.ui.content_table.setCellWidget(row_cnt, col_index, QPushButton('Delete'))
self.ui.content_table.cellWidget(row_cnt, col_index).clicked.connect(self.remove_entry)
def remove_entry(self):
table = self.ui.content_table
# --------------Remove Row------------
column = table.currentColumn()
row = table.currentRow()
table.removeRow(row)
# -------------Remove dict entry--------
#name = table.cellWidget(row,0).text()
#self.del_gericht(name)
def create_content_table(self):
# Creates the Widgets inside the Table
table = self.ui.content_table
table.setRowCount(len(self.Kochbuch))
header_items = [table.model().headerData(i, Qt.Horizontal) for i in range(table.model().columnCount())]
row_label = []
col_cnt = table.model().columnCount()
for row_index, val in enumerate(self.Kochbuch.items()):
#row_label.append(str(row_index + 1) + ' ' + str(val[0]))
for col_index in range(col_cnt):
table.setCellWidget(row_index,col_index,QLineEdit())
if col_index == col_cnt - 1: # Add Delete Button
table.setCellWidget(row_index, col_index, QPushButton('Delete'))
table.cellWidget(row_index,col_index).clicked.connect(self.remove_entry)
#self.ui.content_table.setVerticalHeaderLabels(row_label)
self.set_text_to_table(header_items)
def set_text_to_table(self,header_items):
table = self.ui.content_table
for row_index, val in enumerate(self.Kochbuch.items()):
table.cellWidget(row_index, 0).setText(val[0]) # Name column/ set Name
#print(val[1].values())
for col_index, item in enumerate(header_items[1:]):
try:
table.cellWidget(row_index, col_index + 1).setText(val[1][item])
except KeyError:
if item == None: # Used, that the delete button text will not be overwritten
pass
else:
# Set unfilled category empty
table.cellWidget(row_index, col_index + 1).setText('')
def add_gericht(self, entries:dict):
# Olf func args: name: str, Fisch: bool, Nudeln: bool, Vortag: bool, SE: bool, WE: bool, WE_Wichtung: float, Wichtung: float
# Gerichte werden gespeichert in dict()
# Jedes Gericht wird dabei Kategorisiert in:
# - Fisch: Bool
# - Nudeln: Bool
# - Vortag: Bool (Wenn Sonntags viel Bestellt wird das Essen vom Vortag machen)
# - SE: Bool (Sonntagsessen)
# - WE: Bool (Wochenendessen, wie z.B Holen/Bestellen)
# - WE_Wichtung: Float (Jedes Gericht soll die chance haben am WE dran zu kommen, Holen/Bestellen oder z.B. Rolladen sollen bevorzugt werden)
# - Wichtung: Float (Warkeit des Gerichtes ausgewählt zu werden, um doppelte zu vermeiden) 1.0 = Kommt dran; 0 = wird nicht dran kommen
# etc.
# -------------------------------------------------------------------------------
# Tortillas = dict() # Ein Gericht dict()
# Tortillas['Fisch'] = False
# Tortillas['Nudeln'] = False
# Tortillas['Vortag'] = False
# Tortillas['SE'] = False
# Tortillas['WE'] = False
# Tortillas['WE_Wichtung'] = 0.1
# Tortillas['Wichtung'] = 1.0
# Gerichte['Tortillas'] = Tortillas
# -------------------------------------------------------------------------------
'''name_dict = dict()
name_dict['Fisch'] = Fisch
name_dict['Nudeln'] = Nudeln
name_dict['Vortag'] = Vortag
name_dict['SE'] = SE
name_dict['WE'] = WE
name_dict['WE_Wichtung'] = WE_Wichtung
name_dict['Wichtung'] = Wichtung'''
name = entries['Name']
self.Kochbuch[name] = entries
def del_gericht(self,name):
self.Kochbuch.pop(name)
def update_kochbuch(self, Kochbuch: dict, name: str, kategorie: str, value: any):
# Da hier mit dict im dict gearbeitet wird und dasa äußerste dict ein SQLdict ist,
# müssen hier die Einträge ein bisschen umständlich verändert werden, um vom RAM in die SQL datei zu schreiben
update = Kochbuch[name]
update[kategorie] = value
Kochbuch[name] = update
def choose(self,dishes):
choosed_dish = np.asarray(dishes)
Wichtungen = choosed_dish[:,1].astype(np.float)
#choosed_dish = np.sort(choosed_dish)
# Find maximas in Wichtung column
max_indizes = np.where(Wichtungen==np.amax(Wichtungen))
finds = []
for i in max_indizes[0]:
finds.append(choosed_dish[i])
# Choose dish
# If len of finds > 1 use random int to choose dish
if len(finds) > 1:
dish_index = random.randint(0,len(finds) - 1)
return finds[dish_index]
else:
return finds
#print('test','\n',choosed_dish)
def gen_week_table(self):
# generate table of dishes day wise
dishes = [i for i in self.Kochbuch.items()]
dishes_cnt = len(dishes)
usable_dishes = []
possible_dishes_mon = []
possible_dishes_tue = []
possible_dishes_wed = []
possible_dishes_thu = []
possible_dishes_fri = []
possible_dishes_sat = []
possible_dishes_sun = []
saison = 'Winter'
for index, dish in enumerate(dishes):
dish = dish[1]
# Perform standard check, to reduce dishes according to seasons and weigth
#if float(dish['Wichtung']) > 0.7 and dish['Saison'] == saison: # Standard check
if float(dish['Wichtung']) > 0.7 and (dish['Saison'] == saison or dish['Saison'] == 'None'):
usable_dishes.append(dish)
# -----------Monday-------------
# ------------------------------
# Mondays should prefer Nudeln ---> Boni for Nudeln == True
if dish['Fisch'] == 'False':
if dish['Nudeln'] == 'True':
possible_dishes_mon.append([dish['Name'], float(dish['Wichtung']) + 0.3])
else:
possible_dishes_mon.append([dish['Name'], float(dish['Wichtung'])])
#-----------------------------------------------------------------------------
# -----------Tuesday/Wednesday/Thursday-------------
# --------------------------------------------------
# Days without preferations
if dish['Fisch'] == 'False' and dish['Vortag'] == 'False': # Standard check
possible_dishes_tue.append([dish['Name'], float(dish['Wichtung'])])
possible_dishes_wed.append([dish['Name'], float(dish['Wichtung'])])
possible_dishes_thu.append([dish['Name'], float(dish['Wichtung'])])
# -----------------------------------------------------------------------------
# -----------Friday-------------
# ------------------------------
# Fish prefered
if dish['WE'] == 'True' and dish['SE'] == 'False':
if dish['Fisch'] == 'True':
possible_dishes_fri.append([dish['Name'], float(dish['Wichtung']) + 0.3])
else:
possible_dishes_fri.append([dish['Name'], float(dish['Wichtung'])])
# -----------------------------------------------------------------------------
# -----------Saturday-------------
# --------------------------------
# WE category prefered
if dish['Fisch'] == 'False' and dish['SE'] == 'False' and dish['Vortag'] == 'False':
if dish['WE'] == 'True':
possible_dishes_sat.append([dish['Name'], float(dish['Wichtung']) + 0.3])
else:
possible_dishes_sat.append([dish['Name'], float(dish['Wichtung'])])
# -----------------------------------------------------------------------------
# -----------Sunday-------------
# ------------------------------
# SE highly prefered
if dish['Fisch'] == 'False' and dish['Vortag'] == 'False':
if dish['SE'] == 'True':
possible_dishes_sun.append([dish['Name'], float(dish['Wichtung']) + 0.5])
else:
possible_dishes_sun.append([dish['Name'], float(dish['Wichtung'])])
print('============================================================================')
print('=================================Wochenplan=================================')
print('============================================================================')
print('Monday: ', self.choose(possible_dishes_mon)[0])
print('----------------------------------------------------------------------------')
print('Tuesday: ', self.choose(possible_dishes_tue)[0])
print('----------------------------------------------------------------------------')
print('Wednesday: ', self.choose(possible_dishes_wed)[0])
print('----------------------------------------------------------------------------')
print('Thurday: ', self.choose(possible_dishes_thu)[0])
print('----------------------------------------------------------------------------')
print('Friday: ', self.choose(possible_dishes_fri)[0])
print('----------------------------------------------------------------------------')
print('Saturday: ', self.choose(possible_dishes_sat)[0])
print('----------------------------------------------------------------------------')
print('Sunday: ', self.choose(possible_dishes_sun)[0])
print('============================================================================')
#print(self.choose(possible_dishes_mon)[0])
Speiseplan = collections.OrderedDict()
class SQLiteRepository(Repository):
repository_name: str = 'sqlite'
extension: str = '.db'
def __init__(self, repository_path: str, commit_on_close: bool = True,
verbosity: int = 0):
super().__init__(repository_path, commit_on_close=commit_on_close,
verbosity=verbosity)
self.sqlite_repository = None
self.table_name = None
@contextmanager
def connect(self, table_name: str) -> 'SQLiteRepository':
yield self.open(table_name)
self.close()
def open(self, table_name: str):
self.sqlite_repository = SqliteDict(
self.repository_path, tablename=table_name, encode=json.dumps,
decode=json.loads, flag='c')
self.table_name = table_name
return self
def close(self):
if self.sqlite_repository is not None:
if self.commit_on_close:
self.commit()
self.sqlite_repository.close()
self.sqlite_repository = None
self.table_name = None
def commit(self):
self.sqlite_repository.commit()
def keys(self) -> List[str]:
return list(self.sqlite_repository.keys())
def update(self, key: str, update_obj: dict):
self.sqlite_repository[key] = update_obj
def upsert(self, key: str, obj: dict):
self.sqlite_repository[key] = obj
def get(self, key: str) -> dict:
try:
return self.sqlite_repository[key]
except KeyError:
raise InvalidEntryError(key)
def get_multiple(self, keys: List[str]) -> Dict[str, dict]:
values = {key: element for key, element in self.sqlite_repository.items() if key in keys}
if len(set(keys)) != len(values):
invalids = set(keys).difference(values.keys())
raise InvalidEntryError(', '.join(list(invalids)))
return values
def get_all(self) -> Dict[str, dict]:
return {key: element for key, element in self.sqlite_repository.items()}
def remove(self, key: str):
try:
del self.sqlite_repository[key]
except KeyError:
raise InvalidEntryError(key)
def remove_multiple(self, keys: List[str]):
for key in keys:
self.remove(key)
def clear(self):
self.sqlite_repository.clear()
def __repr__(self):
return f"SQLiteRepository(file='{self.repository_path}', open={self.sqlite_repository is not None}, " \
f"table='{self.table_name}')"
"""
sqlitedict支持Multi Thread Safe Write。也就是可以用多线程或是多个脚本对同一个
字典进行写操作。
"""
from __future__ import print_function
import os
import time
import string
import random
from multiprocessing.dummy import Pool
from sqlitedict import SqliteDict
file = "./sqlitedict.sqlite"
mydict = SqliteDict(file, autocommit=True)
mydict.clear()
def multiple_script():
"""多个脚本调用同一个字典。
"""
s = "x" * 10000
st = time.clock()
for i in range(10000):
mydict[str(i)] = s
elapsed = time.clock() - st
print("elapsed %.6f seconds." % elapsed)
assert len(mydict) == 10000
class Broker(object):
def __init__(self, urls, id):
self.id = id
self.peer_urls = urls['peers']
self.queued_msgs = {}
self.replicated_msgs = {}
self.capacity = MAX_MSG_CAPACITY
# self.consume_lock = threading.Lock()
# self.consuming = False
asyncio.set_event_loop(asyncio.new_event_loop())
# self.loop = ioloop.IOLoop.current()
self.__init_metrics()
self.__init_sockets(urls)
self.__set_callback()
self.loop = IOLoop.instance()
self.rocks = SqliteDict(
'./remotedb-%d.sqlite' % (self.id), autocommit=True)
# self.AAL = SqliteDict('./local_aal-%d.sqlite' % (self.id), autocommit=True)
self.AAL={}
# self.DAL = SqliteDict('./local_dal-%d.sqlite' % (self.id), autocommit=True)
self.DAL={}
self.rocks.clear()
# self.AAL.clear()
# self.DAL.clear()
def __init_sockets(self, urls):
context = zmq.Context()
self.frontend_sock = context.socket(zmq.ROUTER)
self.backend_sock = context.socket(zmq.ROUTER)
self.peerend_sock = context.socket(zmq.ROUTER)
self.frontend_sock.bind(urls["producer"])
self.backend_sock.bind(urls["consumer"])
self.peerend_sock.bind(urls["self"])
self.context = zmq.Context.instance()
self.socket = context.socket(zmq.REQ)
self.socket.identity = (u"Broker-%d" % (self.id)).encode('ascii')
def __set_callback(self):
self.frontend = ZMQStream(self.frontend_sock)
self.backend = ZMQStream(self.backend_sock)
self.peerend = ZMQStream(self.peerend_sock)
self.frontend.on_recv(self.handle_frontend)
self.backend.on_recv(self.handle_backend)
self.peerend.on_recv(self.handle_peerend)
def __init_metrics(self):
pass
def __lost_peer(self, url):
pass
def __new_peer_online(self, url):
pass
def __insert_add_ahead_log(self, prod_id, msg_id, dst):
""" Implementation of add-ahead-log (AAL)
"""
if DEBUG == True:
print('Thread', self.id, 'inserting AAL', prod_id, msg_id, dst)
if prod_id not in self.AAL.keys():
self.AAL[prod_id] = {}
self.AAL[prod_id][msg_id] = dst
def __remove_add_ahead_log(self, prod_id, msg_id):
if DEBUG == True:
print("removing AAL", prod_id, msg_id)
del self.AAL[prod_id][msg_id]
def __insert_delete_ahead_log(self, prod_id, msg_id, dst):
""" Implementation of delete-ahead-log (DAL)
"""
if DEBUG == True:
print('Thread', self.id, 'inserting DAL', prod_id, msg_id, dst)
if prod_id not in self.DAL.keys():
self.DAL[prod_id] = {}
self.DAL[prod_id][msg_id] = dst
def __remove_delete_ahead_log(self, prod_id, msg_id):
if DEBUG == True:
print("removing DAL", prod_id, msg_id)
del self.DAL[prod_id][msg_id]
def __send_replicates(self, prod_id, msg_id, payload):
for url in self.peer_urls:
self.socket.connect(url)
self.__insert_add_ahead_log(prod_id, msg_id, url)
self.socket.send_multipart([b'REPLICATE', prod_id, msg_id, payload])
reply = self.socket.recv()
if reply == b'REPLICATED':
self.__remove_add_ahead_log(prod_id, msg_id)
self.socket.disconnect(url)
def __delete_replicates(self, prod_id, msg_id):
""" needs an efficient way to store metadata of all replicates
"""
for url in self.peer_urls:
self.socket.connect(url)
self.__insert_delete_ahead_log(prod_id, msg_id, url)
self.socket.send_multipart([b'DELETE', prod_id, msg_id])
reply = self.socket.recv()
if reply == b'DELETED':
self.__remove_delete_ahead_log(prod_id, msg_id)
self.socket.disconnect(url)
def __persist_msgs(self, prod_id, msg_id, msg):
if prod_id not in self.rocks.keys():
self.rocks[prod_id] = {}
self.rocks[prod_id][msg_id] = msg
def __retrieve_msgs(self, num=1):
if len(list(self.rocks.keys())) == 0:
return
for _ in range(num):
prod_id = random.choice(list(self.rocks.keys()))
msg_id, msg = self.rocks[prod_id].popitem()
self.queued_msgs[prod_id][msg_id] = msg
def __get_size_of_queues(self, replica=False):
return sum([len(queue) for queue in self.queued_msgs.values()])
def __enqueue_a_msg(self, prod_id, msg_id, payload, replica=False):
if not replica:
if self.__get_size_of_queues() <= self.capacity * PERSIST_THRESHOLD:
if prod_id not in self.queued_msgs.keys():
self.queued_msgs[prod_id] = {}
self.queued_msgs[prod_id][msg_id] = payload
else:
self.__persist_msgs(prod_id, msg_id, payload) # sorta redundant
else:
if prod_id not in self.replicated_msgs.keys():
self.replicated_msgs[prod_id] = {}
self.replicated_msgs[prod_id][msg_id] = payload
def __get_a_msg(self):
try:
prod_id = random.choice(list(self.queued_msgs.keys()))
msg_id = random.choice(list(self.queued_msgs[prod_id].keys()))
# msg_id, payload = self.queued_msgs[prod_id].popitem()
return [prod_id, msg_id, self.queued_msgs[prod_id][msg_id]]
except IndexError:
return [None, None, None]
def __dequeue_a_msg(self, prod_id, msg_id, replica=False):
if not replica:
if self.__get_size_of_queues() == self.capacity * PERSIST_THRESHOLD:
self.__retrieve_msgs()
if DEBUG == True:
print(self.id, "dequing:", self.queued_msgs[prod_id])
try:
return self.queued_msgs[prod_id].pop(msg_id)
except IndexError:
if DEBUG == True:
print("Already consumed. (Delivered twice)")
return None
else:
if DEBUG == True:
print(self.id, "dequing replica:", self.replicated_msgs[prod_id])
try:
return self.replicated_msgs[prod_id].pop(msg_id)
except KeyError:
return None
def __consume_a_msg(self, cons_id):
prod_id, msg_id, payload = self.__get_a_msg()
if prod_id is None:
next_timer = threading.Timer(NEXT_CONSUME_TIME, self.__consume_a_msg, [cons_id])
next_timer.start()
return
else:
pass
# print('get_a_msg', prod_id, msg_id, payload)
self.backend.send_multipart([cons_id, prod_id, msg_id, payload])
def handle_frontend(self, msg):
if DEBUG == True:
print(self.id, 'handling frontend:', msg)
start_time = time.time_ns()
prod_id, msg_id, payload = msg[:3]
self.__enqueue_a_msg(prod_id, msg_id, payload)
self.__send_replicates(prod_id, msg_id, payload)
# print("Time for frontend is", time.time_ns() - start_time)
def handle_backend(self, msg):
if DEBUG == True:
print(self.id, 'handling backend:', msg)
cons_id, command = msg[:2]
start_time = time.time_ns()
if command == b'CONSUME':
start_time = time.time_ns()
self.__consume_a_msg(cons_id)
elif command == b'CONSUMED':
prod_id, msg_id = msg[2:4]
res = self.__dequeue_a_msg(prod_id, msg_id)
if res is not None:
self.__delete_replicates(prod_id, msg_id)
# print("Time for backend is", time.time_ns() - start_time)
def handle_peerend(self, msg):
if DEBUG == True:
print(self.id, 'handling peerend', msg)
broker_id, _, command = msg[:3]
start_time = None
if command == b'REPLICATE':
start_time = time.time_ns()
prod_id, msg_id, payload = msg[3:6]
self.__enqueue_a_msg(prod_id, msg_id, payload, replica=True)
# self.peerend.send_multipart([broker_id, b'', b'REPLICATED', prod_id, msg_id])
self.peerend.send_multipart([broker_id, b'', b'REPLICATED'])
# print("Time for replication is", time.time_ns() - start_time)
elif command == b'DELETE':
start_time = time.time_ns()
prod_id, msg_id = msg[3:5]
self.__dequeue_a_msg(prod_id, msg_id, replica=True)
# self.peerend.send_multipart([broker_id, b'', b'DELETED', prod_id, msg_id])
self.peerend.send_multipart([broker_id, b'', b'DELETED'])
# print("Time for deletion is", time.time_ns() - start_time)
# elif command == b"REPLICATED":
# self.__remove_add_ahead_log(prod_id, msg_id)
# elif command == b"DELETED":
# self.__remove_delete_ahead_log(prod_id, msg_id)
def run(self):
self.loop.start()
def stop(self):
self.loop.stop()
