def load_thread_gatherer(self):
all = []
all_num = util.get_all_cards_num()
all_pages = int(math.ceil(all_num / 100))
# Paging for ui control between downloads
for i in range(all_pages):
req_start = time.time()
try:
new_cards = Card.where(page=i).where(pageSize=100).all()
except MtgException as err:
util.log(str(err), util.LogLevel.Error)
return
all = all + new_cards
req_end = time.time()
# Check if the action was canceled during download
if self.cancel_token:
GObject.idle_add(self.download_canceled)
return
# Activate download UI
self.app.ui.get_object("dl_spinner").set_visible(False)
self.app.ui.get_object("dl_progress_bar").set_visible(True)
self.app.ui.get_object("dl_progress_label").set_visible(True)
passed = str(round(req_end - req_start, 3))
GObject.idle_add(self.load_update_ui, all, all_num, passed)
return all
def db_save_changes(self):
try:
self.connection.commit()
except sqlite3.Error as err:
self.connection.rollback()
util.log("Database Error", util.LogLevel.Error)
util.log(str(err), util.LogLevel.Error)
def wants_card_remove(self, card: mtgsdk.Card, list: str):
"""Remove a card from a wants list"""
l = self.wants[list]
l.remove(card)
self.db.wants_card_remove(list, card.multiverse_id)
util.log("Removed '{}' from wants list '{}'".format(card.name, list),
util.LogLevel.Info)
def download_finished(self):
"""Download thread finished without errors"""
self.cancel_token = False
self.app.set_online(False)
self.app.ui.get_object("loadDataDialog").hide()
self.app.push_status("Card data downloaded")
util.log("Card data download finished", util.LogLevel.Info)
def do_cancel_download(self, item: Gtk.MenuItem):
"""The cancel button was pressed, set cancel_token to stop download thread"""
self.cancel_token = True
# Delete Dialog
self.app.ui.get_object("loadDataDialog").hide()
self.app.push_status("Download canceled")
util.log("Download canceled by user", util.LogLevel.Info)
def db_operation(self, sql: str, args: tuple = ()):
"""Perform an arbitrary sql operation on the database"""
cur = self.connection.cursor()
try:
cur.execute(sql, args)
except sqlite3.OperationalError as err:
util.log("Database Error", util.LogLevel.Error)
util.log(str(err), util.LogLevel.Error)
def wants_rename(self, old: str, new: str):
if old == new:
return
self.wants[new] = self.wants[old]
del self.wants[old]
self.db.wants_rename(old, new)
util.log("Want List '" + old + "' renamed to '" + new + "'",
util.LogLevel.Info)
def tag_rename(self, old, new):
if old == new:
return
self.tags[new] = self.tags[old]
del self.tags[old]
self.db.tag_rename(old, new)
util.log("Tag '" + old + "' renamed to '" + new + "'",
util.LogLevel.Info)
def save_data(self):
util.log("Saving Data to database", util.LogLevel.Info)
start = time.time()
self.db.db_save_changes()
end = time.time()
util.log("Finished in {}s".format(str(round(end - start, 3))),
util.LogLevel.Info)
self.push_status("All data saved.")
pass
def lib_card_add_bulk(self, cards: list, tag: str = None):
for card in cards:
if tag is not None:
self.tag_card(card, tag)
self.db.tag_card_add(tag, card.multiverse_id)
self.lib_card_add(card, tag)
self.db.lib_card_add(card)
util.log("Added {} cards to library.".format(str(len(cards))),
util.LogLevel.Info)
self.push_status("Added {} cards to library.".format(str(len(cards))))
def lib_new_tag_and_add(self, item, cards):
response = self.app.show_name_enter_dialog("Enter name for new Tag",
"")
if not response == "":
self.app.tag_new(response)
self.tag_cards(cards, response)
else:
util.log("No tag name entered", util.LogLevel.Warning)
self.app.push_status("No name for new tag entered")
self.reload_library(self.active_tag)
def load_user_data(self):
util.log(
"Loading User Data from form '{}'".format(
util.get_root_filename(util.DB_NAME)), util.LogLevel.Info)
start = time.time()
self.library = self.db.lib_get_all()
self.tags = self.db.tag_get_all()
self.wants = self.db.wants_get_all()
end = time.time()
util.log("Finished in {}s".format(str(round(end - start, 3))),
util.LogLevel.Info)
self.push_status("All data loaded.")
def get_mana_icons(self, mana_string):
if not mana_string:
util.log("No mana string provided", util.LogLevel.Info)
return
icon_list = re.findall("{(.*?)}", mana_string.replace("/", "-"))
icon_name = "_".join(icon_list)
try:
icon = self.precon_icons[icon_name]
except KeyError:
icon = util.create_mana_icons(self.mana_icons, mana_string)
self.precon_icons[icon_name] = icon
return icon
def new_wants_and_add(self, menu_item):
# Get selected cards
card_list = self.app.ui.get_object("searchResults").get_child()
cards = card_list.get_selected_cards()
response = self.app.show_name_enter_dialog("Enter name for new Want List", "")
if not response == "":
self.app.wants_new(response)
for card in cards.values():
self.app.wants_card_add(response, card)
else:
util.log("No list name entered", util.LogLevel.Warning)
self.app.push_status("No name for new wants list entered")
self.reload_search_view()
def new_tag_and_add(self, menu_item):
# Get selected cards
card_list = self.app.ui.get_object("searchResults").get_child()
cards = card_list.get_selected_cards()
response = self.app.show_name_enter_dialog("Enter name for new Tag", "")
if not response == "":
self.app.tag_new(response)
for card in cards.values():
self.app.lib_card_add(card, response)
else:
util.log("No tag name entered", util.LogLevel.Warning)
self.app.push_status("No name for new tag entered")
self.reload_search_view()
def do_card_data_user(self, menu_item):
"""
Handler for Clear User Data menu item.
"""
response = self.app.show_dialog_yn(
"Deleting All User Data",
"You are about to delete all data in the "
"library.\nThis can not be undone.\nProceed?")
if response == Gtk.ResponseType.YES:
util.log("Deleting all local card data", util.LogLevel.Info)
self.app.db_delete_user_data()
util.log("Done", util.LogLevel.Info)
self.app.push_status("Library deleted")
def lib_card_remove(self, card):
# Check if card is tagged
is_tagged, tags = self.db.tag_card_check_tagged(card)
if is_tagged:
for tag in tags:
self.tags[tag].remove(card.multiverse_id)
self.db.tag_card_remove(tag, card.multiverse_id)
del self.library[card.multiverse_id]
self.db.lib_card_remove(card)
util.log("Removed {} from library".format(card.name),
util.LogLevel.Info)
self.push_status(card.name + " removed from library")
def do_card_data_card(self, item):
"""Handler for Clear Card Data menu item"""
response = self.app.show_dialog_yn(
"Deleting All Card Data",
"You are about to delete all local card data.\n"
"Further searches will use the internet to search "
"for cards.\nProceed?")
if response == Gtk.ResponseType.YES:
util.log("Deleting all library data", util.LogLevel.Info)
self.app.db_delete_card_data()
util.log("Done", util.LogLevel.Info)
self.app.push_status(
"Local card data deleted. Switching to online mode.")
def db_insert_data_card(self, cards_json):
"""Insert download from mtgjson"""
c_rows = []
s_rows = []
for data in cards_json.values():
cards = []
for raw in data["cards"]:
c = Card(raw)
c.image_url = util.CARD_IMAGE_URL.format(c.multiverse_id)
c.set = data["code"]
c.set_name = data["name"]
cards.append(c)
for c in cards:
c_rows.append(self.card_to_table_mapping(c))
set = Set(data)
s_rows.append(self.set_to_table_mapping(set))
# Use separate connection to commit changes immediately
con = sqlite3.connect(self.db_file)
try:
with con:
sql_string = "INSERT INTO `cards` VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?," \
"?,?,?,?,?,?,?,?,?,?,?)"
con.executemany(sql_string, c_rows)
sql_string = "INSERT INTO `sets` VALUES (?,?,?,?,?,?,?,?,?,?,?)"
con.executemany(sql_string, s_rows)
except sqlite3.OperationalError as err:
util.log("Database Error", util.LogLevel.Error)
util.log(str(err), util.LogLevel.Error)
except sqlite3.IntegrityError as err:
util.log("Database Error", util.LogLevel.Error)
util.log(str(err), util.LogLevel.Error)
def load_thread(self):
"""Worker thread to download info using the mtgsdk"""
# Gatherer uses rate limit on Card.all()
# Takes ~10 minutes to download all cards
# all = self.load_thread_gatherer()
# Download from mtgjson.com
GObject.idle_add(self.load_show_insert_ui, "Downloading...")
# Waiting in case a canceled thread is still running.
while self.cancel_token:
continue
util.log("Starting download", util.LogLevel.Info)
s = time.time()
all_json = util.net_all_cards_mtgjson()
e = time.time()
util.log("Finished in {}s".format(round(e - s, 3)), util.LogLevel.Info)
if self.cancel_token:
GObject.idle_add(self.download_canceled)
return
self.app.db_delete_card_data()
GObject.idle_add(self.load_show_insert_ui, "Saving data to disk...")
util.log("Saving to sqlite", util.LogLevel.Info)
s = time.time()
GObject.idle_add(self.app.db.db_insert_data_card, all_json)
e = time.time()
util.log("Finished in {}s".format(round(e - s, 3)), util.LogLevel.Info)
self.download_finished()
def db_card_insert(self, card: Card):
"""Insert single card data into database"""
# Use own connection so that inserts are commited directly
con = sqlite3.connect(self.db_file)
try:
with con:
# Map card object to database tables
db_values = self.card_to_table_mapping(card)
sql_string = "INSERT INTO `cards` VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?," \
"?,?,?,?,?,?,?,?,?,?,?)"
# Insert into database
con.execute(sql_string, db_values)
except sqlite3.OperationalError as err:
util.log("Database Error", util.LogLevel.Error)
util.log(str(err), util.LogLevel.Error)
except sqlite3.IntegrityError:
pass
def load_update_ui(self, current_list: list, max_cards: int,
time_passed: str):
"""Called from withing the worker thread. Updates the download dialog with infos."""
# Get info widgets
info_label = self.app.ui.get_object("dl_info_label")
progress_label = self.app.ui.get_object("dl_progress_label")
bar = self.app.ui.get_object("dl_progress_bar")
# Compute numbers for display
size_human = util.sizeof_fmt(sys.getsizeof(current_list))
size_bytes = sys.getsizeof(current_list)
percent = len(current_list) / max_cards
# Update UI
info_label.set_text("Downloading Cards...")
progress_label.set_text("{:.1%} ({})".format(percent, size_human))
bar.set_fraction(percent)
util.log(
"Downloading: {:.1%} | {} Bytes | {}s".format(
percent, size_bytes, time_passed), util.LogLevel.Info)
def db_clear_data_card(self):
"""Delete all resource data from database"""
con = sqlite3.connect(self.db_file)
try:
with con:
con.execute("DELETE FROM cards")
con.execute("DELETE FROM sets")
except sqlite3.OperationalError as err:
util.log("Database Error", util.LogLevel.Error)
util.log(str(err), util.LogLevel.Error)
except sqlite3.IntegrityError as err:
util.log("Database Error", util.LogLevel.Error)
util.log(str(err), util.LogLevel.Error)
def do_download_card_data(self, item: Gtk.MenuItem):
"""Download button was pressed in the menu bar. Starts a thread to load data from the internet"""
info_string = "Start downloading card information from the internet?\n" \
"You can cancel the download at any point."
response = self.app.show_dialog_yn("Download Card Data", info_string)
if response == Gtk.ResponseType.NO:
return
# Launch download info dialog
dl_dialog = self.app.ui.get_object("loadDataDialog")
dl_dialog.set_transient_for(self.app.ui.get_object("mainWindow"))
dl_dialog.show()
# Hide Progress UI until download started
self.app.ui.get_object("dl_progress_bar").set_visible(False)
self.app.ui.get_object("dl_progress_label").set_visible(False)
# Create and start the download in a separate thread so it will not block the UI
thread = threading.Thread(target=self.load_thread)
thread.daemon = True
thread.start()
util.log("Attempt downloading all cards. This may take a while...",
util.LogLevel.Info)
def update(self, library: Dict[str, dict]):
self.store.clear()
if library is None:
return
self.lib = library
# Disable update if tree is filtered (performance)
if self.filtered:
self.tree.freeze_child_notify()
util.log("Updating tree view", util.LogLevel.Info)
start = time.time()
all_wants = self.app.get_wanted_card_ids()
for card in library.values():
if card['multiverse_id'] is not None:
color = self.get_row_color(card, self.app.library, all_wants, self.row_colors)
mana_cost = None
if not card.get('types').__contains__("Land"):
mana_cost = self.app.get_mana_icons(card.get('mana_cost'))
item = [card['multiverse_id'],
card['name'],
" ".join(card.get('supertypes') or ""),
" ".join(card.get('types') or ""),
card.get('rarity'),
card.get('power'),
card.get('toughness'),
", ".join(card.get('printings') or ""),
mana_cost,
card.get('cmc'),
card.get('set_name'),
color,
card.get('original_text')]
self.store.append(item)
end = time.time()
util.log("Time to build Table: " + str(round(end - start, 3)) + "s", util.LogLevel.Info)
util.log("Total entries: " + str(len(self.lib)), util.LogLevel.Info)
# Reactivate update for filtered trees
if self.filtered:
self.tree.thaw_child_notify()
def db_override_user_data(self):
"""Called after import of user data. Overrides existing user data in database"""
util.log("Clearing old user data", util.LogLevel.Info)
self.db.db_clear_data_user()
util.log("Attempt loading user data to database", util.LogLevel.Info)
start = time.time()
# Library
for card in self.library.values():
self.db.lib_card_add(card)
# Tags
for tag, card_ids in self.tags.items():
self.db.tag_new(tag)
for card_id in card_ids:
self.db.tag_card_add(tag, card_id)
# Wants
for list_name, cards in self.wants.items():
self.db.wants_new(list_name)
for card in cards:
self.db.wants_card_add(list_name, card.multiverse_id)
end = time.time()
util.log("Finished in {}s".format(str(round(end - start, 3))),
util.LogLevel.Info)
self.push_status("User data imported")
def db_delete_card_data(self):
"""Called before before rebuilding local data storage"""
util.log("Clearing local card data", util.LogLevel.Info)
self.db.db_clear_data_card()
self.set_online(True)
util.log("Done", util.LogLevel.Info)
def download_canceled(self):
"""The download thread was canceled and finished executing"""
self.cancel_token = False
util.log("Download thread ended", util.LogLevel.Info)
def db_delete_user_data(self):
"""Delete all user data"""
util.log("Clearing all user data", util.LogLevel.Info)
self.db.db_clear_data_user()
util.log("Done", util.LogLevel.Info)
def search_cards(self, term: str, filters: dict) -> dict:
"""Return a dict of cards based on a search term and filters"""
cards = {}
# Check if a local database can be used for searching
if self.app.config["local_db"]:
util.log("Starting local search for '" + term + "'", util.LogLevel.Info)
start = time.time()
cards = self.app.db.search_by_name_filtered(term, filters, 100)
end = time.time()
util.log("Card info fetched in {}s".format(round(end - start, 3)), util.LogLevel.Info)
else:
util.log("Starting online search for '" + term + "'", util.LogLevel.Info)
util.log("Used Filters: " + str(filters), util.LogLevel.Info)
# Load card info from internet
try:
util.log("Fetching card info ...", util.LogLevel.Info)
start = time.time()
cards = Card.where(name=term) \
.where(colorIdentity=",".join(filters["mana"])) \
.where(types=filters["type"]) \
.where(set=filters["set"]) \
.where(rarity=filters["rarity"]) \
.where(pageSize=50) \
.where(page=1).all()
cards = [card.__dict__ for card in cards]
end = time.time()
util.log("Card info fetched in {}s".format(round(end - start, 3)), util.LogLevel.Info)
except (URLError, HTTPError) as err:
util.log(err, util.LogLevel.Error)
return {}
if not self.app.config["show_all_in_search"]:
cards = self._remove_duplicates(cards)
if len(cards) == 0:
# TODO UI show no cards found
util.log("No Cards found", util.LogLevel.Info)
return {}
util.log("Found " + str(len(cards)) + " cards", util.LogLevel.Info)
return {card['multiverse_id']: card for card in cards}
