forked from bitdagger/mtg-scanner
/
referencedb.py
261 lines (230 loc) · 9.13 KB
/
referencedb.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
from __future__ import print_function
import os
import sqlite3
import sys
import urllib
import json
import phash
import time
from progressbar import Bar, Counter, ETA, Percentage, ProgressBar
from datetime import datetime as dt
from mtgexception import MTGException
"""Reference DB module
This module is responsible for handling all the reference data
"""
DateLimit = "2003-10-02" # Only use sets from Mirrodin and after (Modern legal)
class MTG_Reference_DB:
"""Attributes:
PATH (string): Path of the reference database file
JSON_URL (string): URL to the card data json file
IMAGE_URL (string): URL to the card image on the gatherer
IMAGE_FILE (string): Path to local image files
connection (sqlite): Established sqlite connection
"""
PATH = 'reference.db'
JSON_URL = 'http://mtgjson.com/json/AllSets.json'
IMAGE_URL = 'http://gatherer.wizards.com/Handlers/Image.ashx' + \
'?multiverseid=%d&type=card'
IMAGE_FILE = 'img/%d.jpg'
def __init__(self):
try:
self.connection = sqlite3.connect(self.PATH)
except sqlite3.Error, e:
print("Error %s:" % e.args[0])
sys.exit(1)
def check_rebuild(self):
"""Check to see if the reference db needs to be rebuilt
"""
rebuild = False
try:
cursor = self.connection.cursor()
cursor.execute("""SELECT name FROM sqlite_master
WHERE type='table';""")
data = cursor.fetchall()
if (not len(data)):
rebuild = True
else:
if (
((u'Sets',) not in data) or ((u'Cards',) not in data) or
((u'Hashes',) not in data)
):
rebuild = True
else:
cursor.execute("""SELECT
(SELECT COUNT(*) FROM Sets) as SetCount,
(SELECT COUNT(*) FROM Cards) as CardCount,
(SELECT COUNT(*) FROM Hashes) as HashCount""")
data = cursor.fetchone()
if (not data[0] or not data[1] or not data[2]):
rebuild = True
elif (data[1] != data[2]):
rebuild = True
else:
cursor.execute("SELECT MultiverseID FROM Cards")
data = cursor.fetchall()
for card in data:
if (not os.path.isfile(self.IMAGE_FILE % card[0])):
rebuild = True
except sqlite3.Error, e:
print("Error %s:" % e.args[0])
sys.exit(1)
return rebuild
def import_cards(self):
"""Import card data from MTGJSON
"""
print('Fetching card data...')
data = json.load(urllib.urlopen(self.JSON_URL))
try:
cursor = self.connection.cursor()
cursor.execute("""CREATE TABLE IF NOT EXISTS Sets
(ID INTEGER PRIMARY KEY,
Name TEXT NOT NULL,
Code TEXT NOT NULL)""")
cursor.execute("""CREATE TABLE IF NOT EXISTS Cards (
MultiverseID INTEGER NOT NULL PRIMARY KEY,
SetID INTEGER NOT NULL,
Name TEXT NOT NULL)""")
cursor.execute("DELETE FROM Sets")
cursor.execute("DELETE FROM Cards")
self.connection.commit()
except sqlite3.Error, e:
self.connection.rollback()
print("Error %s:" % e.args[0])
sys.exit(1)
print('Importing card data...')
dateLimit = dt.strptime(DateLimit, "%Y-%m-%d")
try:
for setid in data:
UseSet = True
if (
(data[setid]['type'] != 'expansion' and
data[setid]['type'] != 'core')
):
UseSet = False
else:
date = data[setid].get('releaseDate', 'ERR')
if (
date == 'ERR' or
dt.strptime(date, "%Y-%m-%d") < dateLimit
):
UseSet = False
if (UseSet):
code = data[setid].get(
'gathererCode',
data[setid].get('code', 'ERR')
)
cursor.execute("""SELECT ID FROM Sets
WHERE Code = ?""", (code,))
r = cursor.fetchone()
if (r is None):
cursor.execute("""INSERT INTO Sets (Name, Code)
VALUES(?, ?)""", (data[setid]['name'], code))
sid = cursor.lastrowid
else:
sid = r[0]
for card in data[setid]['cards']:
cursor.execute("""SELECT * FROM Cards WHERE
MultiverseID = ?""", (str(card['multiverseid']),))
if (cursor.fetchone() is None):
cursor.execute(
"""INSERT INTO Cards
(MultiverseID, SetID, Name) VALUES(?, ?, ?)""",
(card['multiverseid'], sid, card['name'])
)
self.connection.commit()
except sqlite3.Error, e:
self.connection.rollback()
print("Error %s:" % e.args[0])
sys.exit(1)
def download_images(self):
"""Download images from the gatherer
"""
print('Downloading images...')
try:
cursor = self.connection.cursor()
cursor.execute("SELECT MultiverseID FROM Cards")
cards = cursor.fetchall()
if (len(cards)):
pbar = ProgressBar(
widgets=[
Percentage(), ': ', Counter(),
'/' + str(len(cards)) + ' ', Bar(), ' ', ETA()
]
)
for card in pbar(cards):
MultiverseID = card[0]
path = self.IMAGE_FILE % MultiverseID
if (not os.path.isfile(path)):
urllib.urlretrieve(self.IMAGE_URL % MultiverseID, path)
os.system(
"convert " + path +
" -background black -flatten " + path)
except sqlite3.Error, e:
self.connection.rollback()
print("Error %s:" % e.args[0])
sys.exit(1)
def calculate_hashes(self):
"""Calculate the hashes for all the cards
"""
print('Calculating hashes...')
try:
cursor = self.connection.cursor()
cursor.execute("""CREATE TABLE IF NOT EXISTS Hashes
(MultiverseID INTEGER NOT NULL PRIMARY KEY,
Hash TEXT NOT NULL)""")
cursor.execute("DELETE FROM Hashes")
self.connection.commit()
cursor.execute("SELECT MultiverseID FROM Cards")
cards = cursor.fetchall()
if (len(cards)):
pbar = ProgressBar(
widgets=[
Percentage(), ' ', Bar(), ' ', ETA()
]
)
for card in pbar(cards):
MultiverseID = card[0]
path = self.IMAGE_FILE % MultiverseID
cursor.execute("""SELECT * FROM Hashes WHERE
MultiverseID = ?""", (MultiverseID,))
if (cursor.fetchone() is None):
ihash = phash.dct_imagehash(path)
cursor.execute(
"""INSERT INTO Hashes
(MultiverseID, Hash) VALUES(?, ?)""",
(MultiverseID, str(ihash))
)
self.connection.commit()
except sqlite3.Error, e:
self.connection.rollback()
print("Error %s:" % e.args[0])
sys.exit(1)
def get_hashes(self):
"""Get the list of hashes
"""
hashes = {}
try:
cursor = self.connection.cursor()
cursor.execute("SELECT MultiverseID, Hash FROM Hashes")
for row in cursor.fetchall():
hashes[row[0]] = row[1]
return hashes
except sqlite3.Error, e:
print("Error %s:" % e.args[0])
sys.exit(1)
def get_card_info(self, MultiverseID):
"""Get the name and set code for a single card
"""
try:
cursor = self.connection.cursor()
cursor.execute("""SELECT C.Name, S.Code FROM Cards C
LEFT JOIN Sets S on S.ID = C.SetID
WHERE C.MultiverseID = ?""", (MultiverseID,))
r = cursor.fetchone()
if (r is None):
raise MTGException('No such card')
return r[0], r[1]
except sqlite3.Error, e:
self.connection.rollback()
print("Error %s:" % e.args[0])
sys.exit(1)