def lineify_fileobjs(ifo, ofo, strip=False): from pyutil.strutil import pop_trailing_newlines, split_on_newlines for l in ifo: for sl in split_on_newlines(pop_trailing_newlines(l)): if strip: sl = sl.strip() ofo.write(pop_trailing_newlines(sl) + '\n')
def _find_missing_names(self, fname):
f = open(fname, 'r')
names = dictutil.NumDict()
for line in f.xreadlines():
mo = CARDNAME_FIELD_RE.match(strutil.pop_trailing_newlines(line))
if mo:
if mo.group(5):
names.add_num(_fixnames(mo.group(2)), int(mo.group(5)))
else:
names.inc(_fixnames(mo.group(2)))
for k, v, in names.items():
if v != 1:
print "k: %s, v: %s" % (k, v,)
for card in self.cards():
if card['Card Name'] not in names:
print "%s in db and not in file" % card['Card Name']
for name in names.keys():
if not self.has_key(name):
print "%s in file and not in db" % name
def test_short_input(self):
self.assertTrue(strutil.pop_trailing_newlines("\r\n") == "")
self.assertTrue(strutil.pop_trailing_newlines("\r") == "")
self.assertTrue(strutil.pop_trailing_newlines("x\r\n") == "x")
self.assertTrue(strutil.pop_trailing_newlines("x\r") == "x")
def test_short_input(self):
self.failUnless(strutil.pop_trailing_newlines("\r\n") == "")
self.failUnless(strutil.pop_trailing_newlines("\r") == "")
self.failUnless(strutil.pop_trailing_newlines("x\r\n") == "x")
self.failUnless(strutil.pop_trailing_newlines("x\r") == "x")
def import_list(self, fname):
"""
This reads in a spoiler list in any currently known Wizards of the Coast text format and populates self.
"""
f = open(fname, 'r')
id2cs = dictutil.UtilDict() # k: tuple of (set name, card number,), v: list of card objects
setname = None
cardstotal = None # a self-check to see if we got them all
namesfound = dictutil.NumDict() # a self-check to see if we got them all
incard = false
thiscard = None
thiskey = None
thisval = None
prevline = None # just for debugging
for line in f.xreadlines():
print line
line = strutil.pop_trailing_newlines(line)
mo = CARDNAME_FIELD_RE.match(line)
if mo:
if mo.group(5):
namesfound.add_num(_fixnames(mo.group(2)), int(mo.group(5)))
else:
namesfound.inc(_fixnames(mo.group(2)))
incard = true
thiscard = Card()
if setname:
thiscard['Set Name'] = setname
if incard:
if len(line) == 0:
if thiskey == 'Rarity':
incard = false
thiscard[thiskey] = thisval
elif (line[0] in string.whitespace) or ((line.find(":") == -1) and (line.find("\t") == -1) and (line.find(" ") == -1) and ((thisval and len(thisval) > 50) or (thiskey == 'Flavor Text') or ((line[0] in string.ascii_uppercase) and (thiskey == 'Card Text')))) or (thisval and (thisval.strip()[0] == '"') and (thisval.strip()[-1] != '"') and (thiskey == 'Card Text') and (line.find("Flavor Text:") == -1)):
# Some spoiler lists have typos in which continued lines don't start with white space. We'll assume that if this line doesn't have a separator (none of ':', '\t', or ' '), *and* if the previous line was more than 50 chars, that this is one of those typos.
# Some spoiler lists have typos in which attributions of quotes start on the line after the quote (in a Flavor Text field) but have no start with white space. We'll assume that if this line begins with '-', doesn't have a separator (none of ':', '\t', or ' '), *and* if the previous line was a Flavor Text field, then this is one of those typos.
# Some older spoiler lists have linebreaks between lines of Card Text. We'll assume that if the previous line was a Card Text field, and this line doesn't have a separator (none of ':', '\t', or ' '), *and* if the first character of the line is a capital letter, that this is one of those. In addition, we'll assume that if the previous line was a Card Text field, and it began with a quote character but didn't end with one, that this is one of those.
# Some spoiler lists (e.g. le_spoiler_en.txt in Chromeshell Crab) have typos where multilines are begun with a double-quote but not terminated with a double-quote! So if the line contains "Flavor Text:" then we don't follow the "keep going til you find a double-quote since you started with a double-quote" rule.
assert thiskey is not None
assert thisval is not None
thisval += ' ' + line.strip()
elif (line.find(":") == -1) and (line.find("\t") == -1) and (line.find(" ") == -1) and (line[0] in string.ascii_uppercase) and (not thiscard.has_key('Card Text')):
# Some spoiler lists (e.g. "Dry Spell" in homelands.txt), have typos where the "Card Text:" param name is missing! We'll assume that if this line doesn't have a separator (none of ':', '\t', or ' '), *and* none of the previous assumptions have been taken, *and* if the first character of the line is a capital letter, *and* if there has not been any 'Card Text' field processed for this card yet, then this is one of those.
if thiskey:
self._process_key_and_val(thiskey, thisval, thiscard)
thiskey = None
thisval = None
thiskey = 'Card Text'
thisval = line
elif RARITY_RE.match(strip_whitespace_and_quotes(line)):
# Some spoiler lists (e.g. "Gaea's Touch" in the_dark.txt), have typos where the "Rarity:" param name is missing! We'll assume that if this val matches the RARITY_RE then this is one of those.
if thiskey:
self._process_key_and_val(thiskey, thisval, thiscard)
thiskey = None
thisval = None
thiskey = 'Rarity'
thisval = line
else:
self._process_key_and_val(thiskey, thisval, thiscard)
thiskey = None
thisval = None
# on to the next key and val!
sepindex = line.find(':')
if sepindex == -1:
# Some spoiler lists have typos in which the ':' is missing... So we'll break on the first occurence of two-spaces or a tab.
twospaceindex = line.find(' ')
tabindex = line.find('\t')
if (twospaceindex != -1) and ((twospaceindex <= tabindex) or (tabindex == -1)):
sepindex = twospaceindex
elif (tabindex != -1) and ((tabindex <= twospaceindex) or (twospaceindex == -1)):
sepindex = tabindex
assert sepindex != -1, "line: %s, line.find('\t'): %s, line.find(' '): %s" % (`line`, line.find('\t'), line.find(' '),)
thiskey = strip_whitespace_and_quotes(line[:sepindex])
thisval = line[sepindex+1:].strip()
if thiskey == "Card #":
id2cs.setdefault((thiscard['Set Name'], thisval,), []).append(thiscard)
incard = false
thiscard[thiskey] = thisval
elif (not setname) and SPOILER_NAME_RE.match(line):
assert (SPOILER_NAME_RE.match(line).group(1) is None) != (SPOILER_NAME_RE.match(line).group(2) is None)
if SPOILER_NAME_RE.match(line).group(1):
setname = SPOILER_NAME_RE.match(line).group(1)
else:
setname = SPOILER_NAME_RE.match(line).group(2)
elif CARDS_TOTAL_RE.match(line):
cardstotal = int(CARDS_TOTAL_RE.match(line).group(1))
prevline = line # just for debugging
if setname == "Antiquities":
cardstotal -= 1 # error in antiquities.txt cardstotal, AFAICT
if setname == "Urza's Legacy":
# whoops -- this one is in a completely different format.
self.import_urzas_legacy_spoiler(fname)
# Okay now fix up any obsolete or inconsistent bits.
map(Card._update, self.cards())
# Okay now merge any "double cards" into a single entry.
for ((setname, cardnum,), cs,) in id2cs.items():
if len(cs) > 1:
assert len(cs) == 2, "len(cs): %s, cs: %s" % (len(cs), cs,)
one = cs[0]
two = cs[1]
subone, subtwo = one['Card Name'], two['Card Name']
for thiscard in (one, two,):
assert thiscard['Card Name'], thiscard.data
thiscard['This Card Name'] = thiscard['Card Name']
mo = DOUBLE_CARD_NAME_RE.match(thiscard['Card Name'])
if mo is not None:
subone, subtwo, thisone = mo.groups()
thiscard['This Card Name'] = thisone
if subone != one['This Card Name']:
temp = one
one = two
two = temp
newname = one['This Card Name'] + "/" + two['This Card Name']
assert not self.has_key(newname), "newname: %s" % newname
newc = Card(one)
newc["Card Name"] = newname
for k in ("Mana Cost", "Card Color", "Type & Class", "Artist",):
if one.get(k, "") == two.get(k, ""):
if one.has_key(k):
newc[k] = one[k]
else:
newc[k] = one.get(k,'') + "/" + two.get(k,'')
if one.get("Card Text", "") == two.get("Card Text", ""):
if one.has_key("Card Text"):
newc["Card Text"] = one["Card Text"]
else:
newc["Card Text"] = one['This Card Name'] + " -- " + one.get("Card Text",'') + "\n" + two['This Card Name'] + " -- " + two.get("Card Text",'')
del self[one['Card Name']]
try:
del self[two['Card Name']]
except:
print two.data
print two
raise
del newc['This Card Name']
self[newname] = newc
print "cardstotal: %s, after merging, we had %s" % (cardstotal, len(self.cards()),)
if (cardstotal is not None) and (cardstotal != len(self.cards())) and ((cardstotal != namesfound.sum()) or (len(self.cards()) != len(namesfound))):
print "after merge, cardstotal: %s, namesfound.sum(): %s, len(namesfound): %s, len(self.cards()): %s" % (cardstotal, namesfound.sum(), len(namesfound), len(self.cards()),)
