def lineify_fileobjs(ifo, ofo, strip=False): from pyutil.strutil import pop_trailing_newlines, split_on_newlines for l in ifo: for sl in split_on_newlines(pop_trailing_newlines(l)): if strip: sl = sl.strip() ofo.write(pop_trailing_newlines(sl) + '\n')
def _find_missing_names(self, fname): f = open(fname, 'r') names = dictutil.NumDict() for line in f.xreadlines(): mo = CARDNAME_FIELD_RE.match(strutil.pop_trailing_newlines(line)) if mo: if mo.group(5): names.add_num(_fixnames(mo.group(2)), int(mo.group(5))) else: names.inc(_fixnames(mo.group(2))) for k, v, in names.items(): if v != 1: print "k: %s, v: %s" % (k, v,) for card in self.cards(): if card['Card Name'] not in names: print "%s in db and not in file" % card['Card Name'] for name in names.keys(): if not self.has_key(name): print "%s in file and not in db" % name
def test_short_input(self): self.assertTrue(strutil.pop_trailing_newlines("\r\n") == "") self.assertTrue(strutil.pop_trailing_newlines("\r") == "") self.assertTrue(strutil.pop_trailing_newlines("x\r\n") == "x") self.assertTrue(strutil.pop_trailing_newlines("x\r") == "x")
def test_short_input(self): self.failUnless(strutil.pop_trailing_newlines("\r\n") == "") self.failUnless(strutil.pop_trailing_newlines("\r") == "") self.failUnless(strutil.pop_trailing_newlines("x\r\n") == "x") self.failUnless(strutil.pop_trailing_newlines("x\r") == "x")
def import_list(self, fname): """ This reads in a spoiler list in any currently known Wizards of the Coast text format and populates self. """ f = open(fname, 'r') id2cs = dictutil.UtilDict() # k: tuple of (set name, card number,), v: list of card objects setname = None cardstotal = None # a self-check to see if we got them all namesfound = dictutil.NumDict() # a self-check to see if we got them all incard = false thiscard = None thiskey = None thisval = None prevline = None # just for debugging for line in f.xreadlines(): print line line = strutil.pop_trailing_newlines(line) mo = CARDNAME_FIELD_RE.match(line) if mo: if mo.group(5): namesfound.add_num(_fixnames(mo.group(2)), int(mo.group(5))) else: namesfound.inc(_fixnames(mo.group(2))) incard = true thiscard = Card() if setname: thiscard['Set Name'] = setname if incard: if len(line) == 0: if thiskey == 'Rarity': incard = false thiscard[thiskey] = thisval elif (line[0] in string.whitespace) or ((line.find(":") == -1) and (line.find("\t") == -1) and (line.find(" ") == -1) and ((thisval and len(thisval) > 50) or (thiskey == 'Flavor Text') or ((line[0] in string.ascii_uppercase) and (thiskey == 'Card Text')))) or (thisval and (thisval.strip()[0] == '"') and (thisval.strip()[-1] != '"') and (thiskey == 'Card Text') and (line.find("Flavor Text:") == -1)): # Some spoiler lists have typos in which continued lines don't start with white space. We'll assume that if this line doesn't have a separator (none of ':', '\t', or ' '), *and* if the previous line was more than 50 chars, that this is one of those typos. # Some spoiler lists have typos in which attributions of quotes start on the line after the quote (in a Flavor Text field) but have no start with white space. We'll assume that if this line begins with '-', doesn't have a separator (none of ':', '\t', or ' '), *and* if the previous line was a Flavor Text field, then this is one of those typos. # Some older spoiler lists have linebreaks between lines of Card Text. We'll assume that if the previous line was a Card Text field, and this line doesn't have a separator (none of ':', '\t', or ' '), *and* if the first character of the line is a capital letter, that this is one of those. In addition, we'll assume that if the previous line was a Card Text field, and it began with a quote character but didn't end with one, that this is one of those. # Some spoiler lists (e.g. le_spoiler_en.txt in Chromeshell Crab) have typos where multilines are begun with a double-quote but not terminated with a double-quote! So if the line contains "Flavor Text:" then we don't follow the "keep going til you find a double-quote since you started with a double-quote" rule. assert thiskey is not None assert thisval is not None thisval += ' ' + line.strip() elif (line.find(":") == -1) and (line.find("\t") == -1) and (line.find(" ") == -1) and (line[0] in string.ascii_uppercase) and (not thiscard.has_key('Card Text')): # Some spoiler lists (e.g. "Dry Spell" in homelands.txt), have typos where the "Card Text:" param name is missing! We'll assume that if this line doesn't have a separator (none of ':', '\t', or ' '), *and* none of the previous assumptions have been taken, *and* if the first character of the line is a capital letter, *and* if there has not been any 'Card Text' field processed for this card yet, then this is one of those. if thiskey: self._process_key_and_val(thiskey, thisval, thiscard) thiskey = None thisval = None thiskey = 'Card Text' thisval = line elif RARITY_RE.match(strip_whitespace_and_quotes(line)): # Some spoiler lists (e.g. "Gaea's Touch" in the_dark.txt), have typos where the "Rarity:" param name is missing! We'll assume that if this val matches the RARITY_RE then this is one of those. if thiskey: self._process_key_and_val(thiskey, thisval, thiscard) thiskey = None thisval = None thiskey = 'Rarity' thisval = line else: self._process_key_and_val(thiskey, thisval, thiscard) thiskey = None thisval = None # on to the next key and val! sepindex = line.find(':') if sepindex == -1: # Some spoiler lists have typos in which the ':' is missing... So we'll break on the first occurence of two-spaces or a tab. twospaceindex = line.find(' ') tabindex = line.find('\t') if (twospaceindex != -1) and ((twospaceindex <= tabindex) or (tabindex == -1)): sepindex = twospaceindex elif (tabindex != -1) and ((tabindex <= twospaceindex) or (twospaceindex == -1)): sepindex = tabindex assert sepindex != -1, "line: %s, line.find('\t'): %s, line.find(' '): %s" % (`line`, line.find('\t'), line.find(' '),) thiskey = strip_whitespace_and_quotes(line[:sepindex]) thisval = line[sepindex+1:].strip() if thiskey == "Card #": id2cs.setdefault((thiscard['Set Name'], thisval,), []).append(thiscard) incard = false thiscard[thiskey] = thisval elif (not setname) and SPOILER_NAME_RE.match(line): assert (SPOILER_NAME_RE.match(line).group(1) is None) != (SPOILER_NAME_RE.match(line).group(2) is None) if SPOILER_NAME_RE.match(line).group(1): setname = SPOILER_NAME_RE.match(line).group(1) else: setname = SPOILER_NAME_RE.match(line).group(2) elif CARDS_TOTAL_RE.match(line): cardstotal = int(CARDS_TOTAL_RE.match(line).group(1)) prevline = line # just for debugging if setname == "Antiquities": cardstotal -= 1 # error in antiquities.txt cardstotal, AFAICT if setname == "Urza's Legacy": # whoops -- this one is in a completely different format. self.import_urzas_legacy_spoiler(fname) # Okay now fix up any obsolete or inconsistent bits. map(Card._update, self.cards()) # Okay now merge any "double cards" into a single entry. for ((setname, cardnum,), cs,) in id2cs.items(): if len(cs) > 1: assert len(cs) == 2, "len(cs): %s, cs: %s" % (len(cs), cs,) one = cs[0] two = cs[1] subone, subtwo = one['Card Name'], two['Card Name'] for thiscard in (one, two,): assert thiscard['Card Name'], thiscard.data thiscard['This Card Name'] = thiscard['Card Name'] mo = DOUBLE_CARD_NAME_RE.match(thiscard['Card Name']) if mo is not None: subone, subtwo, thisone = mo.groups() thiscard['This Card Name'] = thisone if subone != one['This Card Name']: temp = one one = two two = temp newname = one['This Card Name'] + "/" + two['This Card Name'] assert not self.has_key(newname), "newname: %s" % newname newc = Card(one) newc["Card Name"] = newname for k in ("Mana Cost", "Card Color", "Type & Class", "Artist",): if one.get(k, "") == two.get(k, ""): if one.has_key(k): newc[k] = one[k] else: newc[k] = one.get(k,'') + "/" + two.get(k,'') if one.get("Card Text", "") == two.get("Card Text", ""): if one.has_key("Card Text"): newc["Card Text"] = one["Card Text"] else: newc["Card Text"] = one['This Card Name'] + " -- " + one.get("Card Text",'') + "\n" + two['This Card Name'] + " -- " + two.get("Card Text",'') del self[one['Card Name']] try: del self[two['Card Name']] except: print two.data print two raise del newc['This Card Name'] self[newname] = newc print "cardstotal: %s, after merging, we had %s" % (cardstotal, len(self.cards()),) if (cardstotal is not None) and (cardstotal != len(self.cards())) and ((cardstotal != namesfound.sum()) or (len(self.cards()) != len(namesfound))): print "after merge, cardstotal: %s, namesfound.sum(): %s, len(namesfound): %s, len(self.cards()): %s" % (cardstotal, namesfound.sum(), len(namesfound), len(self.cards()),)