def mutate(xd, words, chance=1):
nmutations = 0
for hwd, vwd, i, j, r, c in each_word_cross(xd):
hwd_a, pivot_char, hwd_b = hwd[:i], hwd[i], hwd[i:][1:]
vwd_a, pivot_char, vwd_b = vwd[:j], vwd[j], vwd[j:][1:]
progress("%s[%s]%s/%s[%s]%s" % (hwd_a, pivot_char, hwd_b, vwd_a, pivot_char, vwd_b))
mutations_this_square = []
for ch in string.ascii_uppercase:
if ch == pivot_char:
continue
new_hwd = hwd_a + ch + hwd_b
new_vwd = vwd_a + ch + vwd_b
if new_vwd in words and new_hwd in words:
mutations_this_square.append((new_hwd, new_vwd, ch))
if mutations_this_square:
most_common = sorted(mutations_this_square, key=lambda x: len(words[x[0]]) + len(words[x[1]]))[-1]
new_hwd, new_vwd, best_replacement = most_common
if random.random() < chance:
nmutations += 1
xd.grid[r] = splice(xd.grid[r], c, best_replacement)
info("-> %s/%s (%s)" % (new_hwd, new_vwd, "".join(br for h, v, br in mutations_this_square)))
return nmutations
def mutate(xd, words, chance=1):
nmutations = 0
for hwd, vwd, i, j, r, c in each_word_cross(xd):
hwd_a, pivot_char, hwd_b = hwd[:i], hwd[i], hwd[i:][1:]
vwd_a, pivot_char, vwd_b = vwd[:j], vwd[j], vwd[j:][1:]
progress("%s[%s]%s/%s[%s]%s" %
(hwd_a, pivot_char, hwd_b, vwd_a, pivot_char, vwd_b))
mutations_this_square = []
for ch in string.ascii_uppercase:
if ch == pivot_char:
continue
new_hwd = hwd_a + ch + hwd_b
new_vwd = vwd_a + ch + vwd_b
if new_vwd in words and new_hwd in words:
mutations_this_square.append((new_hwd, new_vwd, ch))
if mutations_this_square:
most_common = sorted(
mutations_this_square,
key=lambda x: len(words[x[0]]) + len(words[x[1]]))[-1]
new_hwd, new_vwd, best_replacement = most_common
if random.random() < chance:
nmutations += 1
xd.grid[r] = splice(xd.grid[r], c, best_replacement)
info("-> %s/%s (%s)" % (new_hwd, new_vwd, "".join(
br for h, v, br in mutations_this_square)))
return nmutations
def main():
p = args_parser("process huge puzzles archive into separate .zip and create sources.tsv")
p.add_argument("-s", "--source", default=None, help="ExternalSource")
args = get_args(parser=p)
outf = open_output()
if args.source:
source = args.source
else:
source = parse_pathname(args.inputs[0]).base
subzips = {}
for inputfn in args.inputs:
for fn, contents, dt in xdfile.utils.find_files_with_time(inputfn):
if not contents:
continue
m = re.match(r"^([a-z]{2,4})[\-0-9]{1}\d.*", parse_pathname(fn).base, flags=re.IGNORECASE)
prefix = m.group(1).lower() if m else "misc"
if prefix not in subzips:
zf = xdfile.utils.OutputZipFile(os.path.join(args.output, prefix + ".zip"))
sources = []
subzips[prefix] = (zf, sources)
else:
zf, sources = subzips[prefix]
progress("Processing %s -> %s" % (fn, prefix))
zf.write_file(fn, contents, dt)
sources.append(xd_sources_row(fn, source, iso8601(dt)))
for zf, sources in subzips.values():
zf.write_file("sources.tsv", xd_sources_header + "".join(sources))
def main():
global boiled_clues
args = get_args('create clue index')
outf = open_output()
boiled_clues = load_clues()
biggest_clues = "<li>%d total clues, which boil down to %d distinct clues" % (len(clues()), len(boiled_clues))
bcs = [ (len(v), bc, answers_from(v)) for bc, v in boiled_clues.items() ]
nreused = len([bc for n, bc, _ in bcs if n > 1])
biggest_clues += "<li>%d (%d%%) of these clues are used in more than one puzzle" % (nreused, nreused*100/len(boiled_clues))
cluepages_to_make = set()
# add all boiled clues from all input .xd files
for fn, contents in find_files(*args.inputs, ext='.xd'):
progress(fn)
xd = xdfile.xdfile(contents.decode('utf-8'), fn)
for pos, mainclue, mainanswer in xd.iterclues():
cluepages_to_make.add(boil(mainclue))
# add top 100 most used boiled clues from corpus
biggest_clues += '<h2>Most used clues</h2>'
biggest_clues += '<table class="clues most-used-clues">'
biggest_clues += th("clue", "# uses", "answers used with this clue")
for n, bc, ans in sorted(bcs, reverse=True)[:100]:
cluepages_to_make.add(bc)
biggest_clues += td(mkhref(unboil(bc), bc), n, html_select_options(ans))
biggest_clues += '</table>'
most_ambig = "<h2>Most ambiguous clues</h2>"
most_ambig += '(clues with the largest number of different answers)'
most_ambig += '<table class="clues most-different-answers">'
most_ambig += th("Clue", "answers")
for n, bc, ans in sorted(bcs, reverse=True, key=lambda x: len(set(x[2])))[:100]:
cluepages_to_make.add(bc)
clue = mkhref(unboil(bc), bc)
if 'quip' in bc or 'quote' in bc or 'theme' in bc or 'riddle' in bc:
most_ambig += td(clue, html_select_options(ans), rowclass="theme")
else:
most_ambig += td(clue, html_select_options(ans))
most_ambig += '</table>'
for bc in cluepages_to_make:
contents = mkwww_cluepage(bc)
if contents:
outf.write_html('pub/clue/%s/index.html' % bc, contents, title=bc)
outf.write_html('pub/clue/index.html', biggest_clues + most_ambig, title="Clues")
def load_clues():
ret = {
} # ["pubid"] = { ["ANSWER"] = { ["simplified clue text"] = set(fullclues) } }
for r in parse_tsv(file("clues.tsv").read(), "AnswerClue"):
try:
pubid, dt, answer, clue = r
except Exception as e:
print(str(e), r)
continue
progress(dt, every=100000)
if not clue:
continue
if "Across" in clue or "Down" in clue: # skip self-referential clues
continue
boiled_clue = boil(clue)
clue = "%s [%s%s]" % (clue, pubid, dt)
if pubid not in ret:
answers = {}
ret[pubid] = answers
else:
answers = ret[pubid]
if answer not in answers:
clues = {}
answers[answer] = clues
else:
clues = answers[answer]
if boiled_clue not in clues:
clues[boiled_clue] = set()
clues[boiled_clue].add(clue)
progress()
return ret
def main():
p = args_parser(
'process huge puzzles archive into separate .zip and create sources.tsv'
)
p.add_argument('-s', '--source', default=None, help='ExternalSource')
args = get_args(parser=p)
outf = open_output()
if args.source:
source = args.source
else:
source = parse_pathname(args.inputs[0]).base
subzips = {}
for inputfn in args.inputs:
for fn, contents, dt in xdfile.utils.find_files_with_time(inputfn):
if not contents:
continue
m = re.match(r'^([a-z]{2,4})[\-0-9]{1}\d.*',
parse_pathname(fn).base,
flags=re.IGNORECASE)
prefix = m.group(1).lower() if m else 'misc'
if prefix not in subzips:
zf = xdfile.utils.OutputZipFile(
os.path.join(args.output, prefix + ".zip"))
sources = []
subzips[prefix] = (zf, sources)
else:
zf, sources = subzips[prefix]
progress("Processing %s -> %s" % (fn, prefix))
zf.write_file(fn, contents, dt)
sources.append(xd_sources_row(fn, source, iso8601(dt)))
for zf, sources in subzips.values():
zf.write_file("sources.tsv", xd_sources_header + "".join(sources))
def load_clues():
ret = {} # ["pubid"] = { ["ANSWER"] = { ["simplified clue text"] = set(fullclues) } }
for r in parse_tsv(file("clues.tsv").read(), "AnswerClue"):
try:
pubid, dt, answer, clue = r
except Exception as e:
print(str(e), r)
continue
progress(dt, every=100000)
if not clue:
continue
if "Across" in clue or "Down" in clue: # skip self-referential clues
continue
boiled_clue = boil(clue)
clue = "%s [%s%s]" % (clue, pubid, dt)
if pubid not in ret:
answers = {}
ret[pubid] = answers
else:
answers = ret[pubid]
if answer not in answers:
clues = {}
answers[answer] = clues
else:
clues = answers[answer]
if boiled_clue not in clues:
clues[boiled_clue] = set()
clues[boiled_clue].add(clue)
progress()
return ret
def main():
args = utils.get_args('generates .html diffs with deep clues for all puzzles in similar.tsv')
outf = utils.open_output()
similars = utils.parse_tsv('gxd/similar.tsv', 'Similar')
xdids_todo = [ parse_pathname(fn).base for fn in args.inputs ]
if not xdids_todo:
xdids_todo = [ xdid for xdid, matches in metadb.get_similar_grids().items() if matches ]
for mainxdid in xdids_todo:
progress(mainxdid)
mainxd = xdfile.get_xd(mainxdid)
if not mainxd:
continue
matches = metadb.get_similar_grids().get(mainxdid, [])
xddates = {}
xddates[mainxdid] = mainxd.date() # Dict to store XD dates for further sort
html_grids = {}
html_clues = {}
# these are added directly to similar.tsv
nstaleclues = 0
nstaleanswers = 0
ntotalclues = 0
poss_answers = [] # TODO:
pub_uses = {} # [pubid] -> set(ClueAnswer)
dcl_html = ''
deepcl_html = [] # keep deep clues to parse later - per row
for pos, mainclue, mainanswer in mainxd.iterclues():
deepcl_html = [] # Temporary to be replaced late
mainca = ClueAnswer(mainxdid, mainxd.date(), mainanswer, mainclue)
# 'grid position' column
deepcl_html.append('<td class="pos">%s.</td>' % pos)
# find other uses of this clue, and other answers, in a single pass
for clueans in find_clue_variants(mainclue):
if clueans.answer != mainanswer:
poss_answers.append(clueans)
if clueans.answer == mainanswer:
if clueans.pubid in pub_uses:
otherpubs = pub_uses[clueans.pubid]
else:
otherpubs = set() # set of ClueAnswer
pub_uses[clueans.pubid] = otherpubs
otherpubs.add(clueans)
# add 'other uses' to clues_html
stale = False
deepcl_html.append('<td class="other-uses">')
if len(pub_uses) > 0:
sortable_uses = []
for pubid, uses in pub_uses.items():
# show the earliest unboiled clue
for u in sorted(uses, key=lambda x: x.date or ""):
# only show those published earlier
if u.date and u.date <= mainxd.date():
if pubid == mainxdid and u.date == mainxd.date():
pass
else:
stale = True
sortable_uses.append((u.date, u, 1))
deepcl_html.append(html_select([ (clue, nuses) for dt, clue, nuses in sorted(sortable_uses, key=lambda x: x[0], reverse=True) ], top_option=mainclue))
else:
deepcl_html.append('<div class="original">%s</div>' % esc(mainclue))
deepcl_html.append('</td>')
# add 'other answers' to clues_html
deepcl_html.append('<td class="other-answers">')
deepcl_html.append(html_select_options(poss_answers, strmaker=lambda ca: ca.answer, force_top=mainca))
deepcl_html.append('</td>')
# add 'other clues' to clues_html
deepcl_html.append('<td class="other-clues">')
# bclues is all boiled clues for this particular answer: { [bc] -> #uses }
bclues = load_answers().get(mainanswer, [])
stale_answer = False
if bclues:
uses = []
for bc, nuses in bclues.items():
# then find all clues besides this one
clue_usages = [ ca for ca in load_clues().get(bc, []) if ca.answer == mainanswer and ca.date < mainxd.date() ]
if clue_usages:
stale_answer = True
if nuses > 1:
# only use one (the most recent) ClueAnswer per boiled clue
# but use the clue only (no xdid)
ca = sorted(clue_usages, key=lambda ca: ca.date or "z")[-1].clue
else:
ca = sorted(clue_usages, key=lambda ca: ca.date or "z")[-1]
uses.append((ca, nuses))
if uses:
deepcl_html.append(html_select(uses))
deepcl_html.append('</td>') # end 'other-clues'
if stale_answer:
nstaleanswers += 1
if stale:
nstaleclues += 1
ntotalclues += 1
# Quick and dirty - to be replaced
dcl_html += '<tr>' + ' '.join(deepcl_html) + '</tr>'
# Store in list to make further formatting as html table easier
mainxd = xdfile.get_xd(mainxdid)
if mainxd:
html_grids[mainxdid] = grid_diff_html(mainxd)
# Add for main XD
diff_l = []
for pos, mainclue, mainanswer in mainxd.iterclues():
diff_h = mktag('div','fullgrid main') + '%s. ' %pos
diff_h += mainclue
diff_h += mktag('span', tagclass='main', inner=' ~ ' + mainanswer.upper())
diff_l.append(diff_h)
html_clues[mainxdid] = diff_l
# Process for all matches
for xdid in matches:
xd = xdfile.get_xd(xdid)
if not xd:
continue
xddates[xdid] = xd.date()
# output each grid
html_grids[xdid] = grid_diff_html(xd, compare_with=mainxd)
diff_l = []
# output comparison of each set of clues
for pos, clue, answer in xd.iterclues():
diff_h = mktag('div','fullgrid') + '%s. ' %pos
# Sometimes can return clue == None
sm = difflib.SequenceMatcher(lambda x: x == ' ', mainxd.get_clue(pos) or '', clue)
if sm.ratio() < 0.50:
diff_h += clue
else:
# Compare based on op codes
for opcode in sm.get_opcodes():
c, a1, a2, b1, b2 = opcode
if c == 'equal':
diff_h += '<span class="match">%s</span>' % clue[b1:b2]
else:
diff_h += '<span class="diff">%s</span>' % clue[b1:b2]
diff_h += mktag('span', tagclass=(answer == mainxd.get_answer(pos)) and 'match' or 'diff', inner=' ~ ' + answer.upper())
diff_h += mktag('/div')
diff_l.append(diff_h)
html_clues[xdid] = diff_l
# Wrap into table
diff_h = mktag('table') + mktag('tr')
# Sort by date
sortedkeys = sorted(xddates.items(), key=operator.itemgetter(1))
for w, dt in sortedkeys:
# Wrap into table
diff_h += mktag('td') + html_grids[w] + mktag('/td')
diff_h += mktag('/tr')
for i, clue in enumerate(html_clues[sortedkeys[0][0]]):
diff_h += mktag('tr')
for w, dt in sortedkeys:
if i < len(html_clues[w]):
diff_h += mktag('td') + html_clues[w][i] + mktag('/td')
diff_h += mktag('/tr')
# Process deepclues
diff_h += mktag('table') + dcl_html + mktag('/table')
diff_h += mktag('/table')
outf.write_html('pub/deep/%s/index.html' % mainxdid, diff_h,
title='Deep clue comparison for ' + mainxdid)
def main():
global args
parsers = {
'.xml': [parse_ccxml, parse_uxml],
'.json': [parse_ujson],
'.puz': [parse_puz],
'.html': [parse_xwordinfo],
'.pdf': [],
'.jpg': [],
'.gif': [],
'.xd': [], # special case, just copy the input, in case re-emitting screws it up
}
p = args_parser('convert crosswords to .xd format')
p.add_argument('--copyright', default=None, help='Default value for unspecified Copyright headers')
p.add_argument('--extsrc', default=None, help='Value for receipts.ExternalSource')
p.add_argument('--intsrc', default=None, help='Value for receipts.InternalSource')
p.add_argument('--pubid', default=None, help='PublicationAbbr (pubid) to use')
args = get_args(parser=p)
outf = open_output()
for input_source in args.inputs:
try:
# collect 'sources' metadata
source_files = {}
# collect receipts
receipts = []
for fn, contents, dt in find_files_with_time(input_source, ext='.tsv'):
progress(fn)
for row in parse_tsv_data(contents.decode('utf-8'), "Source"):
innerfn = strip_toplevel(row.SourceFilename)
if innerfn in source_files:
warn("%s: already in source_files!" % innerfn)
continue
source_files[innerfn] = row
# enumerate all files in this source, reverse-sorted by time
# (so most recent edition gets main slot in case of shelving
# conflict)
for fn, contents, dt in sorted(find_files_with_time(input_source, strip_toplevel=False), reverse=True, key=lambda x: x[2]):
if fn.endswith(".tsv") or fn.endswith(".log"):
continue
if not contents: # 0-length files
continue
innerfn = strip_toplevel(fn)
if innerfn in source_files:
srcrow = source_files[innerfn]
CaptureTime = srcrow.DownloadTime
ExternalSource = args.extsrc or srcrow.ExternalSource
SourceFilename = innerfn
else:
debug("%s not in sources.tsv" % innerfn)
CaptureTime = iso8601(dt)
ExternalSource = args.extsrc or parse_pathname(input_source).filename
SourceFilename = innerfn
ReceivedTime = iso8601(time.time())
InternalSource = args.intsrc or parse_pathname(input_source).filename
already_received = metadb.check_already_received(ExternalSource, SourceFilename)
xdid = ""
prev_xdid = "" # unshelved by default
existing_xdids = set(r.xdid for r in already_received)
if existing_xdids:
if len(existing_xdids) > 1:
warn('previously received this same file under multiple xdids:' + ' '.join(existing_xdids))
else:
prev_xdid = existing_xdids.pop()
debug('already received as %s' % prev_xdid)
# try each parser by extension
ext = parse_pathname(fn).ext.lower()
possible_parsers = parsers.get(ext, parsers[".puz"])
progress(fn)
if ext == ".xd":
outf.write_file(fn, contents.decode('utf-8'), dt)
elif not possible_parsers:
rejected = "no parser"
else:
rejected = ""
for parsefunc in possible_parsers:
try:
try:
xd = parsefunc(contents, fn)
except IncompletePuzzleParse as e:
error("%s %s" % (fn, e))
xd = e.xd
if not xd:
continue
xd.filename = replace_ext(strip_toplevel(fn), ".xd")
if not xd.get_header("Copyright"):
if args.copyright:
xd.set_header("Copyright", args.copyright)
catalog.deduce_set_seqnum(xd)
xdstr = xd.to_unicode()
mdtext = "|".join((ExternalSource,InternalSource,SourceFilename))
xdid = prev_xdid or catalog.deduce_xdid(xd, mdtext)
path = catalog.get_shelf_path(xd, args.pubid, mdtext)
outf.write_file(path + ".xd", xdstr, dt)
rejected = ""
break # stop after first successful parsing
except xdfile.NoShelfError as e:
error("could not shelve: %s" % str(e))
rejected += "[shelver] %s " % str(e)
except Exception as e:
error("%s could not convert [%s]: %s" % (parsefunc.__name__, fn, str(e)))
rejected += "[%s] %s " % (parsefunc.__name__, str(e))
# raise
if rejected:
error("could not convert: %s" % rejected)
# only add receipt if first time converting this source
if already_received:
debug("already received %s:%s" % (ExternalSource, SourceFilename))
else:
receipts.append([
CaptureTime,
ReceivedTime,
ExternalSource,
InternalSource,
SourceFilename,
xdid
])
for r in receipts:
metadb.append_row('gxd/receipts', r)
except Exception as e:
error(str(e))
if args.debug:
raise
def main():
global args
parsers = {
'.xml': [parse_ccxml, parse_uxml],
'.json': [parse_ujson],
'.puz': [parse_puz],
'.html': [parse_xwordinfo],
'.pdf': [],
'.jpg': [],
'.gif': [],
'.xd':
[], # special case, just copy the input, in case re-emitting screws it up
}
p = args_parser('convert crosswords to .xd format')
p.add_argument('--copyright',
default=None,
help='Default value for unspecified Copyright headers')
p.add_argument('--extsrc',
default=None,
help='Value for receipts.ExternalSource')
p.add_argument('--intsrc',
default=None,
help='Value for receipts.InternalSource')
p.add_argument('--pubid',
default=None,
help='PublicationAbbr (pubid) to use')
args = get_args(parser=p)
outf = open_output()
for input_source in args.inputs:
try:
# collect 'sources' metadata
source_files = {}
# collect receipts
receipts = []
for fn, contents, dt in find_files_with_time(input_source,
ext='.tsv'):
progress(fn)
for row in parse_tsv_data(contents.decode('utf-8'), "Source"):
innerfn = strip_toplevel(row.SourceFilename)
if innerfn in source_files:
warn("%s: already in source_files!" % innerfn)
continue
source_files[innerfn] = row
# enumerate all files in this source, reverse-sorted by time
# (so most recent edition gets main slot in case of shelving
# conflict)
for fn, contents, dt in sorted(find_files_with_time(
input_source, strip_toplevel=False),
reverse=True,
key=lambda x: x[2]):
if fn.endswith(".tsv") or fn.endswith(".log"):
continue
if not contents: # 0-length files
continue
innerfn = strip_toplevel(fn)
if innerfn in source_files:
srcrow = source_files[innerfn]
CaptureTime = srcrow.DownloadTime
ExternalSource = args.extsrc or srcrow.ExternalSource
SourceFilename = innerfn
else:
debug("%s not in sources.tsv" % innerfn)
CaptureTime = iso8601(dt)
ExternalSource = args.extsrc or parse_pathname(
input_source).filename
SourceFilename = innerfn
ReceivedTime = iso8601(time.time())
InternalSource = args.intsrc or parse_pathname(
input_source).filename
already_received = metadb.check_already_received(
ExternalSource, SourceFilename)
xdid = ""
prev_xdid = "" # unshelved by default
existing_xdids = set(r.xdid for r in already_received)
if existing_xdids:
if len(existing_xdids) > 1:
warn(
'previously received this same file under multiple xdids:'
+ ' '.join(existing_xdids))
else:
prev_xdid = existing_xdids.pop()
debug('already received as %s' % prev_xdid)
# try each parser by extension
ext = parse_pathname(fn).ext.lower()
possible_parsers = parsers.get(ext, parsers[".puz"])
progress(fn)
if ext == ".xd":
outf.write_file(fn, contents.decode('utf-8'), dt)
elif not possible_parsers:
rejected = "no parser"
else:
rejected = ""
for parsefunc in possible_parsers:
try:
try:
xd = parsefunc(contents, fn)
except IncompletePuzzleParse as e:
error("%s %s" % (fn, e))
xd = e.xd
if not xd:
continue
xd.filename = replace_ext(strip_toplevel(fn),
".xd")
if not xd.get_header("Copyright"):
if args.copyright:
xd.set_header("Copyright", args.copyright)
catalog.deduce_set_seqnum(xd)
xdstr = xd.to_unicode()
mdtext = "|".join((ExternalSource, InternalSource,
SourceFilename))
xdid = prev_xdid or catalog.deduce_xdid(xd, mdtext)
path = catalog.get_shelf_path(
xd, args.pubid, mdtext)
outf.write_file(path + ".xd", xdstr, dt)
rejected = ""
break # stop after first successful parsing
except xdfile.NoShelfError as e:
error("could not shelve: %s" % str(e))
rejected += "[shelver] %s " % str(e)
except Exception as e:
error("%s could not convert [%s]: %s" %
(parsefunc.__name__, fn, str(e)))
rejected += "[%s] %s " % (parsefunc.__name__,
str(e))
# raise
if rejected:
error("could not convert: %s" % rejected)
# only add receipt if first time converting this source
if already_received:
debug("already received %s:%s" %
(ExternalSource, SourceFilename))
else:
receipts.append([
CaptureTime, ReceivedTime, ExternalSource,
InternalSource, SourceFilename, xdid
])
for r in receipts:
metadb.append_row('gxd/receipts', r)
except Exception as e:
error(str(e))
if args.debug:
raise
def main():
args = utils.get_args(
'generates .html diffs with deep clues for all puzzles in similar.tsv')
outf = utils.open_output()
similars = utils.parse_tsv('gxd/similar.tsv', 'Similar')
xds_todo = []
for fn, contents in find_files(*args.inputs, ext='.xd'):
xd = xdfile.xdfile(contents.decode('utf-8'), fn)
xds_todo.append(xd)
for mainxd in xds_todo:
mainxdid = mainxd.xdid()
progress(mainxdid)
matches = metadb.xd_similar(mainxdid)
xddates = {}
xddates[mainxdid] = mainxd.date(
) # Dict to store XD dates for further sort
html_grids = {}
# these are added directly to similar.tsv
nstaleclues = 0
nstaleanswers = 0
ntotalclues = 0
dcl_html = '<tr>'
dcl_html += '<th></th>'
dcl_html += '<th>Clue</th>'
dcl_html += '<th>ANSWERs</th>'
dcl_html += '<th>Alt. clue possibilities</th>'
dcl_html += '</tr>'
deepcl_html = [] # keep deep clues to parse later - per row
for pos, mainclue, mainanswer in mainxd.iterclues():
if not pos:
continue
poss_answers = [] # TODO:
pub_uses = {} # [pubid] -> set(ClueAnswer)
deepcl_html = [] # Temporary to be replaced late
mainca = ClueAnswer(mainxdid, mainxd.date(), mainanswer, mainclue)
# 'grid position' column
deepcl_html.append('<td class="pos">%s.</td>' % pos)
# find other uses of this clue, and other answers, in a single pass
for clueans in find_clue_variants(mainclue):
if clueans.answer != mainanswer:
poss_answers.append(clueans)
if clueans.answer == mainanswer:
if clueans.pubid in pub_uses:
otherpubs = pub_uses[clueans.pubid]
else:
otherpubs = set() # set of ClueAnswer
pub_uses[clueans.pubid] = otherpubs
otherpubs.add(clueans)
# add 'other uses' to clues_html
deepcl_html.append('<td class="other-uses">')
prev = prev_uses(pub_uses, mainxd, mainclue)
if prev:
deepcl_html.append('<a href="/pub/clue/%s">%s [x%s]</a>' %
(boil(mainclue), mainclue, len(prev)))
nstaleclues += 1
else:
deepcl_html.append(mainclue)
deepcl_html.append('</td>')
# add 'other answers' to clues_html
deepcl_html.append('<td class="other-answers">')
deepcl_html.append(
html_select_options(poss_answers,
strmaker=lambda ca: ca.answer,
force_top=mainca,
add_total=False))
deepcl_html.append('</td>')
# add 'other clues' to clues_html
deepcl_html.append('<td class="other-clues">')
other_clues = html_other_clues(mainanswer, mainclue, mainxd)
if other_clues:
deepcl_html.append(other_clues)
nstaleanswers += 1
deepcl_html.append('</td>') # end 'other-clues'
ntotalclues += 1
# Quick and dirty - to be replaced
dcl_html += '<tr>' + ' '.join(deepcl_html) + '</tr>'
# Process deepclues
diff_h = '<div class="main-container">'
diff_h += grid_to_html(mainxd)
diff_h += mktag('table', 'deepclues') + dcl_html + mktag('/table')
diff_h += '</div>'
info('writing deepclues for %s' % mainxdid)
outf.write_html('pub/deep/%s/index.html' % mainxdid,
diff_h,
title='Deep clue analysis for ' + mainxdid)
def main():
args = utils.get_args("generates .html diffs for all puzzles in similar.tsv")
outf = utils.open_output()
similars = utils.parse_tsv("gxd/similar.tsv", "Similar")
xdids_todo = args.inputs or [xdid for xdid, matches in metadb.get_similar_grids().items() if matches]
for mainxdid in xdids_todo:
progress(mainxdid)
mainxd = xdfile.get_xd(mainxdid)
if not mainxd:
continue
matches = metadb.get_similar_grids().get(mainxdid, [])
xddates = {}
xddates[mainxdid] = mainxd.date() # Dict to store XD dates for further sort
html_grids = {}
html_clues = {}
# Store in list to make further formatting as html table easier
html_grids[mainxdid] = grid_diff_html(xdfile.get_xd(mainxdid))
# Add for main XD
diff_l = []
for pos, mainclue, mainanswer in mainxd.iterclues():
diff_h = mktag("div", "fullgrid main") + "%s. " % pos
diff_h += mainclue
diff_h += mktag("span", tagclass="main", inner=" ~ " + mainanswer.upper())
diff_l.append(diff_h)
html_clues[mainxdid] = diff_l
# Process for all matches
for xdid in matches:
xd = xdfile.get_xd(xdid)
# Continue if can't load xdid
if not xd:
continue
xddates[xdid] = xd.date()
# output each grid
html_grids[xdid] = grid_diff_html(xd, compare_with=mainxd)
diff_l = []
# output comparison of each set of clues
for pos, clue, answer in xd.iterclues():
diff_h = mktag("div", "fullgrid") + "%s. " % pos
# Sometimes can return clue == None
mainclue = mainxd.get_clue_for_answer(answer)
sm = difflib.SequenceMatcher(lambda x: x == " ", mainclue or "", clue)
debug("MCLUE: %s [%s]" % (mainclue, sm.ratio()))
if mainclue is None or sm.ratio() < 0.40:
diff_h += clue
else:
# Compare based on op codes
for opcode in sm.get_opcodes():
c, a1, a2, b1, b2 = opcode
if c == "equal":
diff_h += '<span class="match">%s</span>' % clue[b1:b2]
else:
diff_h += '<span class="diff">%s</span>' % clue[b1:b2]
tagclass = "match" if mainclue or answer == mainxd.get_answer(pos) else "diff"
diff_h += mktag("span", tagclass=tagclass, inner=" ~ " + answer.upper())
diff_h += mktag("/div")
diff_l.append(diff_h)
html_clues[xdid] = diff_l
# Wrap into table
diff_h = mktag("table") + mktag("tr")
# Sort by date
sortedkeys = sorted(xddates.items(), key=operator.itemgetter(1))
for w, dt in sortedkeys:
# Wrap into table
diff_h += mktag("td") + html_grids[w] + mktag("/td")
diff_h += mktag("/tr")
for i, clue in enumerate(html_clues[sortedkeys[0][0]]):
diff_h += mktag("tr")
for w, dt in sortedkeys:
if i < len(html_clues[w]):
diff_h += mktag("td") + html_clues[w][i] + mktag("/td")
diff_h += mktag("/tr")
diff_h += mktag("/table")
outf.write_html("pub/%s/index.html" % mainxdid, diff_h, title="Comparison for " + mainxdid)
def main():
p = utils.args_parser(desc="annotate puzzle clues with earliest date used in the corpus")
p.add_argument("-a", "--all", default=False, help="analyze all puzzles, even those already in similar.tsv")
p.add_argument("-l", "--limit", default=100, help="limit amount of puzzles to be analyzed [default=100]")
args = get_args(parser=p)
outf = open_output()
num_processed = 0
prev_similar = metadb.read_rows("gxd/similar")
for fn, contents in find_files(*args.inputs, ext=".xd"):
progress(fn)
mainxd = xdfile(contents.decode("utf-8"), fn)
if mainxd.xdid() in prev_similar:
continue # skip reprocessing .xd that are already in similar.tsv
""" find similar grids (pct, xd) for the mainxd in the corpus.
Takes about 1 second per xd. sorted by pct.
"""
similar_grids = sorted(find_similar_to(mainxd, corpus(), min_pct=0.20), key=lambda x: x[0], reverse=True)
num_processed += 1
if num_processed > int(args.limit):
break
if similar_grids:
info("similar: " + " ".join(("%s=%s" % (xd2.xdid(), pct)) for pct, xd1, xd2 in similar_grids))
mainpubid = mainxd.publication_id()
maindate = mainxd.date()
# go over each clue/answer, find all other uses, other answers, other possibilities.
# these are added directly to similar.tsv
nstaleclues = 0
nstaleanswers = 0
ntotalclues = 0
for pos, mainclue, mainanswer in mainxd.iterclues():
progress(mainanswer)
poss_answers = []
pub_uses = {} # [pubid] -> set(ClueAnswer)
mainca = ClueAnswer(mainpubid, maindate, mainanswer, mainclue)
# find other uses of this clue, and other answers, in a single pass
for clueans in find_clue_variants(mainclue):
if clueans.answer != mainanswer:
poss_answers.append(clueans)
if clueans.answer == mainanswer:
if clueans.pubid in pub_uses:
otherpubs = pub_uses[clueans.pubid]
else:
otherpubs = set() # set of ClueAnswer
pub_uses[clueans.pubid] = otherpubs
otherpubs.add(clueans)
# bclues is all boiled clues for this particular answer: { [bc] -> #uses }
bclues = load_answers().get(mainanswer, [])
stale_answer = False
if bclues:
uses = []
for bc, nuses in bclues.items():
# then find all clues besides this one
clue_usages = [
ca for ca in load_clues().get(bc, []) if ca.answer == mainanswer and ca.date < maindate
]
if clue_usages:
stale_answer = True
if nuses > 1:
# only use one (the most recent) ClueAnswer per boiled clue
# but use the clue only (no xdid)
ca = sorted(clue_usages, key=lambda ca: ca.date or "z")[-1].clue
else:
ca = sorted(clue_usages, key=lambda ca: ca.date or "z")[-1]
uses.append((ca, nuses))
# summary row to similar.tsv
metadb.append_row(
"gxd/similar",
[
mainxd.xdid(), # xdid
int(100 * sum(pct / 100.0 for pct, xd1, xd2 in similar_grids)), # similar_grid_pct
nstaleclues, # reused_clues
nstaleanswers, # reused_answers
ntotalclues, # total_clues
" ".join(("%s=%s" % (xd2.xdid(), pct)) for pct, xd1, xd2 in similar_grids), # matches
],
)
def main():
args = utils.get_args('generates .html diffs for all puzzles in similar.tsv')
outf = utils.open_output()
similars = utils.parse_tsv('gxd/similar.tsv', 'Similar')
xdids_todo = {}
for row in metadb.xd_similar_all():
if row.xdid not in xdids_todo:
xdids_todo[row.xdid] = []
xdids_todo[row.xdid].append(row)
for mainxdid in xdids_todo:
progress(mainxdid)
mainxd = xdfile.get_xd(mainxdid)
if not mainxd:
warn('%s not in corpus' % mainxdid)
continue
matches = xdids_todo[mainxdid]
info('generating diffs for %s (%d matches)' % (mainxdid, len(matches)))
xddates = {}
xddates[mainxdid] = mainxd.date() # Dict to store XD dates for further sort
html_grids = {}
html_clues = {}
# Store in list to make further formatting as html table easier
html_grids[mainxdid] = grid_diff_html(xdfile.get_xd(mainxdid))
# Add for main XD
diff_l = []
for pos, mainclue, mainanswer in mainxd.iterclues():
if not mainclue:
continue
diff_h = mktag('div','fullgrid main') + '%s. ' %pos
diff_h += mainclue
diff_h += mktag('span', tagclass='main', inner=' ~ ' + mainanswer.upper())
diff_l.append(diff_h)
html_clues[mainxdid] = diff_l
# Process for all matches
for row in matches:
xdid = row.match_xdid
xd = xdfile.get_xd(xdid)
# Continue if can't load xdid
if not xd:
continue
xddates[xdid] = xd.date()
# output each grid
html_grids[xdid] = grid_diff_html(xd, compare_with=mainxd)
diff_l = []
# output comparison of each set of clues
for pos, clue, answer in xd.iterclues():
diff_h = mktag('div','fullgrid') + '%s. ' %pos
if not clue:
continue
# Sometimes can return clue == None
mainclue = mainxd.get_clue_for_answer(answer)
sm = difflib.SequenceMatcher(lambda x: x == ' ', mainclue or '', clue)
debug('MCLUE: %s [%s]' % (mainclue, sm.ratio()))
if mainclue is None or sm.ratio() < 0.40:
diff_h += clue
else:
# Compare based on op codes
for opcode in sm.get_opcodes():
c, a1, a2, b1, b2 = opcode
if c == 'equal':
diff_h += '<span class="match">%s</span>' % clue[b1:b2]
else:
diff_h += '<span class="diff">%s</span>' % clue[b1:b2]
tagclass = 'match' if mainclue or answer == mainxd.get_answer(pos) else 'diff'
diff_h += mktag('span', tagclass=tagclass, inner=' ~ ' + answer.upper())
diff_h += mktag('/div')
diff_l.append(diff_h)
html_clues[xdid] = diff_l
# Wrap into table
diff_h = mktag('table') + mktag('tr')
# Sort by date
sortedkeys = sorted(xddates.items(), key=operator.itemgetter(1))
for w, dt in sortedkeys:
# Wrap into table
diff_h += mktag('td') + html_grids[w] + mktag('/td')
diff_h += mktag('/tr')
for i, clue in enumerate(html_clues[sortedkeys[0][0]]):
diff_h += mktag('tr')
for w, dt in sortedkeys:
if i < len(html_clues[w]):
diff_h += mktag('td') + html_clues[w][i] + mktag('/td')
diff_h += mktag('/tr')
diff_h += mktag('/table')
outf.write_html('pub/%s/index.html' % mainxdid, diff_h, title='Comparison for ' + mainxdid)
def main():
p = utils.args_parser(
desc="annotate puzzle clues with earliest date used in the corpus")
p.add_argument(
'-a',
'--all',
default=False,
help='analyze all puzzles, even those already in similar.tsv')
p.add_argument('-l',
'--limit',
default=100,
help='limit amount of puzzles to be analyzed [default=100]')
args = get_args(parser=p)
outf = open_output()
num_processed = 0
prev_similar = metadb.read_rows('gxd/similar')
for fn, contents in find_files(*args.inputs, ext=".xd"):
progress(fn)
mainxd = xdfile(contents.decode('utf-8'), fn)
if mainxd.xdid() in prev_similar:
continue # skip reprocessing .xd that are already in similar.tsv
""" find similar grids (pct, xd) for the mainxd in the corpus.
Takes about 1 second per xd. sorted by pct.
"""
similar_grids = sorted(find_similar_to(mainxd, corpus(), min_pct=0.20),
key=lambda x: x[0],
reverse=True)
num_processed += 1
if num_processed > int(args.limit):
break
if similar_grids:
info("similar: " + " ".join(("%s=%s" % (xd2.xdid(), pct))
for pct, xd1, xd2 in similar_grids))
mainpubid = mainxd.publication_id()
maindate = mainxd.date()
# go over each clue/answer, find all other uses, other answers, other possibilities.
# these are added directly to similar.tsv
nstaleclues = 0
nstaleanswers = 0
ntotalclues = 0
for pos, mainclue, mainanswer in mainxd.iterclues():
progress(mainanswer)
poss_answers = []
pub_uses = {} # [pubid] -> set(ClueAnswer)
mainca = ClueAnswer(mainpubid, maindate, mainanswer, mainclue)
# find other uses of this clue, and other answers, in a single pass
for clueans in find_clue_variants(mainclue):
if clueans.answer != mainanswer:
poss_answers.append(clueans)
if clueans.answer == mainanswer:
if clueans.pubid in pub_uses:
otherpubs = pub_uses[clueans.pubid]
else:
otherpubs = set() # set of ClueAnswer
pub_uses[clueans.pubid] = otherpubs
otherpubs.add(clueans)
# bclues is all boiled clues for this particular answer: { [bc] -> #uses }
bclues = load_answers().get(mainanswer, [])
stale_answer = False
if bclues:
uses = []
for bc, nuses in bclues.items():
# then find all clues besides this one
clue_usages = [
ca for ca in load_clues().get(bc, [])
if ca.answer == mainanswer and ca.date < maindate
]
if clue_usages:
stale_answer = True
if nuses > 1:
# only use one (the most recent) ClueAnswer per boiled clue
# but use the clue only (no xdid)
ca = sorted(clue_usages,
key=lambda ca: ca.date or "z")[-1].clue
else:
ca = sorted(clue_usages,
key=lambda ca: ca.date or "z")[-1]
uses.append((ca, nuses))
# summary row to similar.tsv
metadb.append_row(
'gxd/similar',
[
mainxd.xdid(), # xdid
int(100 * sum(
pct / 100.0
for pct, xd1, xd2 in similar_grids)), # similar_grid_pct
nstaleclues, # reused_clues
nstaleanswers, # reused_answers
ntotalclues, # total_clues
" ".join(("%s=%s" % (xd2.xdid(), pct))
for pct, xd1, xd2 in similar_grids) # matches
])