def makepage(path):
u.path.chdir(path[:-8])
os.chdir("data")
crumbs = u.path.breadcrumbs(path)
ltitle = path.split("-")[1].replace("_", " ") + " / " + title
p = path.split("-", 1)[1]
pnum = path.split("-")[-2].split("_")[-1]
sys.stdout.write(u.html.head(ltitle))
sys.stdout.write(
"""
{}
<div class="pgdata">
<h2>{}</h2>
""".format(
crumbs, title
)
)
if os.access("OK", os.F_OK):
sys.stdout.write(
"""
<div class="todo">
<h3>To do</h3>
</div>
"""
)
if os.access("comments.txt", os.F_OK):
sys.stdout.write('<pre class="log">\n')
fp = open("comments.txt", "rt", encoding="utf-8")
for line in fp:
sys.stdout.write(u.html.escape(line))
fp.close()
sys.stdout.write("</pre>\n")
elif os.access("QUEUED", os.F_OK):
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split("-", 1)[1].replace("difitems", "diff")))
sys.stdout.write("\n</div>\n")
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path[:-8])
os.chdir('data')
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.head(ltitle))
sys.stdout.write('''
{}
<div class="pgdata">
<h2>{}</h2>
'''.format(crumbs, title))
if os.access('OK', os.F_OK):
sys.stdout.write('''
<div class="todo">
<h3>To do</h3>
</div>
''')
if os.access('comments.txt', os.F_OK):
sys.stdout.write('<pre class="log">\n')
fp = open('comments.txt', 'rt', encoding='utf-8')
for line in fp:
sys.stdout.write(u.html.escape(line))
fp.close()
sys.stdout.write('</pre>\n')
elif os.access('QUEUED', os.F_OK):
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(
u.html.makeError(
path.split('-', 1)[1].replace('difitems', 'diff')))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path[:-8])
if not os.path.isdir('refmaps'):
os.mkdir('refmaps')
os.chdir('refmaps')
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgrefmaps">
<h2>reference point maps</h2>
'''.format(crumbs))
if os.access('../diff/OK', os.F_OK):
pseudo = False
if os.access('../map/PSEUDOMAP', os.F_OK):
pseudo = True
if os.access('current', os.F_OK):
fp = open('current', 'rt')
current = fp.read().split()
curplace, curmethod = [int(x) for x in current[:2]]
if len(current) > 2:
currev = int(current[2])
else:
currev = 0
fp.close()
else:
curplace = curmethod = currev = 0
if currev:
_colors.reverse()
places = {}
truelabels = {}
truelbl = []
fp = open('../data/labels.txt', 'rt', encoding='iso-8859-1')
fp2 = open('../data/truelabels.txt', 'rt', encoding='utf-8')
for line in fp:
a, b = line.strip().split(None, 1)
places[b] = int(a)
l = fp2.readline().strip()
truelabels[b] = l
truelbl.append(l)
fp2.close()
fp.close()
sys.stdout.write('''
<form action="{}bin/refmapsform" method="post">
<input type="hidden" name="p" value="project_{}">
<fieldset>
<legend></legend>
Place: <select name="pl">
<option value="0">--</option>
'''.format(u.config.appurl, pnum))
for place in sorted(places):
if places[place] == curplace:
sel = ' selected="selected"'
placename = place
else:
sel = ''
sys.stdout.write('<option value="{}"{}>{}</option>\n'.format(
places[place], sel, u.html.escape(truelabels[place])))
sel = [''] * 4
sel[curmethod] = ' selected="selected"'
if currev:
checked = ' checked="checked"'
else:
checked = ''
sys.stdout.write('''
</select><br>
<br>
Method: <select name="m">
<option value="0"{0[0]}>linear distances</option>
<option value="1"{0[1]}>quadratic distances</option>
<option value="2"{0[2]}>zero-based linear distances</option>
<option value="3"{0[3]}>zero-based quadratic distances</option>
</select><br>
<br>
<input type="checkbox" name="revcol"{1}> Reverse colours<br>
<br>
<input type="submit" value="Show map">
</fieldset>
</form>
'''.format(sel, checked))
if curplace:
if not os.access('curmap.eps', os.F_OK):
n, lbls, dif = _difread('../diff/diff.txt')
idx = lbls.index(placename)
diffs = [dif[idx][i] for i in range(n) if i != idx]
fmin = min(diffs)
fmax = max(diffs)
if curmethod == 2 or curmethod == 3:
fmin = 0.0
fp = open('current.rgb', 'wt', encoding='iso-8859-1')
fp.write('3\n{}\n1\n0\n0\n'.format(placename))
for i in range(n):
if i == idx:
continue
f = (dif[idx][i] - fmin) / (fmax - fmin)
if curmethod == 1 or curmethod == 3:
f = f * f
fi = int(f * _ncols)
if fi == _ncols:
fi = _ncols - 1
fp.write('{}\n{}\n'.format(lbls[i], _colors[fi]))
fp.close()
os.system(
'maprgb -r -o curmap.eps ../map/map.cfg current.rgb 2> /dev/null'
)
os.system('$PYTHON3 $APPDIR/util/smappost curmap.eps')
if currev:
os.system('ref2star -r curmap.eps')
else:
os.system('ref2star curmap.eps')
if not pseudo and not os.access('plot01.eps', os.F_OK):
if not os.access('geo.dst', os.F_OK):
os.system('difmodin ../data/labels.txt tmp.tbl')
os.system('difmod -o geo.dst ../map/map.geo tmp.tbl')
os.remove('tmp.tbl')
n2, lbls2, geo = _difread('geo.dst')
fp = open('curplot.data', 'wt')
assert n == n2
for i in range(n):
if i == idx:
continue
fp.write('{} {}\n'.format(geo[idx][i], dif[idx][i]))
fp.close()
fp = open('curplace.txt', 'wt', encoding='utf-8')
fp.write(truelabels[placename] + '\n')
fp.close()
os.system('R --no-save < {}util/refplot.R > plot.log 2>&1'.
format(u.config.appdir))
state = 0
fpin = open('plot01.eps', 'rt', encoding='iso-8859-1')
fpout = open('plot01i.eps', 'wt', encoding='iso-8859-1')
for line in fpin:
if state == 0:
fpout.write(line)
if line.startswith('%%BeginProlog'):
fpout.write('''
/mys 10 string def
/myf (image.coo) (w) file def
/mylog {
2 copy transform
exch 2 {
round cvi mys cvs myf exch writestring myf ( ) writestring
} repeat
myf (\\n) writestring
} bind def
''')
state = 1
elif state == 1:
if line.strip().endswith(' c p1'):
a = line.split()
fpout.write(
'{0[0]} {0[1]} mylog {0[2]} c p1\n'.format(
a))
elif line.startswith('%%EOF'):
fpout.write('myf closefile\n')
fpout.write(line)
state = 2
else:
fpout.write(line)
else:
fpout.write(line)
fpout.close()
fpin.close()
os.system('eps2png > eps2png.log 2>&1')
n = -1
fp = open('image.coo', 'rt')
lines = fp.readlines()
fp.close()
i = 0
fp = open('image.coo', 'wt', encoding='iso-8859-1')
for line in lines:
if i == idx:
i += 1
fp.write(
line.strip() + ' ' +
truelbl[i].encode('iso-8859-1', 'xmlcharrefreplace'
).decode('iso-8859-1') + '\n')
i += 1
fp.close()
os.system('mkmap map2')
sys.stdout.write(
u.html.img('project_{}-refmaps-curmap'.format(pnum),
True,
usemap="map1",
idx=1))
if not pseudo:
fp = open('image.html', 'rt', encoding='utf-8')
sys.stdout.write(fp.read())
fp.close()
sys.stdout.write(
'<p>\n' +
u.html.img('project_{}-refmaps-plot01'.format(pnum),
usemap="map2",
noover=True,
idx=2))
elif os.access('../diff/QUEUED', os.F_OK):
os.chdir('../diff/')
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(
u.html.makeError(path.split('-', 1)[1]).replace('refmaps', 'diff'))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path[:-9])
os.chdir('data')
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgitems">
<h2>{}</h2>
'''.format(crumbs, title))
if os.access('OK', os.F_OK) and os.access(
'../map/OK', os.F_OK) and os.access('../diff/OK', os.F_OK):
sys.stdout.write('''
<div class="info">
The map shows the total amount of data available for each location.
<br> <br>
Below the map is a list of all items — the column labels — in your data set.<br>
The number (if any) says how many values are missing for each item.
<br> <br>
Click on a number to get a map of missing values.
</div>
<table class="items" cellspacing="0" cellpadding="0" border="0">
''')
if not os.access('datacount.txt', os.F_OK):
truelabels = {}
fp1 = open('labels.txt', 'rt', encoding='iso-8859-1')
fp2 = open('truelabels.txt', 'rt', encoding='utf-8')
for line in fp1:
lbl = line.split(None, 1)[1].strip()
truelabels[lbl] = fp2.readline().strip()
fp2.close()
fp1.close()
p = {}
fp = open('table.txt', 'rt', encoding='iso-8859-1')
fp.readline()
for line in fp:
i, j = line.rsplit('"', 1)
lbl = re.sub('\\\\(.)', '\\1', i[1:])
n = len([True for i in j.split() if i != 'NA'])
p[lbl] = n
fp.close()
fp = open('datacount.txt', 'wt', encoding='utf-8')
for i in sorted(p):
fp.write('{:6d}\t{}\n'.format(p[i], truelabels[i]))
fp.close()
m = max(p.values())
m *= m
pp = {}
for i in p:
p[i] *= p[i]
pp[i] = m
os.chdir('..')
u.distribute.distmap(p, pp, 'data/datacount')
os.chdir('data')
p = path.split('-', 1)[1].replace('numitems', 'data')
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.img(p + '-datacount', usemap="map1", bw=True))
sys.stdout.write('''
→ <a href="{}bin/getdatacount?{}" target="_blank">download as list</a>
<p>
'''.format(u.config.appurl, pnum))
lines = []
fp = open('NAs.txt', 'rt', encoding='utf-8')
n = -1
for line in fp:
n += 1
a = line.split('\t')
lines.append((a[1].strip(), int(a[0]), n))
fp.close()
lines.sort()
for item, i, n in lines:
if i == 0:
i = ''
else:
i = '<a href="namap?{}-{}" target="_blank">{}</a>'.format(
pnum, n, i)
sys.stdout.write('<tr><td align="right">{}<td>{}\n'.format(
i, u.html.escape(item.strip())))
sys.stdout.write('</table>\n')
elif os.access('../diff/QUEUED', os.F_OK) or os.access(
'../map/QUEUED', os.F_OK):
os.chdir('../diff')
sys.stdout.write(u.html.busy())
else:
if os.access('../diff/QUEUED', os.F_OK):
p = 'map'
else:
p = 'diff'
sys.stdout.write(
u.html.makeError(path.split('-', 1)[1].replace('numitems', p)))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path[:-8])
if not os.path.isdir('distmap'):
os.mkdir('distmap')
os.chdir('distmap')
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgdistmap">
<h2>distribution maps</h2>
'''.format(crumbs))
if not os.access('items.txt', os.F_OK):
items = []
for filename in os.listdir('../data/_'):
if not filename.endswith('.data'):
continue
fname = filename.replace('.data', '')
iname = re.sub('_([0-9]+)_', _num2chr, fname)
items.append((iname.lower(), fname, iname))
fp = open('items.txt', 'wt', encoding='utf-8')
for i, j, k in sorted(items):
fp.write('{}\t{}\n'.format(j, k))
fp.close()
if not os.access('current.txt', os.F_OK):
open('current.txt', 'wt').close()
fp = open('current.txt', 'rt', encoding='utf-8')
current = [x.strip() for x in fp.read().strip().split('\n')]
fp.close()
currentitem = ''
currentvariants = set()
currentregex = ''
if len(current) > 0:
currentitem = current[0]
if len(current) > 1:
currentvariants = set(current[1].split('\t'))
if len(current) > 2:
currentregex = current[2]
REGEX = re.compile(currentregex)
sys.stdout.write('''
<form action="{}bin/distmapform" method="post" enctype="multipart/form-data">
<fieldset><legend></legend>
<input type="hidden" name="hebci_auml" value="ä">
<input type="hidden" name="hebci_divide" value="÷">
<input type="hidden" name="hebci_euro" value="€">
<input type="hidden" name="hebci_middot" value="·">
<input type="hidden" name="hebci_oelig" value="œ">
<input type="hidden" name="hebci_oslash" value="ø">
<input type="hidden" name="hebci_Scaron" value="Š">
<input type="hidden" name="hebci_sect" value="§">
<input type="hidden" name="hebci_thorn" value="þ">
<input type="hidden" name="p" value="project_{}">
<input type="hidden" name="var" value="">
Item: <select name="item">
<option>--</option>
'''.format(u.config.appurl, pnum))
found = False
fp = open('items.txt', 'rt', encoding='utf-8')
for line in fp:
a, b = line.strip().split('\t')
if a == currentitem:
sel = ' selected="selected"'
currenthtml = u.html.escape(b)
found = True
else:
sel = ''
sys.stdout.write('<option value="{}"{}>{}</option>\n'.format(
a, sel, u.html.escape(b)))
fp.close()
sys.stdout.write('''
</select>
<input type="submit" value="Select item">
</fieldset>
</form>
''')
if not found:
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
return
if os.access('../data/UTF', os.F_OK):
encoding = 'utf-8'
else:
encoding = 'iso-8859-1'
if not os.access('currentlist.txt', os.F_OK):
variants = {}
fp = open('../data/_/' + currentitem + '.data', 'rb')
for line in fp:
if line[:1] == b'-':
v = line[1:].strip().decode(encoding)
if not v in variants:
variants[v] = 0
variants[v] += 1
fp.close()
fp = open('currentlist.txt', 'wt', encoding='utf-8')
for v in sorted(variants):
fp.write('{}\t{}\n'.format(variants[v], v))
fp.close()
fp = open('currentlist.txt', 'rt', encoding='utf-8')
lines = fp.readlines()
fp.close()
sys.stdout.write('''
<p>
<form action="{}bin/distmapform" method="post" enctype="multipart/form-data">
<fieldset>
<legend></legend>
<input type="hidden" name="p" value="project_{}">
<input type="hidden" name="item" value="{}">
There are {} variants for <em>{}</em><br>
Select one or more variants:<br><select name="var" multiple="multiple" size="10" class="ipa2">
'''.format(u.config.appurl, pnum, currentitem, len(lines), currenthtml))
for line in lines:
ii, b = line.strip().split('\t', 1)
a = _iname(b)
if a in currentvariants:
sel = ' selected="selected"'
found = True
else:
sel = ''
sys.stdout.write('<option value="{}"{}>{} ({})</option>\n'.format(
a, sel, u.html.escape(b), ii))
sys.stdout.write('''
</select><br>
<br>
— or —<br>
<br>
Regular expression:{}
<input type="hidden" name="hebci_auml" value="ä">
<input type="hidden" name="hebci_divide" value="÷">
<input type="hidden" name="hebci_euro" value="€">
<input type="hidden" name="hebci_middot" value="·">
<input type="hidden" name="hebci_oelig" value="œ">
<input type="hidden" name="hebci_oslash" value="ø">
<input type="hidden" name="hebci_Scaron" value="Š">
<input type="hidden" name="hebci_sect" value="§">
<input type="hidden" name="hebci_thorn" value="þ">
<input type="text" name="regex" size="60" class="ipa2">
<br>
<br>
<input type="submit" value="Show distribution map">
</fieldset>
</form>
'''.format(u.html.help('regex')))
if not (currentvariants or currentregex):
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
return
item = re.sub('_([0-9]+)_', _num2chr, currentitem)
variantset = set(
[re.sub('_([0-9]+)_', _num2chr, x) for x in currentvariants])
if not os.access('distmap.eps', os.F_OK):
placen = {}
placeall = {}
variants = {}
fp = open('../data/_/{}.data'.format(currentitem), 'rb')
for line in fp:
if line[:1] == b':':
place = line[1:].decode('iso-8859-1').strip()
if not place in placen:
placen[place] = 0
placeall[place] = 0
elif line[:1] == b'-':
if currentregex:
l = line[1:].strip().decode(encoding)
if REGEX.search(l):
placen[place] += 1
if not l in variants:
variants[l] = 0
variants[l] += 1
elif line[1:].strip().decode(encoding) in variantset:
placen[place] += 1
placeall[place] += 1
fp.close()
os.chdir('..')
u.distribute.distmap(placen,
placeall,
'distmap/distmap',
normalise=False)
os.chdir('distmap')
if currentregex:
fp = open('currentvariants.txt', 'wt', encoding='utf-8')
for i in sorted(variants):
fp.write('{} ({})\n'.format(i, variants[i]))
fp.close()
if currentregex:
v = currentregex
r = ' RE'
else:
if len(currentvariants) == 1:
v = list(variantset)[0]
r = ''
else:
v = '*'
r = ' SET'
sys.stdout.write('''
<h3>Distribution map for{} "<span class="ipa">{}</span>" in {}</h3>
'''.format(r, u.html.escape(v), u.html.escape(item)))
if currentregex:
sys.stdout.write('<ul>\n')
fp = open('currentvariants.txt', 'rt', encoding='utf-8')
for line in fp:
sys.stdout.write(
'<li><span class="ipa2">{0[0]}</span> {0[1]}\n'.format(
u.html.escape(line.strip()).rsplit(None, 1)))
fp.close()
sys.stdout.write('</ul>\n')
sys.stdout.write(
u.html.img('project_{}-distmap-distmap'.format(pnum),
True,
usemap="map1"))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
project = path.split('-')[1]
# pnum = path.split('-')[-2].split('_')[-1]
if os.access('tip.html', os.F_OK):
tipfile = 'tip.html'
else:
tipfile = '../map/image.html'
sys.stdout.write(
u.html.head(ltitle, tip=True, maptip=True, tipfile=tipfile))
sys.stdout.write('''<script language="JavaScript">
function checkmanualclustering() {
var e = document.cluster.mthd;
var v = e.options[e.selectedIndex].value;
mform = document.getElementById("manualcluster");
mform.style.visibility = "hidden";
mform.style.display = "none";
if (v == "man") {
mform.style.visibility = "visible";
mform.style.display = "block";
mform.style.border = "thin dotted gray";
mform.style.margin = "5px 0px 5px 0px";
document.cluster.n.value="2";
}
}
function checkdeterminantoptions() {
var e = document.determinant.detmethod;
var v = e.options[e.selectedIndex].value;
var shib = document.getElementById('shibopts')
var imp = document.getElementById('impopts')
shib.style.visibility = 'hidden';
shib.style.display = 'none';
imp.style.visibility = 'hidden';
imp.style.display = 'none';
if (v == "shibboleth") {
shib.style.visibility = 'visible';
shib.style.display = 'block';
} else if (v == "importance") {
imp.style.visibility = 'visible';
imp.style.display = 'block';
}
}
function moverows(list1, list2)
{
var sel='';
var seltext='';
for (i = list1.options.length - 1; i >= 0; i--) {
if (list1.options[i].selected == true) {
sel=list1.options[i].value;
seltext=list1.options[i].text;
var newrow = new Option(seltext,sel);
list2.options[list2.length]=newrow;
list1.options[i]=null;
}
}
}
function submitclusterform()
{
var form = document.cluster;
var m = form.mthd.options[form.mthd.selectedIndex].value;
var outlist = form.labelsout;
var inlist = form.labelsin;
if (m == "man") {
form.lin.value = "";
form.lout.value = "";
for (i = inlist.options.length - 1; i >= 0; i--) {
form.lin.value += inlist.options[i].value + "#";
}
for (i = outlist.options.length - 1; i >= 0; i--) {
form.lout.value += outlist.options[i].value + "#";
}
}
form.submit();
}
</script>
''')
sys.stdout.write('''
{}
<div class="pgcludet">
<h2>cluster determinants</h2>
'''.format(crumbs))
if os.access('OK', os.F_OK):
sys.stdout.write('<h3 id="s1">Step 1: cluster</h3>\n')
if os.access('cluster-params', os.F_OK):
fp = open('cluster-params', 'rt')
method, groups = fp.read().split()
fp.close()
elif os.access('../clusters/current.txt', os.F_OK):
fp = open('../clusters/current.txt', 'rt')
method, groups, col = fp.read().split()
fp.close()
else:
method, groups = ('wa', 2)
fp = open('../data/Method', 'rt')
project_type = fp.read().strip()
fp.close()
sys.stdout.write('''<p>
<form name="cluster" action="{}bin/cludetform" method="post" enctype="multipart/form-data">
<input type="hidden" name="p" value="{}">
<input type="hidden" name="action" value="cluster">
<fieldset style="line-height:150%"><legend>change parameters</legend>
Clustering method:
<select name="mthd" onchange="checkmanualclustering()">
'''.format(u.config.appurl, project))
methods['man'] = "Manual"
for i in sorted(methods):
sys.stdout.write('<!--{}:{} -->\n'.format(i, methods[i]))
if i == method:
sys.stdout.write(
'<option selected="selected" value="{}">{}</option>\n'.
format(i, methods[i]))
else:
sys.stdout.write('<option value="{}">{}</option>\n'.format(
i, methods[i]))
sys.stdout.write('''
</select><br>
Number of clusters:
<select name="n">
''')
n = int(groups)
maxnum = min(13,
int(open('../data/stats.txt', 'rt').read().split()[0]))
for i in range(2, maxnum):
if i == n:
sys.stdout.write(
'<option selected="selected">{}</option>\n'.format(i))
else:
sys.stdout.write('<option>{}</option>\n'.format(i))
try:
fp = open('currentcl', 'rt')
curclnum = int(fp.read().rstrip())
fp.close()
except:
curclnum = 1
if os.access('UTF', os.F_OK):
enc = 'utf-8'
else:
enc = 'iso-8859-1'
labels_in = []
labels_out = []
if os.access('clgroups.txt', os.F_OK):
fp = open('clgroups.txt', 'r', encoding=enc)
for line in fp:
(gr, label) = line.split(None, 1)
label = label.strip().strip('"')
if int(gr) == curclnum:
labels_in.append(label)
else:
labels_out.append(label)
else:
fp = open('../data/labels.txt', 'r', encoding=enc)
for line in fp:
label = line.split(None, 1)[1].strip()
labels_out.append(label)
sys.stdout.write('</select><br>')
moptvisible = '"visibility:hidden;display:none"'
if method == 'man':
moptvisible = '"border:thin dotted gray;margin:5px 0px 5px 0px;visibility:visible;display:block"'
sys.stdout.write('''
<div id="manualcluster" style={}>
<input type="hidden" name="lin" value="x">
<input type="hidden" name="lout" value="y">
<table border="0">
<tr><th>Labels outside the cluster</th>
<th></th>
<th>Labels in the cluster</th>
<tr>
<td><select name="labelsout" style="width:25em" size="10" multiple>
'''.format(moptvisible))
for l in labels_out:
sys.stdout.write('<option value="{}">{}</option>\n'.format(l, l))
sys.stdout.write('''
<td valign="middle">
<input type="Button" value=">>"
onClick="moverows(document.cluster.labelsout,document.cluster.labelsin)"><br>
<input type="Button" value="<<"
onClick="moverows(document.cluster.labelsin,document.cluster.labelsout)"><br>
</td>
<td><select name="labelsin" style="width:25em" size="10" multiple>
''')
for l in labels_in:
sys.stdout.write('<option value="{}">{}</option>\n'.format(l, l))
sys.stdout.write('''
</select>
</td>
</tr>
</table>
</div>
''')
sys.stdout.write(
'<input type="button" value="Cluster" onclick="submitclusterform()"></fieldset></form>'
)
if os.access('clmap.png', os.F_OK):
sys.stdout.write(
u.html.img(
p + '-clmap', usemap="map1", idx=1, pseudoforce=True) +
'\n')
sys.stdout.write('''
<h3 id="s2">Step 2: select cluster and determinant options</h3>
<form action="{}bin/cludetform" method="post"
name="determinant" enctype="multipart/form-data">
<input type="hidden" name="p" value="{}">
<input type="hidden" name="action" value="determinant">
'''.format(u.config.appurl, project))
sys.stdout.write('<fieldset style="line-height:150%">\n')
sys.stdout.write('<legend>determinant options</legend>\n')
sys.stdout.write('Cluster of interest: ')
for i in range(1, n + 1):
if i == curclnum:
c = ' checked="checked"'
else:
c = ''
sys.stdout.write('''
<span class="s{0}">
<input type="radio" name="c" value="{0}"{1}>
</span>\n
'''.format(i, c))
if project_type.startswith('lev'):
det_methods = {
'shibboleth': 'Distance based',
'importance': 'Categorical'
}
else:
det_methods = {'shibboleth': 'Distance based'}
if os.access('clusterdet-method', os.F_OK):
fp = open('clusterdet-method', 'r')
det_method = fp.read().strip()
fp.close()
else:
det_method = "shibboleth"
sys.stdout.write('''<br>Determinant:
<select name="detmethod" onchange="checkdeterminantoptions()">
''')
selected = ""
for i in sorted(det_methods):
if i == det_method:
selected = ' selected'
else:
selected = ""
sys.stdout.write('''<option{} value="{}">{}</option>
'''.format(selected, i, det_methods[i]))
sys.stdout.write('</select>{}\n'.format(u.html.help('cludetmethod')))
norm_method = 'zscore'
na_val = 0.8
opt_diff = True
if det_method == 'shibboleth':
optvisible = ' style="visibility:visible;display:block"'
if (os.access('clusterdet-params', os.F_OK)):
fp = open('clusterdet-params', 'r')
line = fp.read().strip()
fp.close()
norm_method = re.search('--norm=(\S+)', line).group(1)
na_val = re.search('--ignore-na=([0-9.-]+)', line).group(1)
opt_diff = re.search('--diff', line)
else:
optvisible = ' style="visibility:hidden;display:none"'
sys.stdout.write(' <div id="shibopts"{}>'.format(optvisible))
if (opt_diff):
ratiosel = ''
else:
ratiosel = 'checked'
if norm_method == 'none':
nselected = "selected"
zselected = ""
else:
zselected = "selected"
nselected = ""
# sys.stdout.write('''<br>Normalization:
# <select name="norm">
# <option value="none" {}>none</option>
# <option value="zscore" {}>z-score</option>
# </select>
# '''.format(nselected, zselected))
sys.stdout.write('''<br>Missing values:
<input type="text" size=3 name="narate" value={}>{}
'''.format(na_val, u.html.help('cludetmethod#MissingData')))
# sys.stdout.write('''<br>Use ratio?
# <input type="checkbox" name="ratio" value="ratio" {}>
# if checked overall scores is `between / within', otherwise
# `between - within'.
# '''.format(ratiosel))
sys.stdout.write('</div>\n')
if det_method == 'importance':
optvisible = ' style="visibility:visible;display:block"'
else:
optvisible = ' style="visibility:hidden;display:none"'
sys.stdout.write('''
<div id="impopts"{} class="accents">
These characters are ignored, unless checked:
<div class="ipa2">
'''.format(optvisible))
accents = {}
if (os.access('accents.txt', os.F_OK)):
fp = open('accents.txt', 'rt')
for line in fp:
accents[int(line)] = False
fp.close()
if (os.access('accentscurrent.txt', os.F_OK)):
fp = open('accentscurrent.txt', 'rt')
for line in fp:
accents[int(line)] = True
fp.close
else:
optvisible = False
if accents:
for i in sorted(accents):
if i == 32:
s = 'SPACE'
else:
s = ' &#{}; '.format(i)
if accents[i]:
v = ' checked="checked"'
else:
v = ''
nm = unicodedata.name('{:c}'.format(i), '')
if nm:
a1 = '<abbr title="{}">'.format(u.html.escape(nm))
a2 = '</abbr>'
else:
a1 = a2 = ''
sys.stdout.write('''<span class="cdc">{}
<input type="checkbox" name="chr{}"{}> {}{}
</span>\n'''.format(a1, i, v, s, a2))
sys.stdout.write('</div></div>' '')
sys.stdout.write('''<br>
<input type="submit" value="Select"><br>
</fieldset>
</form>
<p>
''')
# step 3
if (os.access('score.txt', os.F_OK)):
sys.stdout.write('''
<h3 id="s3">Step 3: select item</h3>
<form action="{}bin/cludetform" method="post" enctype="multipart/form-data">
<input type="hidden" name="p" value="{}">
<input type="hidden" name="action" value="item">
Items sorted by value:
<select name="item">
'''.format(u.config.appurl, project))
try:
fp = open('currentitem', 'rt')
curitem = fp.read().rstrip()
fp.close()
except:
curitem = ''
if det_method == 'importance':
score_name = 'Importance'
wtn_name = 'Representativeness'
btw_name = 'Distinctiveness'
else:
score_name = 'Score'
wtn_name = 'Within difference'
btw_name = 'Between difference'
fp = open('score.txt', 'rt')
for line in fp:
r, w, b, f = line.split("\t")
item = f
if item == curitem:
sel = ' selected="selected"'
else:
sel = ''
sys.stdout.write('''
<option value="{}"{}>{:.2f} ({:.2f} : {:.2f}) {}
</option>\n
'''.format(item, sel, float(r), float(w), float(b),
_toStrHtml(item)))
sys.stdout.write('''
</select>
<input type="submit" value="Select item">
<br>The format of the select list is '{} ({} : {}) item.
<br>→
<a href="cludetlist?p={}&t=success" target="_blank">download as list
</a><br>
</form>
<p>
'''.format(score_name, wtn_name, btw_name, project))
if (os.access('score-failed.txt', os.F_OK)):
fp = open('score-failed.txt')
linecount = 0
for line in fp:
linecount = linecount + 1
if (linecount > 0):
sys.stdout.write('''
{} items are ignored due to missing data (→
<a href="cludetlist?p={}&t=fail" target="_blank">
download as list</a>)
'''.format(linecount, project))
if (os.access('currentlist.txt', os.F_OK)):
fp = open('currentitem', 'rt')
curitem = fp.read().strip()
fp.close()
fp = open('score.txt')
for line in fp:
r, wtn, btw, item = line.strip().split()
if item == curitem:
break
fp.close()
sys.stdout.write('''
Current item: {}
<table cellspacing="0" cellpadding="0" border="0">
<tr><td>{}: <td>{}
<tr><td>— {}: <td>{}
<tr><td>— {}: <td>{}
</table>
'''.format(_toStrHtml(curitem), score_name, r, wtn_name, wtn,
btw_name, btw))
if (det_method == 'shibboleth'
and os.access('itemmap.png', os.F_OK)):
sys.stdout.write('''<br>Difference map for <i>{}</i>
(note: darker link color means higher difference)<br>
'''.format(curitem))
sys.stdout.write(
u.html.img(
p +
'-itemmap', usemap="map1", idx=1, pseudoforce=True) +
'\n')
if project_type.startswith('lev'):
# step 4
if (os.access('currentitem', os.F_OK)):
sys.stdout.write('''
<h3 id="s4">Step 4: show the distribution of relevant forms</h3>
<form action="{}bin/cludetform" method="post"
enctype="multipart/form-data">
<input type="hidden" name="p" value="{}">
<input type="hidden" name="action" value="formdist">
<table><tr><th style="padding-right:2em">Patterns in the cluster
</th>
<th>Patterns not in the cluster
</th>
</tr>
'''.format(u.config.appurl, project))
formsin = {}
formsout = {}
# if (os.access('selectedforms.txt', os.F_OK)):
# fp = open("currentlist.txt", "rt")
# for line in
fp = open("currentlist.txt", "rt")
for line in fp:
cin, cout, form = line.strip().split('\t')
if int(cin) == 0:
formsout[form] = (cin, cout)
else:
formsin[form] = (cin, cout)
fp.close()
selectedforms = set()
if (os.access('selectedforms.txt', os.F_OK)):
fp = open('selectedforms.txt', 'rt')
for line in fp:
selectedforms.add(_toStrHtml(line.strip()))
select_len = len(formsin)
if select_len > 10: select_len = 10
sys.stdout.write('''
<tr valign="top"> <td>
<select name="formsin" multiple="multiple" size="{}" class="ipaw">
'''.format(select_len))
for form, (cin, cout) in sorted(
formsin.items(),
key=lambda x: int(x[1][0]) / int(x[1][1]),
reverse=True):
if (form in selectedforms): sel = ' selected="selected"'
else: sel = ''
sys.stdout.write('''<!-- {} -->\n'''.format(form))
sys.stdout.write(
'''<option value="{}"{}>{} ({}:{})</option>\n
'''.format(_iname(form), sel, _toStrHtml(form, True), cin,
cout))
select_len = len(formsout)
if select_len > 10: select_len = 10
sys.stdout.write('''</select></td><td>\n
<select name="formsout" multiple="multiple" size="{}" class="ipa2">
'''.format(select_len))
for form, (cin, cout) in formsout.items():
if (form in selectedforms): sel = ' selected="selected"'
else: sel = ''
sys.stdout.write(
'''<option value="{}"{}>{} ({}:{})</option>\n
'''.format(_iname(form), sel, _toStrHtml(form, True), cin,
cout))
sys.stdout.write('</select></td></tr></table>\n')
sys.stdout.write(
'<input type="submit" value="Show distribution map">\n')
sys.stdout.write('</form>')
if (os.access('distmap.png', os.F_OK)):
sys.stdout.write(
u.html.img(p + '-distmap',
usemap="map1",
idx=1,
pseudoforce=True) + '\n')
elif os.access('QUEUED', os.F_OK):
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]))
sys.stdout.write('</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path[:-6])
if not os.path.isdir('align'):
os.mkdir('align')
os.chdir('align')
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.head(ltitle))
sys.stdout.write('''
{}
<div class="pgalign">
<h2>alignments</h2>
'''.format(crumbs))
if os.access('../data/OK', os.F_OK):
fp = open('../data/Method', 'rt')
m = methods[fp.read().strip()]
fp.close()
sys.stdout.write('Method: {}\n'.format(m))
if os.access('../data/tokens-int.txt', os.F_OK):
features = True
else:
features = False
if not os.access('tokenlist.txt', os.F_OK):
if features:
_tokens2list()
else:
_charcount2list()
if os.access('current', os.F_OK):
fp = open('current', 'rt', encoding='utf-8')
current = fp.readline().strip()
try:
curplace = fp.readline().strip()
except:
curplace = '0'
fp.close()
else:
current = ''
curplace = '0'
sys.stdout.write('''
<form action="{}bin/alignform" method="post">
<input type="hidden" name="p" value="project_{}">
<fieldset>
<legend></legend>
Item: <select name="n">
<option>--</option>
'''.format(u.config.appurl, pnum))
filenames = [x[:-5] for x in os.listdir('../data/_') if x.endswith('.data')]
lines = []
for filename in sorted(filenames):
itemname = re.sub('_([0-9]+)_', _num2chr, filename)
if itemname == current:
sel = ' selected="selected"'
else:
sel = ''
lines.append((itemname.lower(), '<option value="{}"{}>{}</option>\n'.format(filename, sel, u.html.escape(itemname))))
for a, b in sorted(lines):
sys.stdout.write(b)
sys.stdout.write('''
</select><br>
<br>
Place: <select name="l">
<option value="0"> -- random places --</option>
''')
lines = []
truelabels = {}
pseudolabels = {}
fp = open('../data/labels.txt', 'rt', encoding='iso-8859-1')
fp2 = open('../data/truelabels.txt', 'rt', encoding='utf-8')
for line in fp:
a, b = line.split(None, 1)
b = b.strip()
lines.append('{}\t{}'.format(b, a))
l = fp2.readline().strip()
truelabels[b] = l
pseudolabels[l.encode('iso-8859-1', 'xmlcharrefreplace').decode('iso-8859-1')] = l
fp2.close()
fp.close()
lines.sort()
for line in lines:
b, a = line.split('\t', 1)
if a == curplace:
sel = ' selected="selected"'
else:
sel = ''
sys.stdout.write('<option value="{}"{}>{}</option>\n'.format(a, sel, u.html.escape(truelabels[b])))
sys.stdout.write('</select>\n<br> <br>\n<input type="submit" value="Show alignments">\n</fieldset>\n</form>\n')
if current:
sys.stdout.write('<h3>{}</h3>\n'.format(u.html.escape(current)))
try:
fp = open('page', 'rt')
except:
page = 0
pages = 0
fp = open('alignments.txt', 'rb')
else:
page = int(fp.read())
fp.close()
fp = open('pages', 'rt')
pages = int(fp.read())
fp.close()
fp = open('alignments{}.txt'.format(page), 'rb')
if page:
sys.stdout.write('Page: ')
if page == 1:
sys.stdout.write(' «')
else:
sys.stdout.write(' <a href="alignpage?{1}-{0}">«</a>'.format(page - 1, pnum))
for i in range(1, pages + 1):
if i == page:
sys.stdout.write(' <b>{}</b>'.format(i))
else:
sys.stdout.write(' <a href="alignpage?{1}-{0}">{0}</a>'.format(i, pnum))
if page == pages:
sys.stdout.write(' »')
else:
sys.stdout.write(' <a href="alignpage?{1}-{0}">»</a>'.format(page + 1, pnum))
sys.stdout.write('<p>\n')
intab = False
inItem = False
for line in fp:
line = line.strip(b'\n')
if not line:
if intab:
sys.stdout.write('</table>\n')
intab = False
elif line[:1] == b'[':
line = line.decode('iso-8859-1')
lbl = pseudolabels[line.partition(']')[2].strip()]
if inItem:
sys.stdout.write(' — ')
inItem = False
else:
inItem = True
sys.stdout.write(u.html.escape(lbl) + '\n')
else:
line = line.decode('utf-8')
if not intab:
sys.stdout.write('<table class="align">\n')
rownum = 0
intab = True
rownum += 1
if rownum < 3:
sys.stdout.write('<tr class="ipa2">')
else:
sys.stdout.write('<tr>')
prev = '0'
for i in line[1:].split('\t'):
if rownum == 3:
if i == prev:
i = ''
else:
ii = i
i = '{:g}'.format(float(i) - float(prev))
prev = ii
if i == '[[SP]]':
ii = '<span class="space">SP</span>'
else:
ii = u.html.escape(i)
sys.stdout.write('<td> {} \n'.format(ii))
if rownum == 3:
sys.stdout.write('<td class="total"> {} \n'.format(line.split('\t')[-1]))
else:
sys.stdout.write('<td class="white"> \n')
fp.close()
if page:
sys.stdout.write('<p>Page: ')
if page == 1:
sys.stdout.write(' «')
else:
sys.stdout.write(' <a href="alignpage?{1}-{0}">«</a>'.format(page - 1, pnum))
for i in range(1, pages + 1):
if i == page:
sys.stdout.write(' <b>{}</b>'.format(i))
else:
sys.stdout.write(' <a href="alignpage?{1}-{0}">{0}</a>'.format(i, pnum))
if page == pages:
sys.stdout.write(' »')
else:
sys.stdout.write(' <a href="alignpage?{1}-{0}">»</a>'.format(page + 1, pnum))
sys.stdout.write('<p>\n')
elif os.access('../data/QUEUED', os.F_OK):
os.chdir('../data/')
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]).replace('align', 'data'))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
project = path.split('-')[1]
# pnum = path.split('-')[-2].split('_')[-1]
if os.access('tip.html', os.F_OK):
tipfile = 'tip.html'
else:
tipfile = '../map/image.html'
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True, tipfile=tipfile))
sys.stdout.write('''<script language="JavaScript">
function checkmanualclustering() {
var e = document.cluster.mthd;
var v = e.options[e.selectedIndex].value;
mform = document.getElementById("manualcluster");
mform.style.visibility = "hidden";
mform.style.display = "none";
if (v == "man") {
mform.style.visibility = "visible";
mform.style.display = "block";
mform.style.border = "thin dotted gray";
mform.style.margin = "5px 0px 5px 0px";
document.cluster.n.value="2";
}
}
function checkdeterminantoptions() {
var e = document.determinant.detmethod;
var v = e.options[e.selectedIndex].value;
var shib = document.getElementById('shibopts')
var imp = document.getElementById('impopts')
shib.style.visibility = 'hidden';
shib.style.display = 'none';
imp.style.visibility = 'hidden';
imp.style.display = 'none';
if (v == "shibboleth") {
shib.style.visibility = 'visible';
shib.style.display = 'block';
} else if (v == "importance") {
imp.style.visibility = 'visible';
imp.style.display = 'block';
}
}
function moverows(list1, list2)
{
var sel='';
var seltext='';
for (i = list1.options.length - 1; i >= 0; i--) {
if (list1.options[i].selected == true) {
sel=list1.options[i].value;
seltext=list1.options[i].text;
var newrow = new Option(seltext,sel);
list2.options[list2.length]=newrow;
list1.options[i]=null;
}
}
}
function submitclusterform()
{
var form = document.cluster;
var m = form.mthd.options[form.mthd.selectedIndex].value;
var outlist = form.labelsout;
var inlist = form.labelsin;
if (m == "man") {
form.lin.value = "";
form.lout.value = "";
for (i = inlist.options.length - 1; i >= 0; i--) {
form.lin.value += inlist.options[i].value + "#";
}
for (i = outlist.options.length - 1; i >= 0; i--) {
form.lout.value += outlist.options[i].value + "#";
}
}
form.submit();
}
</script>
''')
sys.stdout.write('''
{}
<div class="pgcludet">
<h2>cluster determinants</h2>
'''.format(crumbs))
if os.access('OK', os.F_OK):
sys.stdout.write('<h3 id="s1">Step 1: cluster</h3>\n')
if os.access('cluster-params', os.F_OK):
fp = open('cluster-params', 'rt')
method, groups = fp.read().split()
fp.close()
elif os.access('../clusters/current.txt', os.F_OK):
fp = open('../clusters/current.txt', 'rt')
method, groups, col = fp.read().split()
fp.close()
else:
method, groups = ('wa', 2)
fp = open('../data/Method', 'rt')
project_type = fp.read().strip()
fp.close()
sys.stdout.write('''<p>
<form name="cluster" action="{}bin/cludetform" method="post" enctype="multipart/form-data">
<input type="hidden" name="p" value="{}">
<input type="hidden" name="action" value="cluster">
<fieldset style="line-height:150%"><legend>change parameters</legend>
Clustering method:
<select name="mthd" onchange="checkmanualclustering()">
'''.format(u.config.appurl, project))
methods['man'] = "Manual"
for i in sorted(methods):
sys.stdout.write('<!--{}:{} -->\n'.format(i, methods[i]))
if i == method:
sys.stdout.write('<option selected="selected" value="{}">{}</option>\n'.format(i, methods[i]))
else:
sys.stdout.write('<option value="{}">{}</option>\n'.format(i, methods[i]))
sys.stdout.write('''
</select><br>
Number of clusters:
<select name="n">
''')
n = int(groups)
maxnum = min(13, int(open('../data/stats.txt', 'rt').read().split()[0]))
for i in range(2, maxnum):
if i == n:
sys.stdout.write('<option selected="selected">{}</option>\n'.format(i))
else:
sys.stdout.write('<option>{}</option>\n'.format(i))
try:
fp = open('currentcl', 'rt')
curclnum = int(fp.read().rstrip())
fp.close()
except:
curclnum = 1
if os.access('UTF', os.F_OK):
enc = 'utf-8'
else:
enc = 'iso-8859-1'
labels_in = []
labels_out = []
if os.access('clgroups.txt', os.F_OK):
fp = open ('clgroups.txt', 'r', encoding=enc)
for line in fp:
(gr, label) = line.split(None, 1)
label = label.strip().strip('"')
if int(gr) == curclnum:
labels_in.append(label)
else:
labels_out.append(label)
else:
fp = open ('../data/labels.txt', 'r', encoding=enc)
for line in fp:
label = line.split(None, 1)[1].strip()
labels_out.append(label)
sys.stdout.write('</select><br>')
moptvisible = '"visibility:hidden;display:none"';
if method == 'man':
moptvisible = '"border:thin dotted gray;margin:5px 0px 5px 0px;visibility:visible;display:block"';
sys.stdout.write('''
<div id="manualcluster" style={}>
<input type="hidden" name="lin" value="x">
<input type="hidden" name="lout" value="y">
<table border="0">
<tr><th>Labels outside the cluster</th>
<th></th>
<th>Labels in the cluster</th>
<tr>
<td><select name="labelsout" style="width:25em" size="10" multiple>
'''.format(moptvisible))
for l in labels_out:
sys.stdout.write('<option value="{}">{}</option>\n'.format(l, l))
sys.stdout.write('''
<td valign="middle">
<input type="Button" value=">>"
onClick="moverows(document.cluster.labelsout,document.cluster.labelsin)"><br>
<input type="Button" value="<<"
onClick="moverows(document.cluster.labelsin,document.cluster.labelsout)"><br>
</td>
<td><select name="labelsin" style="width:25em" size="10" multiple>
''')
for l in labels_in:
sys.stdout.write('<option value="{}">{}</option>\n'.format(l, l))
sys.stdout.write('''
</select>
</td>
</tr>
</table>
</div>
''')
sys.stdout.write('<input type="button" value="Cluster" onclick="submitclusterform()"></fieldset></form>')
if os.access('clmap.png', os.F_OK):
sys.stdout.write(u.html.img(p + '-clmap', usemap="map1",
idx=1, pseudoforce=True) + '\n')
sys.stdout.write('''
<h3 id="s2">Step 2: select cluster and determinant options</h3>
<form action="{}bin/cludetform" method="post"
name="determinant" enctype="multipart/form-data">
<input type="hidden" name="p" value="{}">
<input type="hidden" name="action" value="determinant">
'''.format(u.config.appurl, project))
sys.stdout.write('<fieldset style="line-height:150%">\n')
sys.stdout.write('<legend>determinant options</legend>\n');
sys.stdout.write('Cluster of interest: ')
for i in range(1, n + 1):
if i == curclnum:
c = ' checked="checked"'
else:
c = ''
sys.stdout.write('''
<span class="s{0}">
<input type="radio" name="c" value="{0}"{1}>
</span>\n
'''.format(i, c))
if project_type.startswith('lev'):
det_methods = {
'shibboleth': 'Distance based',
'importance': 'Categorical'
}
else:
det_methods = {
'shibboleth': 'Distance based'
}
if os.access('clusterdet-method', os.F_OK):
fp = open('clusterdet-method', 'r')
det_method = fp.read().strip()
fp.close()
else:
det_method = "shibboleth"
sys.stdout.write('''<br>Determinant:
<select name="detmethod" onchange="checkdeterminantoptions()">
''')
selected = ""
for i in sorted(det_methods):
if i == det_method:
selected = ' selected'
else:
selected = ""
sys.stdout.write('''<option{} value="{}">{}</option>
'''.format(selected, i, det_methods[i]))
sys.stdout.write('</select>{}\n'.format(u.html.help('cludetmethod')))
norm_method = 'zscore'
na_val = 0.8
opt_diff = True
if det_method == 'shibboleth':
optvisible = ' style="visibility:visible;display:block"'
if (os.access('clusterdet-params', os.F_OK)):
fp = open('clusterdet-params', 'r')
line = fp.read().strip()
fp.close()
norm_method = re.search('--norm=(\S+)', line).group(1)
na_val = re.search('--ignore-na=([0-9.-]+)', line).group(1)
opt_diff = re.search('--diff', line)
else:
optvisible = ' style="visibility:hidden;display:none"'
sys.stdout.write(' <div id="shibopts"{}>'.format(optvisible))
if(opt_diff):
ratiosel=''
else:
ratiosel='checked'
if norm_method == 'none':
nselected = "selected"
zselected = ""
else:
zselected = "selected"
nselected = ""
# sys.stdout.write('''<br>Normalization:
# <select name="norm">
# <option value="none" {}>none</option>
# <option value="zscore" {}>z-score</option>
# </select>
# '''.format(nselected, zselected))
sys.stdout.write('''<br>Missing values:
<input type="text" size=3 name="narate" value={}>{}
'''.format(na_val,u.html.help('cludetmethod#MissingData')))
# sys.stdout.write('''<br>Use ratio?
# <input type="checkbox" name="ratio" value="ratio" {}>
# if checked overall scores is `between / within', otherwise
# `between - within'.
# '''.format(ratiosel))
sys.stdout.write('</div>\n')
if det_method == 'importance':
optvisible = ' style="visibility:visible;display:block"'
else:
optvisible = ' style="visibility:hidden;display:none"'
sys.stdout.write('''
<div id="impopts"{} class="accents">
These characters are ignored, unless checked:
<div class="ipa2">
'''.format(optvisible))
accents = {}
if (os.access('accents.txt', os.F_OK)):
fp = open('accents.txt', 'rt')
for line in fp:
accents[int(line)] = False
fp.close()
if (os.access('accentscurrent.txt', os.F_OK)):
fp = open('accentscurrent.txt', 'rt')
for line in fp:
accents[int(line)] = True
fp.close
else:
optvisible = False
if accents:
for i in sorted(accents):
if i == 32:
s = 'SPACE'
else:
s = ' &#{}; '.format(i)
if accents[i]:
v = ' checked="checked"'
else:
v = ''
nm = unicodedata.name('{:c}'.format(i), '')
if nm:
a1 = '<abbr title="{}">'.format(u.html.escape(nm))
a2 = '</abbr>'
else:
a1 = a2 = ''
sys.stdout.write('''<span class="cdc">{}
<input type="checkbox" name="chr{}"{}> {}{}
</span>\n'''.format(a1, i, v, s, a2))
sys.stdout.write('</div></div>''')
sys.stdout.write('''<br>
<input type="submit" value="Select"><br>
</fieldset>
</form>
<p>
''')
# step 3
if (os.access('score.txt', os.F_OK)):
sys.stdout.write('''
<h3 id="s3">Step 3: select item</h3>
<form action="{}bin/cludetform" method="post" enctype="multipart/form-data">
<input type="hidden" name="p" value="{}">
<input type="hidden" name="action" value="item">
Items sorted by value:
<select name="item">
'''.format(u.config.appurl, project))
try:
fp = open('currentitem', 'rt')
curitem = fp.read().rstrip()
fp.close()
except:
curitem = ''
if det_method == 'importance':
score_name = 'Importance'
wtn_name = 'Representativeness'
btw_name = 'Distinctiveness'
else:
score_name = 'Score'
wtn_name = 'Within difference'
btw_name = 'Between difference'
fp = open('score.txt', 'rt')
for line in fp:
r, w, b, f = line.split("\t")
item = f
if item == curitem:
sel = ' selected="selected"'
else:
sel = ''
sys.stdout.write('''
<option value="{}"{}>{:.2f} ({:.2f} : {:.2f}) {}
</option>\n
'''.format(item, sel,
float(r), float(w), float(b), _toStrHtml(item)))
sys.stdout.write('''
</select>
<input type="submit" value="Select item">
<br>The format of the select list is '{} ({} : {}) item.
<br>→
<a href="cludetlist?p={}&t=success" target="_blank">download as list
</a><br>
</form>
<p>
'''.format(score_name, wtn_name, btw_name, project))
if (os.access('score-failed.txt', os.F_OK)):
fp = open('score-failed.txt')
linecount = 0
for line in fp:
linecount = linecount + 1
if (linecount > 0):
sys.stdout.write('''
{} items are ignored due to missing data (→
<a href="cludetlist?p={}&t=fail" target="_blank">
download as list</a>)
'''.format(linecount, project))
if (os.access('currentlist.txt', os.F_OK)):
fp = open('currentitem', 'rt')
curitem = fp.read().strip()
fp.close()
fp = open('score.txt')
for line in fp:
r, wtn, btw, item = line.strip().split()
if item == curitem:
break
fp.close()
sys.stdout.write('''
Current item: {}
<table cellspacing="0" cellpadding="0" border="0">
<tr><td>{}: <td>{}
<tr><td>— {}: <td>{}
<tr><td>— {}: <td>{}
</table>
'''.format(_toStrHtml(curitem), score_name, r,
wtn_name, wtn,
btw_name, btw))
if (det_method == 'shibboleth'
and os.access('itemmap.png', os.F_OK)):
sys.stdout.write('''<br>Difference map for <i>{}</i>
(note: darker link color means higher difference)<br>
'''.format(curitem))
sys.stdout.write(u.html.img(p + '-itemmap', usemap="map1",
idx=1, pseudoforce=True) + '\n')
if project_type.startswith('lev'):
# step 4
if (os.access('currentitem', os.F_OK)):
sys.stdout.write('''
<h3 id="s4">Step 4: show the distribution of relevant forms</h3>
<form action="{}bin/cludetform" method="post"
enctype="multipart/form-data">
<input type="hidden" name="p" value="{}">
<input type="hidden" name="action" value="formdist">
<table><tr><th style="padding-right:2em">Patterns in the cluster
</th>
<th>Patterns not in the cluster
</th>
</tr>
'''.format(u.config.appurl, project))
formsin = {}
formsout = {}
# if (os.access('selectedforms.txt', os.F_OK)):
# fp = open("currentlist.txt", "rt")
# for line in
fp = open("currentlist.txt", "rt")
for line in fp:
cin, cout, form = line.strip().split('\t')
if int(cin) == 0:
formsout[form] = (cin, cout)
else:
formsin[form] = (cin, cout)
fp.close()
selectedforms = set()
if (os.access('selectedforms.txt', os.F_OK)):
fp = open('selectedforms.txt', 'rt')
for line in fp:
selectedforms.add(_toStrHtml(line.strip()))
select_len = len(formsin)
if select_len > 10: select_len = 10
sys.stdout.write('''
<tr valign="top"> <td>
<select name="formsin" multiple="multiple" size="{}" class="ipaw">
'''.format(select_len))
for form, (cin, cout) in sorted(formsin.items(),
key=lambda x: int(x[1][0])/int(x[1][1]),
reverse=True):
if (form in selectedforms): sel=' selected="selected"'
else: sel = ''
sys.stdout.write('''<!-- {} -->\n'''.format(form))
sys.stdout.write('''<option value="{}"{}>{} ({}:{})</option>\n
'''.format(_iname(form), sel, _toStrHtml(form, True), cin, cout))
select_len = len(formsout)
if select_len > 10: select_len = 10
sys.stdout.write('''</select></td><td>\n
<select name="formsout" multiple="multiple" size="{}" class="ipa2">
'''.format(select_len))
for form, (cin, cout) in formsout.items():
if (form in selectedforms): sel=' selected="selected"'
else: sel = ''
sys.stdout.write('''<option value="{}"{}>{} ({}:{})</option>\n
'''.format(_iname(form), sel, _toStrHtml(form, True), cin, cout))
sys.stdout.write('</select></td></tr></table>\n')
sys.stdout.write('<input type="submit" value="Show distribution map">\n')
sys.stdout.write('</form>')
if (os.access('distmap.png', os.F_OK)):
sys.stdout.write(u.html.img(p + '-distmap', usemap="map1",
idx=1, pseudoforce=True) + '\n')
elif os.access('QUEUED', os.F_OK):
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]))
sys.stdout.write('</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
project = path.split('-')[1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgmdsmaps">
<h2>mds maps</h2>
'''.format(crumbs))
if os.access('OK', os.F_OK):
corr = _getcor('standard03c.cor')
sys.stdout.write('''
{}
<small>{}</small>
<p>
Above: Classical MDS in three dimensions mapped onto RGB color space.
<p>
Below: Classical MDS into multiple dimensions, one map per dimension.
'''.format(u.html.img(p + '-standard', usemap='map1', idx=999), corr))
for i in range(1, 7):
if not os.access('standard{:02d}.eps'.format(i), os.F_OK):
continue
cor1 = _getcor('standard{:02d}.cor'.format(i))
cor2 = _getcor('standard{:02d}c.cor'.format(i))
stress = _getcor('standard{:02d}.stress'.format(i), 'stress')
sys.stdout.write('\n<p>\n' + u.html.img(
p + '-standard{:02d}'.format(i), True, usemap='map1', idx=i))
if i == 1:
e = 'st'
elif i == 2:
e = 'nd'
elif i == 3:
e = 'rd'
else:
e = 'th'
sys.stdout.write('<small>{}{} dimension: {}'.format(i, e, cor1))
if i > 1:
sys.stdout.write('<br>\n{} dimensions: {}'.format(i, cor2))
if stress:
if i > 1:
pl = 's'
else:
pl = ''
sys.stdout.write('<br>\n{} dimension{}: {}'.format(
i, pl, stress))
sys.stdout.write('</small>\n')
sys.stdout.write('<p>\n' + u.html.img(p + '-plot01'))
if os.access('plot02.eps', os.F_OK):
sys.stdout.write('<p>\n' + u.html.img(p + '-plot02'))
elif os.access('QUEUED', os.F_OK):
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
project = path.split('-')[1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgclusters">
<h2>{}</h2>
'''.format(crumbs, title))
if os.access('OK', os.F_OK):
sys.stdout.write('''
<div class="info">
Warning: the type of clustering used on this page can be unreliable.<br>
Go to <a href="goto?p={}-clumaps">this page</a> to test the validity of the clusters you see here.
</div>
'''.format(project))
fp = open('current.txt', 'rt')
method, groups, col = fp.read().split()
fp.close()
sys.stdout.write('''
{}
<p>
{}
'''.format(u.html.img('{}-map{}{}{}'.format(p, method, groups, col), usemap="map1"),
u.html.img('{}-den{}{}{}'.format(p, method, groups, col))))
sys.stdout.write('''
<p>
<form action="{}bin/clusterform" method="post">
<fieldset><legend></legend>
<input type="hidden" name="p" value="{}">
Clustering method:
<select name="mthd">
'''.format(u.config.appurl, project))
for i in sorted(methods):
if i == method:
sys.stdout.write('<option selected="selected" value="{}">{}</option>\n'.format(i, methods[i]))
else:
sys.stdout.write('<option value="{}">{}</option>\n'.format(i, methods[i]))
sys.stdout.write('''
</select><br>
<br>
Number of clusters:
<select name="n">
''')
n = int(groups)
maxnum = min(13, int(open('../data/stats.txt', 'rt').read().split()[0]))
for i in range(2, maxnum):
if i == n:
sys.stdout.write('<option selected="selected">{}</option>\n'.format(i))
else:
sys.stdout.write('<option>{}</option>\n'.format(i))
sys.stdout.write('''
</select><br>
<br>
Use color:
<select name="col">
''')
for i in sorted(colors):
if i == col:
sys.stdout.write('<option selected="selected" value="{}">{}</option>\n'.format(i, colors[i]))
else:
sys.stdout.write('<option value="{}">{}</option>\n'.format(i, colors[i]))
sys.stdout.write('''
</select><br>
<br>
<input type="submit" value="Change settings">
</fieldset>
</form>
<p>
''')
if col == 'bw':
groups = 1
sys.stdout.write(u.html.img('{}-den{}{}{}alt'.format(p, method, groups, col)))
elif os.access('QUEUED', os.F_OK):
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]))
if os.access('reset', os.F_OK):
for filename in os.listdir('.'):
if filename.startswith('current') or filename.startswith('tmp'):
os.remove(filename)
open('OK', 'wt').close()
sys.stdout.write('<p>\n<a href="">Continue</a>\n')
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path[:-6])
if not os.path.isdir('items'):
os.mkdir('items')
os.chdir('items')
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgitems">
<h2>{}</h2>
'''.format(crumbs, title))
if os.access('../data/OK', os.F_OK) and os.access('../map/OK', os.F_OK):
items = []
method = open('../data/Method', 'rt').read().strip()
sys.stdout.write('''
<div class="info">
The map shows the total amount of data available for each location.
<br> <br>
Below the map is a list of all items — the column labels — in your data set.<br>
The number says how many instances are available for each item.
''')
if method.startswith('levfeat'):
sys.stdout.write('''<br> <br>
If there are two numbers for an item, the first is the number of instances you supplied,<br>
the second number is the number of instances that are actually used.
{}
'''.format(u.html.more('items')))
sys.stdout.write('''
<br> <br>
Click on a number to get a data map for a single item.
</div>
''')
if not os.access('datacount.txt', os.F_OK):
truelabels = {}
fp1 = open('../data/labels.txt', 'rt', encoding='iso-8859-1')
fp2 = open('../data/truelabels.txt', 'rt', encoding='utf-8')
for line in fp1:
lbl = line.split(None, 1)[1].strip()
truelabels[lbl] = fp2.readline().strip()
fp2.close()
fp1.close()
if method.startswith('levfeat'):
e = '.ftr'
else:
e = '.data'
p = {}
pn = {}
for filename in os.listdir('../data/_'):
if not filename.endswith(e):
continue
pseen = set()
fp = open('../data/_/' + filename, 'rb')
for line in fp:
if line[:1] == b':':
lbl = line[1:].decode('iso-8859-1').strip()
if not lbl in p:
p[lbl] = 0
if not lbl in pn:
pn[lbl] = 0
elif line[:1] == b'-' or line[:1] == b'+':
p[lbl] += 1
if not lbl in pseen:
pn[lbl] += 1
pseen.add(lbl)
fp.close()
fp = open('datacount.txt', 'wt', encoding='utf-8')
for i in sorted(p):
fp.write('{:6d}\t{:6d}\t{}\n'.format(pn[i], p[i], truelabels[i]))
fp.close()
m = max(pn.values())
m *= m
pp = {}
for i in pn:
pn[i] *= pn[i]
pp[i] = m
os.chdir('..')
u.distribute.distmap(pn, pp, 'items/datacount')
os.chdir('items')
p = path.split('-', 1)[1]
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.img(p + '-datacount', usemap="map1", bw=True))
sys.stdout.write('''
→ <a href="{}bin/getdatacount?{}" target="_blank">download as list</a>
<p>
'''.format(u.config.appurl, pnum))
if not os.access('list2.utxt', os.F_OK):
for filename in os.listdir('../data/_'):
if not filename.endswith('.data'):
continue
item = re.sub('_([0-9]+)_', _num2chr, filename[:-5])
n = 0
fp = open('../data/_/' + filename, 'rb')
for line in fp:
if line[:1] == b'-' or line[:1] == b'+':
n += 1
fp.close()
if method.startswith('levfeat'):
m = 0
fp = open('../data/_/' + filename + '.ftr', 'rb')
for line in fp:
if line[:1] == b'+':
m += 1
fp.close()
items.append((item.lower(), item, n, m))
else:
items.append((item.lower(), item, n))
fp = open('list2.utxt', 'wt', encoding='utf-8')
if method.startswith('levfeat'):
for k, item, n, m in sorted(items):
fp.write('{}\t{}\t{}\n'.format(n, m, item))
else:
for k, item, n in sorted(items):
fp.write('{}\t{}\n'.format(n, item))
fp.close()
sys.stdout.write('<table class="items" cellspacing="0" cellpadding="0" border="0">\n')
fp = open('list2.utxt', 'rt', encoding='utf-8')
i = -1
if method.startswith('levfeat'):
for line in fp:
i += 1
n, m, item = line.split(None, 2)
if n == m:
n = ''
if n != '' and n != '0':
n = '<a href="imap?{}-{}-0" target="_blank">{}</a>'.format(pnum, i, n)
if m != '0':
m = '<a href="imap?{}-{}-1" target="_blank">{}</a>'.format(pnum, i, m)
sys.stdout.write('<tr><td align="right">{}<td align="right">{}<td>{}\n'.format(n, m, u.html.escape(item)))
else:
for line in fp:
i += 1
n, item = line.split(None, 1)
if n != '0':
n = '<a href="imap?{}-{}-0" target="_blank">{}</a>'.format(pnum, i, n)
sys.stdout.write('<tr><td align="right">{}<td>{}\n'.format(n, u.html.escape(item)))
fp.close()
sys.stdout.write('</table>\n')
elif os.access('../data/QUEUED', os.F_OK) or os.access('../map/QUEUED', os.F_OK):
os.chdir('../data/')
sys.stdout.write(u.html.busy())
else:
if os.access('../data/QUEUED', os.F_OK):
p = 'map'
else:
p = 'data'
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]).replace('items', p))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
project = path.split('-')[1]
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.head(ltitle, tip=True))
sys.stdout.write('''
{}
<div class="pgmdsplots">
<h2>mds plots</h2>
'''.format(crumbs))
if os.access('OK', os.F_OK):
fp = open('image.html', 'rt', encoding='utf-8')
for line in fp:
sys.stdout.write(line)
fp.close()
current = set()
if os.access('current', os.F_OK):
fp = open('current', 'rt', encoding='utf-8')
for line in fp:
current.add(line.strip())
fp.close()
places = []
fp = open('../data/truelabels.txt', 'rt', encoding='utf-8')
for line in fp:
places.append(line.strip())
fp.close()
places.sort()
corr = '<small><i>r</i> = {}</small>'.format(_number(open('mds.log', 'rt').read().split()[-1]))
sys.stdout.write('''
{}
{}
<script language="JavaScript"><!--
function clearAll(form) {{
o = form.s.options;
for (var i = 0; i < o.length; i++) {{
o[i].selected = false;
}}
}}
//--></script>
<p>
<form action="{}bin/mdsplotform" method="post" enctype="multipart/form-data">
<fieldset><legend></legend>
<input type="hidden" name="p" value="project_{}">
Location(s):<br><select name="s" multiple="multiple" size="10">
'''.format(u.html.img(p + '-plot2d', usemap="map1", noover=True), corr, u.config.appurl, pnum))
for i in range(len(places)):
if places[i] in current:
s = ' selected="selected"'
else:
s = ''
sys.stdout.write('<option value="{}"{}>{}</option>\n'.format(i, s, u.html.escape(places[i])))
sys.stdout.write('''
</select>
<input onclick="clearAll(this.form)" type="button" value="Clear all"><br>
<br>
<input type="submit" value="Show labels">
</fieldset>
</form>
''')
elif os.access('QUEUED', os.F_OK):
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
project = path.split('-')[1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgcccmaps">
<h2>{}</h2>
'''.format(crumbs, title))
if os.access('OK', os.F_OK):
open('reset', 'wt').close()
try:
fp = open('current', 'rt')
current = int(fp.read())
fp.close()
except:
current = 2
sys.stdout.write('''
{}
<p>
Above: Stochastic clustering followed by multidimensional scaling shows the major cluster divisions.
<p>
Below: Omit parts of the data to see more detail in the remaining part.
'''.format(u.html.img(p + '-ccc', usemap="map1", idx=999)))
for i in range(current):
if os.access('ccc_{}_{}.eps'.format(current, i + 1), os.F_OK):
sys.stdout.write('<p>\n{}\n'.format(u.html.img(p + '-ccc_{}_{}'.format(current, i + 1), usemap="map1", idx=i)))
else:
sys.stdout.write('<div class="warn">Missing: failed map #{}{}</div>\n'.format(i + 1, u.html.help('cccmapsmissing')))
sys.stdout.write('''
<p>
<form action="{}bin/cccmapsform" method="post">
<input type="hidden" name="p" value="{}">
Number of groups:
<select name="n">
'''.format(u.config.appurl, project))
for i in range(2, 9):
if i == current:
sys.stdout.write('<option selected="selected">{}</option>\n'.format(i))
else:
sys.stdout.write('<option>{}</option>\n'.format(i))
sys.stdout.write('''
</select>
<input type="submit" value="Change">
</form>
''')
elif os.access('QUEUED', os.F_OK):
sys.stdout.write(u.html.busy())
else:
tooSmall = False
if os.access('reset', os.F_OK):
reset = True
else:
reset = False
if reset:
fp = open('current', 'rt')
cur = fp.read().strip()
fp.close()
pref = 'ccc_{}_'.format(cur)
for filename in os.listdir('.'):
if filename.endswith('.tbl'):
n = 0
fp = open(filename, 'rt', encoding='iso-8859-1')
for line in fp:
if line[0] == ':':
n += 1
fp.close()
if n < 4:
tooSmall = True
if filename.startswith('current') or filename.startswith('tmp') or filename.startswith(pref):
os.remove(filename)
if tooSmall:
sys.stdout.write('Error: clusters too small' + u.html.help('cccmapserror'))
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]))
if reset:
sys.stdout.write('<p>\n<a href="">Continue</a>\n')
open('OK', 'wt').close()
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
project = path.split('-')[1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgclumaps">
<h2>{}</h2>
'''.format(crumbs, title))
if os.access('OK', os.F_OK):
sys.stdout.write('''
<div class="info">
The results of clustering can be deceptive. So,
the purpose of this page is not just to devide your data into dialects groups, but to test the validity of each group.<br>
<br>
For an introduction, read <a href="../doc/ClusterValidation" target="_blank">this demonstration</a>
</div>
''')
open('reset', 'wt').close()
current2 = ['wm']
try:
fp = open('current', 'rt')
lines = fp.readlines()
fp.close()
current = lines[0].split()
if len(lines) > 1:
current2 = lines[1].split()
except:
current = ['c', '6', 'all']
if current[0] == 'c':
bw = ''
else:
bw = 'bw'
try:
with open('../data/stats.txt', 'r') as fp:
nPlaces = int(fp.read().strip().split()[0])
except:
nPlaces = 2
method = current2[0]
n = min(int(current[1]), nPlaces)
current = current[2:]
logfile = 'plot_{}_{}{}_{}.log'.format(method, n, bw,
'_'.join(current))
if os.access(logfile, os.F_OK):
corr = '<small><i>r</i> = ' + _number(
open(logfile, 'rt').read().strip().split()[-1]) + '</small>'
else:
corr = ''
sys.stdout.write('''
{}
<p>
Above: Clustering using <b>{}</b>.
<p>
Below: An MDS-plot of the clusters from the map shows what major cluster divisions are "real".
You can omit one major group of clusters to inspect the divisions of the remaining clusters.
{}
{}
<p>
'''.format(
u.html.img(p + '-{}{}{}'.format(method, n, bw),
usemap="map1",
pseudoforce=True), methods[method],
u.html.img(
(p +
'-plot_{}_{}{}_{}'.format(method, n, bw, '_'.join(current)))),
corr))
sys.stdout.write('''
<form action="{}bin/clumapsform" method="post">
<input type="hidden" name="p" value="{}">
<input type="hidden" name="action" value="method">
<em>Note: each of the following buttons changes a single option</em>
<p>
Clustering method:
<select name="m">
'''.format(u.config.appurl, project))
for i in sorted(methods):
if i == method:
sys.stdout.write(
'<option selected="selected" value="{}">{}</option>\n'.
format(i, methods[i]))
else:
sys.stdout.write('<option value="{}">{}</option>\n'.format(
i, methods[i]))
sys.stdout.write('''
</select>
<input type="submit" value="Change method">
</form>
<p>
''')
sys.stdout.write('''
<form action="{}bin/clumapsform" method="post">
<input type="hidden" name="p" value="{}">
<input type="hidden" name="action" value="number">
Number of clusters:
<select name="n">
'''.format(u.config.appurl, project))
for i in range(2, 13):
if i == n:
sys.stdout.write(
'<option selected="selected">{}</option>\n'.format(i))
else:
sys.stdout.write('<option>{}</option>\n'.format(i))
sys.stdout.write('''
</select>
<input type="submit" value="Change number">
</form>
<p>
''')
sys.stdout.write('''
<form action="{}bin/clumapsform" method="post">
<input type="hidden" name="p" value="{}">
<input type="hidden" name="action" value="subset">
Clusters in plot:
'''.format(u.config.appurl, project))
if current[0] == 'all':
subset = set(range(1, n + 1))
else:
subset = set(int(x) for x in current)
if bw == 'bw':
for i in range(1, n + 1):
if i in subset:
sys.stdout.write(
'<span class="sym"><input type="checkbox" name="s{0}" checked="checked"><img src="{1}img/sym{0:02d}.png"></span>\n'
.format(i, u.config.appurl))
else:
sys.stdout.write(
'<span class="sym"><input type="checkbox" name="s{0}"<img src="{1}img/sym{0:02d}.png"></span>\n'
.format(i, u.config.appurl))
else:
for i in range(1, n + 1):
if i in subset:
sys.stdout.write(
'<span class="s{0}"><input type="checkbox" name="s{0}" checked="checked"></span>\n'
.format(i))
else:
sys.stdout.write(
'<span class="s{0}"><input type="checkbox" name="s{0}"></span>\n'
.format(i))
sys.stdout.write('''
<input type="submit" value="Change subset">
</form>
<p>
''')
if bw == 'bw':
sys.stdout.write('''
<form action="{}bin/clumapsform" method="post">
<input type="hidden" name="p" value="{}">
<input type="hidden" name="action" value="col">
<input type="submit" value="Switch to colour">
</form>
'''.format(u.config.appurl, project))
else:
sys.stdout.write('''
<form action="{}bin/clumapsform" method="post">
<input type="hidden" name="p" value="{}">
<input type="hidden" name="action" value="bw">
<input type="submit" value="Switch to black/white">
</form>
'''.format(u.config.appurl, project))
elif os.access('QUEUED', os.F_OK):
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]))
if os.access('reset', os.F_OK):
for filename in os.listdir('.'):
if filename.startswith('current') or filename.startswith(
'tmp'):
os.remove(filename)
open('OK', 'wt').close()
sys.stdout.write('<p>\n<a href="">Continue</a>\n')
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path[:-7])
if not os.path.isdir('nummap'):
os.mkdir('nummap')
os.chdir('nummap')
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgdistmap">
<h2>value maps</h2>
'''.format(crumbs))
if not os.access('current.txt', os.F_OK):
fp = open('current.txt', 'wt')
fp.write('-1\n')
fp.close()
fp = open('current.txt', 'rt', encoding='utf-8')
current = int(fp.read())
fp.close()
items = []
n = -1
fp = open('../data/NAs.txt', 'rt', encoding='utf-8')
for line in fp:
n += 1
items.append((line.split(None, 1)[1].strip(), n))
fp.close()
items.sort()
sys.stdout.write('''
<form action="{}bin/nummapform" method="post">
<fieldset><legend></legend>
<input type="hidden" name="p" value="project_{}">
Item: <select name="item">
<option value="-1">--</option>
'''.format(u.config.appurl, pnum))
for item, i in items:
if i == current:
sel = ' selected="selected"'
found = True
else:
sel = ''
sys.stdout.write('<option value="{}"{}>{}</option>\n'.format(
i, sel, u.html.escape(item)))
fp.close()
sys.stdout.write('''
</select>
<input type="submit" value="Select item">
</fieldset>
</form>
''')
if current < 0:
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
return
if not os.access('limits', os.F_OK):
fp = open('../data/table.txt', 'rt', encoding='iso-8859-1')
fp.readline()
hasF = False
for line in fp:
line = line.strip()
v = [float(x) for x in line.split('\t')[1:] if x != 'NA']
if v:
ff1 = min(v)
ff2 = max(v)
if hasF:
if ff1 < f1:
f1 = ff1
if ff2 > f2:
f2 = ff2
else:
f1 = ff1
f2 = ff2
hasF = True
fp.close()
fp = open('limits', 'wt')
fp.write('{:g} {:g}\n'.format(f1, f2))
fp.close()
if not os.access('nummap.eps', os.F_OK):
placen = {}
placeall = {}
fp = open('../data/table.txt', 'rt', encoding='iso-8859-1')
fp.readline()
for line in fp:
i = line.split('\t')
lbl = _unquote(i[0])
if i[current + 1] == 'NA':
continue
value = float(i[current + 1])
placen[lbl] = value
placeall[lbl] = 1
fp.close()
fp = open('limits', 'rt')
f1, f2 = [float(x) for x in fp.read().split()]
fp.close()
if f1 == f2:
for p in placen:
placen[p] = 0.5
else:
for p in placen:
placen[p] = (placen[p] - f1) / (f2 - f1)
os.chdir('..')
u.distribute.distmap(placen, placeall, 'nummap/nummap')
os.chdir('nummap')
"""
sys.stdout.write('''
<h3>Value map for{} "<span class="ipa">{}</span>" in {}</h3>
'''.format(r, u.html.escape(v), u.html.escape(item)))
if currentregex:
sys.stdout.write('<ul>\n')
fp = open('currentvariants.txt', 'rt', encoding='utf-8')
for line in fp:
sys.stdout.write('<li><span class="ipa2">{0[0]}</span> {0[1]}\n'.format(u.html.escape(line.strip()).rsplit(None, 1)))
fp.close()
sys.stdout.write('</ul>\n')
"""
sys.stdout.write(
u.html.img('project_{}-nummap-nummap'.format(pnum),
True,
usemap="map1"))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path[:-8])
if not os.path.isdir('distmap'):
os.mkdir('distmap')
os.chdir('distmap')
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgdistmap">
<h2>distribution maps</h2>
'''.format(crumbs))
if not os.access('items.txt', os.F_OK):
items = []
for filename in os.listdir('../data/_'):
if not filename.endswith('.data'):
continue
fname = filename.replace('.data', '')
iname = re.sub('_([0-9]+)_', _num2chr, fname)
items.append((iname.lower(), fname, iname))
fp = open('items.txt', 'wt', encoding='utf-8')
for i, j, k in sorted(items):
fp.write('{}\t{}\n'.format(j, k))
fp.close()
if not os.access('current.txt', os.F_OK):
open('current.txt', 'wt').close()
fp = open('current.txt', 'rt', encoding='utf-8')
current = [x.strip() for x in fp.read().strip().split('\n')]
fp.close()
currentitem = ''
currentvariants = set()
currentregex = ''
if len(current) > 0:
currentitem = current[0]
if len(current) > 1:
currentvariants = set(current[1].split('\t'))
if len(current) > 2:
currentregex = current[2]
REGEX = re.compile(currentregex)
sys.stdout.write('''
<form action="{}distmapform" method="post" enctype="multipart/form-data">
<fieldset><legend></legend>
<input type="hidden" name="hebci_auml" value="ä">
<input type="hidden" name="hebci_divide" value="÷">
<input type="hidden" name="hebci_euro" value="€">
<input type="hidden" name="hebci_middot" value="·">
<input type="hidden" name="hebci_oelig" value="œ">
<input type="hidden" name="hebci_oslash" value="ø">
<input type="hidden" name="hebci_Scaron" value="Š">
<input type="hidden" name="hebci_sect" value="§">
<input type="hidden" name="hebci_thorn" value="þ">
<input type="hidden" name="p" value="project_{}">
<input type="hidden" name="var" value="">
Item: <select name="item">
<option>--</option>
'''.format(u.config.binrel, pnum))
found = False
fp = open('items.txt', 'rt', encoding='utf-8')
for line in fp:
a, b = line.strip().split('\t')
if a == currentitem:
sel = ' selected="selected"'
currenthtml = u.html.escape(b)
found = True
else:
sel = ''
sys.stdout.write('<option value="{}"{}>{}</option>\n'.format(a, sel, u.html.escape(b)))
fp.close()
sys.stdout.write('''
</select>
<input type="submit" value="Select item">
</fieldset>
</form>
''')
if not found:
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
return
if os.access('../data/UTF', os.F_OK):
encoding = 'utf-8'
else:
encoding = 'iso-8859-1'
if not os.access('currentlist.txt', os.F_OK):
variants = {}
fp = open('../data/_/' + currentitem + '.data', 'rb')
for line in fp:
if line[:1] == b'-':
v = line[1:].strip().decode(encoding)
if not v in variants:
variants[v] = 0
variants[v] += 1
fp.close()
fp = open('currentlist.txt', 'wt', encoding='utf-8')
for v in sorted(variants):
fp.write('{}\t{}\n'.format(variants[v], v))
fp.close()
fp = open('currentlist.txt', 'rt', encoding='utf-8')
lines = fp.readlines()
fp.close()
sys.stdout.write('''
<p>
<form action="{}distmapform" method="post" enctype="multipart/form-data">
<fieldset>
<legend></legend>
<input type="hidden" name="p" value="project_{}">
<input type="hidden" name="item" value="{}">
There are {} variants for <em>{}</em><br>
Select one or more variants:<br><select name="var" multiple="multiple" size="10" class="ipa2">
'''.format(u.config.binrel, pnum, currentitem, len(lines), currenthtml))
for line in lines:
ii, b = line.strip().split('\t', 1)
a = _iname(b)
if a in currentvariants:
sel = ' selected="selected"'
found = True
else:
sel = ''
sys.stdout.write('<option value="{}"{}>{} ({})</option>\n'.format(a, sel, u.html.escape(b), ii))
sys.stdout.write('''
</select><br>
<br>
— or —<br>
<br>
Regular expression:{}
<input type="hidden" name="hebci_auml" value="ä">
<input type="hidden" name="hebci_divide" value="÷">
<input type="hidden" name="hebci_euro" value="€">
<input type="hidden" name="hebci_middot" value="·">
<input type="hidden" name="hebci_oelig" value="œ">
<input type="hidden" name="hebci_oslash" value="ø">
<input type="hidden" name="hebci_Scaron" value="Š">
<input type="hidden" name="hebci_sect" value="§">
<input type="hidden" name="hebci_thorn" value="þ">
<input type="text" name="regex" size="60" class="ipa2">
<br>
<br>
<input type="submit" value="Show distribution map">
</fieldset>
</form>
'''.format(u.html.help('regex')))
if not (currentvariants or currentregex):
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
return
item = re.sub('_([0-9]+)_', _num2chr, currentitem)
variantset = set([re.sub('_([0-9]+)_', _num2chr, x) for x in currentvariants])
if not os.access('distmap.eps', os.F_OK):
placen = {}
placeall = {}
variants = {}
fp = open('../data/_/{}.data'.format(currentitem), 'rb')
for line in fp:
if line[:1] == b':':
place = line[1:].decode('iso-8859-1').strip()
if not place in placen:
placen[place] = 0
placeall[place] = 0
elif line[:1] == b'-':
if currentregex:
l = line[1:].strip().decode(encoding)
if REGEX.search(l):
placen[place] += 1
if not l in variants:
variants[l] = 0
variants[l] += 1
elif line[1:].strip().decode(encoding) in variantset:
placen[place] += 1
placeall[place] += 1
fp.close()
os.chdir('..')
u.distribute.distmap(placen, placeall, 'distmap/distmap', normalise=False)
os.chdir('distmap')
if currentregex:
fp = open('currentvariants.txt', 'wt', encoding='utf-8')
for i in sorted(variants):
fp.write('{} ({})\n'.format(i, variants[i]))
fp.close()
if currentregex:
v = currentregex
r = ' RE'
else:
if len(currentvariants) == 1:
v = list(variantset)[0]
r = ''
else:
v = '*'
r = ' SET'
sys.stdout.write('''
<h3>Distribution map for{} "<span class="ipa">{}</span>" in {}</h3>
'''.format(r, u.html.escape(v), u.html.escape(item)))
if currentregex:
sys.stdout.write('<ul>\n')
fp = open('currentvariants.txt', 'rt', encoding='utf-8')
for line in fp:
sys.stdout.write('<li><span class="ipa2">{0[0]}</span> {0[1]}\n'.format(u.html.escape(line.strip()).rsplit(None, 1)))
fp.close()
sys.stdout.write('</ul>\n')
sys.stdout.write(u.html.img('project_{}-distmap-distmap'.format(pnum), True, usemap="map1"))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
pnum = path.split('-')[-2].split('_')[-1]
p = path.split('-', 1)[1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgdiff">
<h2>{}</h2>
'''.format(crumbs, title))
if os.access('OK', os.F_OK):
if os.access('../map/PSEUDOMAP', os.F_OK):
linc = 'undefined for pseudomap'
else:
fp = open('linc.txt', 'r')
linc = fp.read()
fp.close()
try:
fp = open('../data/Method', 'rt')
m = fp.readline().strip()
fp.close()
except:
pass
else:
mt = 'Method: {}\n<p>'.format(_methods[m])
sys.stdout.write(mt + '\n')
sys.stdout.write('Local incoherence: {}{}\n<p>\n'.format(_number(linc), u.html.help('linc')))
sys.stdout.write('''
→ <a href="{0}getdiff?p=project_{1}&f=L04" target="_blank">download differences</a> (RuG/L04 format)
<p>
→ <a href="{0}getdiff?p=project_{1}&f=tab" target="_blank">download differences</a> (table format)
'''.format(u.config.binurls, pnum))
if m.startswith('levfeat'):
sys.stdout.write('''
<p>
→ <a href="{}getfeat?p=project_{}" target="_blank">download feature definition</a>
'''.format(u.config.binurls, pnum))
sys.stdout.write('''
<p>
{}
<p>
{}
<p>
{}
'''.format(u.html.img(p + '-diff01'),
u.html.img(p + '-diff1', usemap="map1", idx=1),
u.html.img(p + '-diff', usemap="map1", idx=2)))
elif os.access('QUEUED', os.F_OK):
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
pnum = int(path.split('-')[1].split('_')[-1])
sys.stdout.write(u.html.head(ltitle))
sys.stdout.write('''
{}
<div class="pgprob">
<h2>linguistic difference ↔ geographic distance</h2>
'''.format(crumbs))
if os.access('OK', os.F_OK):
sys.stdout.write('''
→ <a href="getplotdata?{}">download R data</a>{}
<p>
'''.format(pnum, u.html.help("plotr")))
try:
fp = open('plot01.log', 'rt', encoding='utf-8')
txt = fp.read()
fp.close()
except:
txt = ''
else:
txt = '<pre class="log">\n' + u.html.escape(txt) + '</pre>\n'
# try:
# fp = open('plot02.log', 'rt', encoding='utf-8')
# txt2 = fp.read()
# fp.close()
# except:
# txt2 = ''
# else:
# txt2 = '<pre class="log">\n' + u.html.escape(txt2) + '</pre>\n'
#
# sys.stdout.write('''
# {}
# <p>
# {}
#
# <p>
# <hr>
# '''.format(u.html.img(p + '-plot02'), txt2))
if (os.access('plot01.eps', os.F_OK)):
sys.stdout.write('''
<p>
{}
<p>
A plot with local regression (red) and asymptotic regression (blue).<br>
A large b/a ratio indicates a large signal/noise ratio in the data.<br>
A small value for c indicates that linguistic variation is measurable over a short geographic distance.
<p>
{}
<p>
<hr>
'''.format(u.html.img(p + '-plot01'), txt))
else:
# TO DO: link to help page on reason for failure
sys.stdout.write('''
<p>
{}
<p>
Asymptotic regression failed.
<p>
{}
<p>
<hr>
'''.format(u.html.img(p + '-plot02'), txt))
sys.stdout.write('''
<p>
{}
'''.format(u.html.img(p + '-plot03')))
elif os.access('QUEUED', os.F_OK):
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
pp = p.rsplit('-', 1)[0]
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgprob">
<h2>{}</h2>
'''.format(crumbs, title))
if os.access('OK', os.F_OK):
fp = open('../data/Method', 'rt')
method = fp.read().strip()
fp.close;
fp = open('current.txt', 'rt')
noise, limit, exponent, color, mthd = fp.read().split()
fp.close()
if color == 'col':
col = ''
else:
col = 'bw'
if not os.access('probbw.eps', os.F_OK):
fpin = open('prob.eps', 'rt', encoding='iso-8859-1')
fpout = open('probbw.eps', 'wt', encoding='iso-8859-1')
for line in fpin:
if not re.match('\\s*\\S+\\s+\\S+\\s+\\S+\\s+cl\\s*$', line):
fpout.write(line)
fpout.close()
fpin.close()
sys.stdout.write('''
{}
<p>
Above: Clustering with noise. The graph shows the relative certainty of certain clusters.
'''.format(u.html.img(p + '-prob' + col)))
# TODO: deze waardes inlezen vanuit bestand 'reset'
# TODO: fix voor IE
if method == 'giw':
eV = '6'
else:
eV = '1.5'
sys.stdout.write('''
<script language="JavaScript"><!--
function defaults(form) {{
form.noise.value = "0.2";
form.limit.value = "60";
form.exp.value = "{}";
form.col.value = "col";
form.mthd.value = "gawa";
}}
//--></script>
<form action="{}probform" method="post">
<fieldset><legend></legend>
<input type="hidden" name="p" value="project_{}">
noise: <select name="noise">
'''.format(eV, u.config.binurls, pnum))
for i in '0.001 0.005 0.01 0.05 0.1 0.2 0.3 0.4 0.5 0.8 1 1.5 2 3 4'.split():
if i == noise:
sel = ' selected="selected"'
else:
sel = ''
sys.stdout.write('<option{}>{}</option>\n'.format(sel, i))
sys.stdout.write('''
</select>
limit: <select name="limit">
''')
for i in '51 60 70 80'.split():
if i == limit:
sel = ' selected="selected"'
else:
sel = ''
sys.stdout.write('<option value="{1}"{0}>{1}%</option>\n'.format(sel, i))
sys.stdout.write('''
</select>
exponent: <select name="exp">
''')
for i in '0.25 0.5 1 1.5 2 4 6'.split():
if i == exponent:
sel = ' selected="selected"'
else:
sel = ''
sys.stdout.write('<option{0}>{1}</option>\n'.format(sel, i))
sys.stdout.write('''
</select>
color: <select name="col">
''')
for i, j in [('col', 'yes'), ('bw', 'no')]:
if i == col:
sel = ' selected="selected"'
else:
sel = ''
sys.stdout.write('<option value="{}"{}>{}</option>\n'.format(i, sel, j))
sys.stdout.write('''
</select><br>
<br>
method: <select name="mthd">
''')
for i, j in [('ga', 'group average'), ('wa', 'weighted average'), ('gawa', 'group average + weighted average')]:
if i == mthd:
sel = ' selected="selected"'
else:
sel = ''
sys.stdout.write('<option value="{}"{}>{}</option>\n'.format(i, sel, j))
sys.stdout.write('''
</select>
<input type="button" value="Restore defaults" onclick="defaults(this.form)">
<input type="submit" value="Update settings">
</fieldset>
</form>
''')
if color == 'col':
sys.stdout.write('''
<p>
For reference: Colors in the graph above correspond to colors in the map below.
<p>
{}
'''.format(u.html.img(pp + '-cccmaps-ccc', usemap="map1")))
elif os.access('QUEUED', os.F_OK):
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]))
if os.access('reset', os.F_OK):
sys.stdout.write('<p>\n<a href="{}probform?p=project_{}">Continue...</a>\n'.format(u.config.binurls, pnum))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
project = path.split('-')[1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgcccmaps">
<h2>{}</h2>
'''.format(crumbs, title))
if os.access('OK', os.F_OK):
open('reset', 'wt').close()
try:
fp = open('current', 'rt')
current = int(fp.read())
fp.close()
except:
current = 2
sys.stdout.write('''
{}
<p>
Above: Stochastic clustering followed by multidimensional scaling shows the major cluster divisions.
<p>
Below: Omit parts of the data to see more detail in the remaining part.
'''.format(u.html.img(p + '-ccc', usemap="map1", idx=999)))
for i in range(current):
if os.access('ccc_{}_{}.eps'.format(current, i + 1), os.F_OK):
sys.stdout.write('<p>\n{}\n'.format(
u.html.img(p + '-ccc_{}_{}'.format(current, i + 1),
usemap="map1",
idx=i)))
else:
sys.stdout.write(
'<div class="warn">Missing: failed map #{}{}</div>\n'.
format(i + 1, u.html.help('cccmapsmissing')))
sys.stdout.write('''
<p>
<form action="{}bin/cccmapsform" method="post">
<input type="hidden" name="p" value="{}">
Number of groups:
<select name="n">
'''.format(u.config.appurl, project))
for i in range(2, 9):
if i == current:
sys.stdout.write(
'<option selected="selected">{}</option>\n'.format(i))
else:
sys.stdout.write('<option>{}</option>\n'.format(i))
sys.stdout.write('''
</select>
<input type="submit" value="Change">
</form>
''')
elif os.access('QUEUED', os.F_OK):
sys.stdout.write(u.html.busy())
else:
tooSmall = False
if os.access('reset', os.F_OK):
reset = True
else:
reset = False
if reset:
fp = open('current', 'rt')
cur = fp.read().strip()
fp.close()
pref = 'ccc_{}_'.format(cur)
for filename in os.listdir('.'):
if filename.endswith('.tbl'):
n = 0
fp = open(filename, 'rt', encoding='iso-8859-1')
for line in fp:
if line[0] == ':':
n += 1
fp.close()
if n < 4:
tooSmall = True
if filename.startswith('current') or filename.startswith(
'tmp') or filename.startswith(pref):
os.remove(filename)
if tooSmall:
sys.stdout.write('Error: clusters too small' +
u.html.help('cccmapserror'))
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]))
if reset:
sys.stdout.write('<p>\n<a href="">Continue</a>\n')
open('OK', 'wt').close()
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path[:-7])
if not os.path.isdir('nummap'):
os.mkdir('nummap')
os.chdir('nummap')
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgdistmap">
<h2>value maps</h2>
'''.format(crumbs))
if not os.access('current.txt', os.F_OK):
fp = open('current.txt', 'wt')
fp.write('-1\n')
fp.close()
fp = open('current.txt', 'rt', encoding='utf-8')
current = int(fp.read())
fp.close()
items = []
n = -1
fp = open('../data/NAs.txt', 'rt', encoding='utf-8')
for line in fp:
n += 1
items.append((line.split(None, 1)[1].strip(), n))
fp.close()
items.sort()
sys.stdout.write('''
<form action="{}bin/nummapform" method="post">
<fieldset><legend></legend>
<input type="hidden" name="p" value="project_{}">
Item: <select name="item">
<option value="-1">--</option>
'''.format(u.config.appurl, pnum))
for item, i in items:
if i == current:
sel = ' selected="selected"'
found = True
else:
sel = ''
sys.stdout.write('<option value="{}"{}>{}</option>\n'.format(i, sel, u.html.escape(item)))
fp.close()
sys.stdout.write('''
</select>
<input type="submit" value="Select item">
</fieldset>
</form>
''')
if current < 0:
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
return
if not os.access('limits', os.F_OK):
fp = open('../data/table.txt', 'rt', encoding='iso-8859-1')
fp.readline()
hasF = False
for line in fp:
line = line.strip()
v = [float(x) for x in line.split('\t')[1:] if x != 'NA']
if v:
ff1 = min(v)
ff2 = max(v)
if hasF:
if ff1 < f1:
f1 = ff1
if ff2 > f2:
f2 = ff2
else:
f1 = ff1
f2 = ff2
hasF = True
fp.close()
fp = open('limits', 'wt')
fp.write('{:g} {:g}\n'.format(f1, f2))
fp.close()
if not os.access('nummap.eps', os.F_OK):
placen = {}
placeall = {}
fp = open('../data/table.txt', 'rt', encoding='iso-8859-1')
fp.readline()
for line in fp:
i = line.split('\t')
lbl = _unquote(i[0])
if i[current + 1] == 'NA':
continue
value = float(i[current + 1])
placen[lbl] = value
placeall[lbl] = 1
fp.close()
fp = open('limits', 'rt')
f1, f2 = [float(x) for x in fp.read().split()]
fp.close()
if f1 == f2:
for p in placen:
placen[p] = 0.5
else:
for p in placen:
placen[p] = (placen[p] - f1) / (f2 - f1)
os.chdir('..')
u.distribute.distmap(placen, placeall, 'nummap/nummap')
os.chdir('nummap')
"""
sys.stdout.write('''
<h3>Value map for{} "<span class="ipa">{}</span>" in {}</h3>
'''.format(r, u.html.escape(v), u.html.escape(item)))
if currentregex:
sys.stdout.write('<ul>\n')
fp = open('currentvariants.txt', 'rt', encoding='utf-8')
for line in fp:
sys.stdout.write('<li><span class="ipa2">{0[0]}</span> {0[1]}\n'.format(u.html.escape(line.strip()).rsplit(None, 1)))
fp.close()
sys.stdout.write('</ul>\n')
"""
sys.stdout.write(u.html.img('project_{}-nummap-nummap'.format(pnum), True, usemap="map1"))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
project = path.split('-')[1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgclusters">
<h2>{}</h2>
'''.format(crumbs, title))
if os.access('OK', os.F_OK):
sys.stdout.write('''
<div class="info">
Warning: the type of clustering used on this page can be unreliable.<br>
Go to <a href="goto?p={}-clumaps">this page</a> to test the validity of the clusters you see here.
</div>
'''.format(project))
fp = open('current.txt', 'rt')
method, groups, col = fp.read().split()
fp.close()
sys.stdout.write('''
{}
<p>
{}
'''.format(
u.html.img('{}-map{}{}{}'.format(p, method, groups, col),
usemap="map1"),
u.html.img('{}-den{}{}{}'.format(p, method, groups, col))))
sys.stdout.write('''
<p>
<form action="{}bin/clusterform" method="post">
<fieldset><legend></legend>
<input type="hidden" name="p" value="{}">
Clustering method:
<select name="mthd">
'''.format(u.config.appurl, project))
for i in sorted(methods):
if i == method:
sys.stdout.write(
'<option selected="selected" value="{}">{}</option>\n'.
format(i, methods[i]))
else:
sys.stdout.write('<option value="{}">{}</option>\n'.format(
i, methods[i]))
sys.stdout.write('''
</select><br>
<br>
Number of clusters:
<select name="n">
''')
n = int(groups)
maxnum = min(13,
int(open('../data/stats.txt', 'rt').read().split()[0]))
for i in range(2, maxnum):
if i == n:
sys.stdout.write(
'<option selected="selected">{}</option>\n'.format(i))
else:
sys.stdout.write('<option>{}</option>\n'.format(i))
sys.stdout.write('''
</select><br>
<br>
Use color:
<select name="col">
''')
for i in sorted(colors):
if i == col:
sys.stdout.write(
'<option selected="selected" value="{}">{}</option>\n'.
format(i, colors[i]))
else:
sys.stdout.write('<option value="{}">{}</option>\n'.format(
i, colors[i]))
sys.stdout.write('''
</select><br>
<br>
<input type="submit" value="Change settings">
</fieldset>
</form>
<p>
''')
if col == 'bw':
groups = 1
sys.stdout.write(
u.html.img('{}-den{}{}{}alt'.format(p, method, groups, col)))
elif os.access('QUEUED', os.F_OK):
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]))
if os.access('reset', os.F_OK):
for filename in os.listdir('.'):
if filename.startswith('current') or filename.startswith(
'tmp'):
os.remove(filename)
open('OK', 'wt').close()
sys.stdout.write('<p>\n<a href="">Continue</a>\n')
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgmap">
<h2>{}</h2>
'''.format(crumbs, title))
if os.access('OK', os.F_OK):
if not os.access('mapcoo.eps', os.F_OK):
os.system('$PYTHON3 $APPDIR/util/ps2coo')
if not os.access('mapcoo.png', os.F_OK):
os.system('eps2png > /dev/null 2>&1')
if not os.access('image.html', os.F_OK):
os.system('$PYTHON3 $APPDIR/util/mkmap')
fp = open('image.html', 'rt', encoding='utf-8')
for line in fp:
sys.stdout.write(line)
fp.close()
sys.stdout.write(
u.html.img(p + '-mapcoo', usemap="map1", idx=1, noover=True))
if not os.access('PSEUDOMAP', os.F_OK):
sys.stdout.write('''
<p>
→ <a href="mapdstget?p={0}&f=tbl">download distances in table format</a><br>
→ <a href="mapdstget?p={0}&f=l04">download distances in L04 format</a><br>
note: these are <em>geographic</em> distances in kilometers
<p>
'''.format(pnum))
sys.stdout.write(
u.html.img(p + '-mapidx', usemap="map1", idx=2, noover=True))
if os.access('WARNINGS.txt', os.F_OK):
sys.stdout.write('<pre class="log">\n')
fp = open('WARNINGS.txt', 'rt', encoding='iso-8859-1')
for line in fp:
sys.stdout.write(u.html.escape(line))
fp.close()
sys.stdout.write('</pre>\n')
lines = []
sys.stdout.write('<table width="100%"><tr><td><pre>\n')
labels = set()
truelabels = {}
fp = open('../data/labels.txt', 'rt', encoding='iso-8859-1')
fp2 = open('../data/truelabels.txt', 'rt', encoding='utf-8')
for line in fp:
lbl = line.strip().split(None, 1)[1]
labels.add(lbl)
truelabels[lbl] = fp2.readline().strip()
fp2.close()
fp.close()
fp = open('map.lbl', 'rt', encoding='iso-8859-1')
for line in fp:
a, b = line.strip().split(None, 1)
b = _unquote(b)
if b in labels:
b = u.html.escape(truelabels[b])
lines.append((b, a))
sys.stdout.write('{:8d} {}\n'.format(int(a), b))
fp.close()
#loc = locale.getlocale()
#locale.setlocale(locale.LC_COLLATE, 'en_US')
sys.stdout.write('</pre>\n<td><pre>\n')
for b, a in sorted(lines):
sys.stdout.write('{:8d} {}\n'.format(int(a), b))
sys.stdout.write('</pre>\n</table>\n')
#locale.setlocale(locale.LC_COLLATE, loc)
elif os.access('QUEUED', os.F_OK):
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
pp = p.rsplit('-', 1)[0]
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgprob">
<h2>{}</h2>
'''.format(crumbs, title))
if os.access('OK', os.F_OK):
fp = open('../data/Method', 'rt')
method = fp.read().strip()
fp.close
fp = open('current.txt', 'rt')
noise, limit, exponent, color, mthd = fp.read().split()
fp.close()
if color == 'col':
col = ''
else:
col = 'bw'
if not os.access('probbw.eps', os.F_OK):
fpin = open('prob.eps', 'rt', encoding='iso-8859-1')
fpout = open('probbw.eps', 'wt', encoding='iso-8859-1')
for line in fpin:
if not re.match('\\s*\\S+\\s+\\S+\\s+\\S+\\s+cl\\s*$',
line):
fpout.write(line)
fpout.close()
fpin.close()
sys.stdout.write('''
{}
<p>
Above: Clustering with noise. The graph shows the relative certainty of certain clusters.
'''.format(u.html.img(p + '-prob' + col)))
# TODO: deze waardes inlezen vanuit bestand 'reset'
# TODO: fix voor IE
if method == 'giw':
eV = '6'
else:
eV = '1.5'
sys.stdout.write('''
<script language="JavaScript"><!--
function defaults(form) {{
form.noise.value = "0.2";
form.limit.value = "60";
form.exp.value = "{}";
form.col.value = "col";
form.mthd.value = "gawa";
}}
//--></script>
<form action="{}bin/probform" method="post">
<fieldset><legend></legend>
<input type="hidden" name="p" value="project_{}">
noise: <select name="noise">
'''.format(eV, u.config.appurl, pnum))
for i in '0.001 0.005 0.01 0.05 0.1 0.2 0.3 0.4 0.5 0.8 1 1.5 2 3 4'.split(
):
if i == noise:
sel = ' selected="selected"'
else:
sel = ''
sys.stdout.write('<option{}>{}</option>\n'.format(sel, i))
sys.stdout.write('''
</select>
limit: <select name="limit">
''')
for i in '51 60 70 80'.split():
if i == limit:
sel = ' selected="selected"'
else:
sel = ''
sys.stdout.write('<option value="{1}"{0}>{1}%</option>\n'.format(
sel, i))
sys.stdout.write('''
</select>
exponent: <select name="exp">
''')
for i in '0.25 0.5 1 1.5 2 4 6'.split():
if i == exponent:
sel = ' selected="selected"'
else:
sel = ''
sys.stdout.write('<option{0}>{1}</option>\n'.format(sel, i))
sys.stdout.write('''
</select>
color: <select name="col">
''')
for i, j in [('col', 'yes'), ('bw', 'no')]:
if i == col:
sel = ' selected="selected"'
else:
sel = ''
sys.stdout.write('<option value="{}"{}>{}</option>\n'.format(
i, sel, j))
sys.stdout.write('''
</select><br>
<br>
method: <select name="mthd">
''')
for i, j in [('ga', 'group average'), ('wa', 'weighted average'),
('gawa', 'group average + weighted average')]:
if i == mthd:
sel = ' selected="selected"'
else:
sel = ''
sys.stdout.write('<option value="{}"{}>{}</option>\n'.format(
i, sel, j))
sys.stdout.write('''
</select>
<input type="button" value="Restore defaults" onclick="defaults(this.form)">
<input type="submit" value="Update settings">
</fieldset>
</form>
''')
if color == 'col':
sys.stdout.write('''
<p>
For reference: Colors in the graph above correspond to colors in the map below.
<p>
{}
'''.format(u.html.img(pp + '-cccmaps-ccc', usemap="map1")))
elif os.access('QUEUED', os.F_OK):
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]))
if os.access('reset', os.F_OK):
sys.stdout.write(
'<p>\n<a href="{}bin/probform?p=project_{}">Continue...</a>\n'.
format(u.config.appurl, pnum))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
title = path.split('-', 1)[1].replace('-', '/').replace('_', ' ')
try:
fp = open('description', 'rt', encoding='utf-8')
description = fp.readline().strip()
fp.close()
except:
description = ''
fp = open('data/Method', 'rt')
method = fp.read().strip()
fp.close()
header = '''<script type="text/javascript"><!--
var visible = false;
function EDIT () {{
s = document.getElementById('edit').style;
if (visible) {{
s.visibility = 'hidden';
s.display = 'none';
visible = false;
}} else {{
s.visibility = 'visible';
s.display = 'block';
visible = true;
}}
s = document.getElementById('title').style;
if (visible) {{
s.visibility = 'hidden';
s.display = 'none';
}} else {{
s.visibility = 'visible';
s.display = 'block';
}}
}}
function CANCEL () {{
s = document.getElementById('edit').style;
s.visibility = 'hidden';
s.display = 'none';
s = document.getElementById('title').style;
s.visibility = 'visible';
s.display = 'block';
visible = false;
}}
function DEL() {{
var answer = confirm("Remove project {0}?\\n\\nThis will destroy all data and results for this project");
if (answer) {{
window.location = "{1}bin/projectremove?p={0}";
}}
}}
//--></script>
'''.format(path.split('_')[-1], u.config.appurl)
sys.stdout.write(u.html.head(title, headers=header))
sys.stdout.write('''
<div class="pgproject">
<h2>{0} <a href="javascript:DEL()" title="Remove"> × <!--✗--></a></h2>
<div id="title">
{1} <a href="javascript:EDIT()" title="Edit"> « </a>
</div>
<div id="edit" style="visibility:hidden;display:none">
<form action="{3}bin/editproject" method="post" enctype="multipart/form-data" accept-charset="utf-8" name="editproject">
<input type="hidden" name="hebci_auml" value="ä">
<input type="hidden" name="hebci_divide" value="÷">
<input type="hidden" name="hebci_euro" value="€">
<input type="hidden" name="hebci_middot" value="·">
<input type="hidden" name="hebci_oelig" value="œ">
<input type="hidden" name="hebci_oslash" value="ø">
<input type="hidden" name="hebci_Scaron" value="Š">
<input type="hidden" name="hebci_sect" value="§">
<input type="hidden" name="hebci_thorn" value="þ">
<input type="hidden" name="p" value="{2}">
Description:<br>
<input type="text" name="description" size="80" value="{1}"><br>
<input type="submit" value="Save">
<input onclick="CANCEL()" type="button" value="Cancel">
</form>
</div>
'''.format(title, description,
path.split('_')[-1], u.config.appurl))
if method.startswith('num'):
num = 'num'
else:
num = ''
sys.stdout.write('''<p>
<table class="project">
<tr><td colspan="2"><hr>
<tr valign="top">
<td>Index
<td><ul>
<li><a href="{0}bin/goto?p={1}-map">places</a>
'''.format(u.config.appurl,
path.split('-', 1)[1]))
if not method.startswith('dif'):
sys.stdout.write('''
<li><a href="{0}bin/goto?p={1}-{2}items">items</a>
'''.format(u.config.appurl,
path.split('-', 1)[1], num))
sys.stdout.write('''
</ul>
<tr valign="top">
<td>Data inspection
<td><ul>
''')
if method.startswith('num'):
sys.stdout.write('''
<li><a href="{0}bin/goto?p={1}-numdata">data overview</a>
<li><a href="{0}bin/goto?p={1}-nummap">value maps</a>
'''.format(u.config.appurl,
path.split('-', 1)[1]))
elif method.startswith('dif'):
sys.stdout.write('''
<li><a href="{0}bin/goto?p={1}-difdata">data overview</a>
'''.format(u.config.appurl,
path.split('-', 1)[1]))
else:
sys.stdout.write('''
<li><a href="{0}bin/goto?p={1}-data">data overview</a>
'''.format(u.config.appurl,
path.split('-', 1)[1]))
sys.stdout.write(
'<li><a href="{0}bin/goto?p={1}-distmap">distribution maps</a>\n'.
format(u.config.appurl,
path.split('-', 1)[1]))
sys.stdout.write('</ul>\n')
if method.startswith('lev'):
sys.stdout.write('''
<tr valign="top">
<td>Measuring technique
<td><ul>
<li><a href="{0}bin/goto?p={1}-align">alignments</a>
</ul>
'''.format(u.config.appurl,
path.split('-', 1)[1]))
sys.stdout.write('''
<tr valign="top">
<td>Differences
<td><ul>
<li><a href="{0}bin/goto?p={1}-diff">statistics and difference maps</a>
'''.format(u.config.appurl,
path.split('-', 1)[1]))
if not os.access('map/PSEUDOMAP', os.F_OK):
sys.stdout.write('''
<li><a href="{0}bin/goto?p={1}-plot">linguistic difference ↔ geographic distance</a>
'''.format(u.config.appurl,
path.split('-', 1)[1]))
sys.stdout.write('''
<li><a href="{0}bin/goto?p={1}-refmaps">reference point maps</a>
</ul>
<tr><td colspan="2"><hr>
<tr valign="top">
<td>Multidimensional Scaling
<td><ul>
<li><a href="{0}bin/goto?p={1}-mdsplots">mds plots</a>
<li><a href="{0}bin/goto?p={1}-mdsmaps">mds maps</a>
</ul>
<tr valign="top">
<td>Discrete clustering
<td><ul>
<li><a href="{0}bin/goto?p={1}-clusters">cluster maps and dendrograms</a>
<li><a href="{0}bin/goto?p={1}-clumaps">cluster validation</a>
</ul>
<tr valign="top">
<td>Fuzzy clustering
<td><ul>
<li><a href="{0}bin/goto?p={1}-prob">probabilistic dendrogram</a>
<li><a href="{0}bin/goto?p={1}-cccmaps">fuzzy cluster maps</a>
</ul>
'''.format(u.config.appurl,
path.split('-', 1)[1]))
if not method.startswith('dif'):
sys.stdout.write('''
<tr><td colspan="2"><hr>
<tr valign="top">
<td>Data mining
<td><ul>
<li><a href="{0}bin/goto?p={1}-cludet">cluster determinants</a>
</ul>
'''.format(u.config.appurl,
path.split('-', 1)[1]))
sys.stdout.write('<tr><td colspan="2"><hr>\n</table>\n')
sys.stdout.write('</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
project = path.split('-')[1]
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.head(ltitle, tip=True))
sys.stdout.write('''
{}
<div class="pgmdsplots">
<h2>mds plots</h2>
'''.format(crumbs))
if os.access('OK', os.F_OK):
fp = open('image.html', 'rt', encoding='utf-8')
for line in fp:
sys.stdout.write(line)
fp.close()
current = set()
if os.access('current', os.F_OK):
fp = open('current', 'rt', encoding='utf-8')
for line in fp:
current.add(line.strip())
fp.close()
places = []
fp = open('../data/truelabels.txt', 'rt', encoding='utf-8')
for line in fp:
places.append(line.strip())
fp.close()
places.sort()
corr = '<small><i>r</i> = {}</small>'.format(
_number(open('mds.log', 'rt').read().split()[-1]))
sys.stdout.write('''
{}
{}
<script language="JavaScript"><!--
function clearAll(form) {{
o = form.s.options;
for (var i = 0; i < o.length; i++) {{
o[i].selected = false;
}}
}}
//--></script>
<p>
<form action="{}bin/mdsplotform" method="post" enctype="multipart/form-data">
<fieldset><legend></legend>
<input type="hidden" name="p" value="project_{}">
Location(s):<br><select name="s" multiple="multiple" size="10">
'''.format(u.html.img(p + '-plot2d', usemap="map1", noover=True), corr,
u.config.appurl, pnum))
for i in range(len(places)):
if places[i] in current:
s = ' selected="selected"'
else:
s = ''
sys.stdout.write('<option value="{}"{}>{}</option>\n'.format(
i, s, u.html.escape(places[i])))
sys.stdout.write('''
</select>
<input onclick="clearAll(this.form)" type="button" value="Clear all"><br>
<br>
<input type="submit" value="Show labels">
</fieldset>
</form>
''')
elif os.access('QUEUED', os.F_OK):
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgmap">
<h2>{}</h2>
'''.format(crumbs, title))
if os.access('OK', os.F_OK):
if not os.access('mapcoo.eps', os.F_OK):
os.system('$PYTHON3 $APPDIR/util/ps2coo')
if not os.access('mapcoo.png', os.F_OK):
os.system('eps2png > /dev/null 2>&1')
if not os.access('image.html', os.F_OK):
os.system('$PYTHON3 $APPDIR/util/mkmap')
fp = open('image.html', 'rt', encoding='utf-8')
for line in fp:
sys.stdout.write(line)
fp.close()
sys.stdout.write(u.html.img(p + '-mapcoo', usemap="map1", idx=1, noover=True))
if not os.access('PSEUDOMAP', os.F_OK):
sys.stdout.write('''
<p>
→ <a href="mapdstget?p={0}&f=tbl">download distances in table format</a><br>
→ <a href="mapdstget?p={0}&f=l04">download distances in L04 format</a><br>
note: these are <em>geographic</em> distances in kilometers
<p>
'''.format(pnum))
sys.stdout.write(u.html.img(p + '-mapidx', usemap="map1", idx=2, noover=True))
if os.access('WARNINGS.txt', os.F_OK):
sys.stdout.write('<pre class="log">\n')
fp = open('WARNINGS.txt', 'rt', encoding='iso-8859-1')
for line in fp:
sys.stdout.write(u.html.escape(line))
fp.close()
sys.stdout.write('</pre>\n')
lines = []
sys.stdout.write('<table width="100%"><tr><td><pre>\n')
labels = set()
truelabels = {}
fp = open('../data/labels.txt', 'rt', encoding='iso-8859-1')
fp2 = open('../data/truelabels.txt', 'rt', encoding='utf-8')
for line in fp:
lbl = line.strip().split(None, 1)[1]
labels.add(lbl)
truelabels[lbl] = fp2.readline().strip()
fp2.close()
fp.close()
fp = open('map.lbl', 'rt', encoding='iso-8859-1')
for line in fp:
a, b = line.strip().split(None, 1)
b = _unquote(b)
if b in labels:
b = u.html.escape(truelabels[b])
lines.append((b, a))
sys.stdout.write('{:8d} {}\n'.format(int(a), b))
fp.close()
#loc = locale.getlocale()
#locale.setlocale(locale.LC_COLLATE, 'en_US')
sys.stdout.write('</pre>\n<td><pre>\n')
for b, a in sorted(lines):
sys.stdout.write('{:8d} {}\n'.format(int(a), b))
sys.stdout.write('</pre>\n</table>\n')
#locale.setlocale(locale.LC_COLLATE, loc)
elif os.access('QUEUED', os.F_OK):
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path[:-6])
if not os.path.isdir('items'):
os.mkdir('items')
os.chdir('items')
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgitems">
<h2>{}</h2>
'''.format(crumbs, title))
if os.access('../data/OK', os.F_OK) and os.access('../map/OK', os.F_OK):
items = []
method = open('../data/Method', 'rt').read().strip()
sys.stdout.write('''
<div class="info">
The map shows the total amount of data available for each location.
<br> <br>
Below the map is a list of all items — the column labels — in your data set.<br>
The number says how many instances are available for each item.
''')
if method.startswith('levfeat'):
sys.stdout.write('''<br> <br>
If there are two numbers for an item, the first is the number of instances you supplied,<br>
the second number is the number of instances that are actually used.
{}
'''.format(u.html.more('items')))
sys.stdout.write('''
<br> <br>
Click on a number to get a data map for a single item.
</div>
''')
if not os.access('datacount.txt', os.F_OK):
truelabels = {}
fp1 = open('../data/labels.txt', 'rt', encoding='iso-8859-1')
fp2 = open('../data/truelabels.txt', 'rt', encoding='utf-8')
for line in fp1:
lbl = line.split(None, 1)[1].strip()
truelabels[lbl] = fp2.readline().strip()
fp2.close()
fp1.close()
if method.startswith('levfeat'):
e = '.ftr'
else:
e = '.data'
p = {}
pn = {}
for filename in os.listdir('../data/_'):
if not filename.endswith(e):
continue
pseen = set()
fp = open('../data/_/' + filename, 'rb')
for line in fp:
if line[:1] == b':':
lbl = line[1:].decode('iso-8859-1').strip()
if not lbl in p:
p[lbl] = 0
if not lbl in pn:
pn[lbl] = 0
elif line[:1] == b'-' or line[:1] == b'+':
p[lbl] += 1
if not lbl in pseen:
pn[lbl] += 1
pseen.add(lbl)
fp.close()
fp = open('datacount.txt', 'wt', encoding='utf-8')
for i in sorted(p):
fp.write('{:6d}\t{:6d}\t{}\n'.format(pn[i], p[i], truelabels[i]))
fp.close()
m = max(pn.values())
m *= m
pp = {}
for i in pn:
pn[i] *= pn[i]
pp[i] = m
os.chdir('..')
u.distribute.distmap(pn, pp, 'items/datacount')
os.chdir('items')
p = path.split('-', 1)[1]
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.img(p + '-datacount', usemap="map1", bw=True))
sys.stdout.write('''
→ <a href="{}getdatacount?{}" target="_blank">download as list</a>
<p>
'''.format(u.config.binrel, pnum))
if not os.access('list2.utxt', os.F_OK):
for filename in os.listdir('../data/_'):
if not filename.endswith('.data'):
continue
item = re.sub('_([0-9]+)_', _num2chr, filename[:-5])
n = 0
fp = open('../data/_/' + filename, 'rb')
for line in fp:
if line[:1] == b'-' or line[:1] == b'+':
n += 1
fp.close()
if method.startswith('levfeat'):
m = 0
fp = open('../data/_/' + filename + '.ftr', 'rb')
for line in fp:
if line[:1] == b'+':
m += 1
fp.close()
items.append((item.lower(), item, n, m))
else:
items.append((item.lower(), item, n))
fp = open('list2.utxt', 'wt', encoding='utf-8')
if method.startswith('levfeat'):
for k, item, n, m in sorted(items):
fp.write('{}\t{}\t{}\n'.format(n, m, item))
else:
for k, item, n in sorted(items):
fp.write('{}\t{}\n'.format(n, item))
fp.close()
sys.stdout.write('<table class="items" cellspacing="0" cellpadding="0" border="0">\n')
fp = open('list2.utxt', 'rt', encoding='utf-8')
i = -1
if method.startswith('levfeat'):
for line in fp:
i += 1
n, m, item = line.split(None, 2)
if n == m:
n = ''
if n != '' and n != '0':
n = '<a href="imap?{}-{}-0" target="_blank">{}</a>'.format(pnum, i, n)
if m != '0':
m = '<a href="imap?{}-{}-1" target="_blank">{}</a>'.format(pnum, i, m)
sys.stdout.write('<tr><td align="right">{}<td align="right">{}<td>{}\n'.format(n, m, u.html.escape(item)))
else:
for line in fp:
i += 1
n, item = line.split(None, 1)
if n != '0':
n = '<a href="imap?{}-{}-0" target="_blank">{}</a>'.format(pnum, i, n)
sys.stdout.write('<tr><td align="right">{}<td>{}\n'.format(n, u.html.escape(item)))
fp.close()
sys.stdout.write('</table>\n')
elif os.access('../data/QUEUED', os.F_OK) or os.access('../map/QUEUED', os.F_OK):
os.chdir('../data/')
sys.stdout.write(u.html.busy())
else:
if os.access('../data/QUEUED', os.F_OK):
p = 'map'
else:
p = 'data'
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]).replace('items', p))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path[:-9])
os.chdir('data')
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgitems">
<h2>{}</h2>
'''.format(crumbs, title))
if os.access('OK', os.F_OK) and os.access('../map/OK', os.F_OK) and os.access('../diff/OK', os.F_OK):
sys.stdout.write('''
<div class="info">
The map shows the total amount of data available for each location.
<br> <br>
Below the map is a list of all items — the column labels — in your data set.<br>
The number (if any) says how many values are missing for each item.
<br> <br>
Click on a number to get a map of missing values.
</div>
<table class="items" cellspacing="0" cellpadding="0" border="0">
''')
if not os.access('datacount.txt', os.F_OK):
truelabels = {}
fp1 = open('labels.txt', 'rt', encoding='iso-8859-1')
fp2 = open('truelabels.txt', 'rt', encoding='utf-8')
for line in fp1:
lbl = line.split(None, 1)[1].strip()
truelabels[lbl] = fp2.readline().strip()
fp2.close()
fp1.close()
p = {}
fp = open('table.txt', 'rt', encoding='iso-8859-1')
fp.readline()
for line in fp:
i, j = line.rsplit('"', 1)
lbl = re.sub('\\\\(.)', '\\1', i[1:])
n = len([True for i in j.split() if i != 'NA'])
p[lbl] = n
fp.close()
fp = open('datacount.txt', 'wt', encoding='utf-8')
for i in sorted(p):
fp.write('{:6d}\t{}\n'.format(p[i], truelabels[i]))
fp.close()
m = max(p.values())
m *= m
pp = {}
for i in p:
p[i] *= p[i]
pp[i] = m
os.chdir('..')
u.distribute.distmap(p, pp, 'data/datacount')
os.chdir('data')
p = path.split('-', 1)[1].replace('numitems', 'data')
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.img(p + '-datacount', usemap="map1", bw=True))
sys.stdout.write('''
→ <a href="{}getdatacount?{}" target="_blank">download as list</a>
<p>
'''.format(u.config.binurls, pnum))
lines = []
fp = open('NAs.txt', 'rt', encoding='utf-8')
n = -1
for line in fp:
n += 1
a = line.split('\t')
lines.append((a[1].strip(), int(a[0]), n))
fp.close()
lines.sort()
for item, i, n in lines:
if i == 0:
i = ''
else:
i = '<a href="namap?{}-{}" target="_blank">{}</a>'.format(pnum, n, i)
sys.stdout.write('<tr><td align="right">{}<td>{}\n'.format(i, u.html.escape(item.strip())))
sys.stdout.write('</table>\n')
elif os.access('../diff/QUEUED', os.F_OK) or os.access('../map/QUEUED', os.F_OK):
os.chdir('../diff')
sys.stdout.write(u.html.busy())
else:
if os.access('../diff/QUEUED', os.F_OK):
p = 'map'
else:
p = 'diff'
sys.stdout.write(u.html.makeError(path.split('-', 1)[1].replace('numitems', p)))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
pnum = path.split('-')[-2].split('_')[-1]
fp = open('Method', 'rt')
method = fp.read().strip()
fp.close()
sys.stdout.write(u.html.head(ltitle))
sys.stdout.write('''
{}
<div class="pgdata">
<h2>{}</h2>
'''.format(crumbs, title))
if os.access('OK', os.F_OK):
if method.startswith('lev'):
sys.stdout.write('''
Contents:
<ul>
<li><a href="#overview">Overview</a>
<li><a href="#charlist">Character list</a>
<li><a href="#search">Sample search</a>
''')
if method.startswith('levfeat'):
sys.stdout.write('''
<li><a href="#errors">Errors</a>
<li><a href="#tokens">Token list</a>
''')
sys.stdout.write('</ul>\n')
sys.stdout.write('<h3 id="overview">Overview{}</h3>\n'.format(u.html.help('dataoverview')))
if os.access('comments.txt', os.F_OK):
sys.stdout.write('<pre class="log">\n')
fp = open('comments.txt', 'rt', encoding='utf-8')
for line in fp:
sys.stdout.write(u.html.escape(line))
fp.close()
sys.stdout.write('</pre>\n')
if method.startswith('levfeat'):
if not os.access('tokenstats.txt', os.F_OK):
sumtokens = 0
uniqtokens = 0
fp = open('tokens-int.txt', 'rt')
for line in fp:
if line.startswith('TOKEN'):
uniqtokens += 1
sumtokens += int(line.split()[-1])
fp.close()
fp = open('tokenstats.txt', 'wt')
fp.write('{} {}\n'.format(sumtokens, uniqtokens))
fp.close()
fp = open('tokenstats.txt', 'rt')
sumtokens, uniqtokens = fp.readline().split()
fp.close()
tokens = '\n<tr><td>Tokens:<td align="right">{}<tr><td>\nUnique tokens:<td align="right">{}\n'.format(sumtokens, uniqtokens)
else:
tokens = ''
fp = open('stats.txt', 'rt')
f = fp.readline().split()
fp.close()
if method.startswith('lev'):
charcount = []
charsum = 0
fp = open('charcount.txt', 'rt')
for line in fp:
c, n = [int(x) for x in line.split()]
charsum += 1
charcount.append((c, n))
fp.close()
characters = '\n<tr><td>Characters:<td align="right">{}\n<tr><td>Unique characters:<td align="right">{}'.format(f[3], charsum)
else:
characters = ''
sys.stdout.write('''
<table class="stats" border="0" cellspacing="0" cellpadding="0">
<tr><td>Places:<td align="right">{0[0]}
<tr><td>Items:<td align="right">{0[1]}
<tr><td>Instances:<td align="right">{0[2]}{1}{2}
</table>
'''.format(f, characters, tokens))
if method.startswith('lev'):
sys.stdout.write('<h3 id="charlist">Character list{}</h3>\n'.format(u.html.help('datacharlist')))
sys.stdout.write('<p>\nClick on number in third column for a distribution map and a list of samples<p>\n')
sys.stdout.write('<table class="charcount" border="1" cellspacing="0" cellpadding="4">\n')
for c, n in charcount:
if c > 255:
f1 = '''
<form method="post" action="http://www.unicode.org/cgi-bin/Code2Chart.pl"
enctype="application/x-www-form-urlencoded" target="_blank">
<input type="hidden" name="HexCode" value="{:04X}">
<input value="?" type="submit">
'''.format(c)
f2 = '</form>'
else:
f1 = f2 = ''
sys.stdout.write('''
<tr><!-- <td align="right"><tt>{0}</tt></td> -->
<td><tt>{0:04X}</tt></td>
<td align="center" class="ipa2"> &#{0}; </td>
<td align="right"><a href="{3}bin/sample?{4}-{0}" target="_blank">{1}</a></td>
<td>{5}<small>{2}</small>{6}</td>
</tr>
'''.format(c, n, unicodedata.name('%c' % c, ''), u.config.appurl, pnum, f1, f2))
sys.stdout.write('</table>\n')
sys.stdout.write('''
<h3 id="search">Sample search{}</h3>
<form action="{}bin/samplerx" method="post" enctype="multipart/form-data" accept-charset="utf-8" target="_blank">
<input type="hidden" name="hebci_auml" value="ä">
<input type="hidden" name="hebci_divide" value="÷">
<input type="hidden" name="hebci_euro" value="€">
<input type="hidden" name="hebci_middot" value="·">
<input type="hidden" name="hebci_oelig" value="œ">
<input type="hidden" name="hebci_oslash" value="ø">
<input type="hidden" name="hebci_Scaron" value="Š">
<input type="hidden" name="hebci_sect" value="§">
<input type="hidden" name="hebci_thorn" value="þ">
<input type="hidden" name="p" value="{}">
Regular expression:{}<br>
<input type="text" name="regex" size="40">
<input type="submit" value="Get samples">
'''.format(u.html.help('datasearch'), u.config.appurl, pnum, u.html.help('regex')))
if method.startswith('levfeat'):
sys.stdout.write('<h3 id="errors">Errors{}</h3>\n'.format(u.html.help('dataerrors')))
if os.access('UTF', os.F_OK):
encoding = 'utf-8'
else:
encoding = 'iso-8859-1'
lines = []
fp = open('errors-float.txt', 'rb')
for line in fp:
if not line.startswith(b'No mapping from real'):
lines.append(line)
fp.close()
if not lines:
sys.stdout.write('No errors\n')
else:
items = []
footer = []
sys.stdout.write('<p>\n<div class="log">\n')
for line in lines:
if line[:2] == b'_/':
a, b, c, line = line.split(b':', 3)
item = re.sub('_([0-9]+)_', _num2chr, a.decode('us-ascii')[2:].replace('.sp', '').replace('.data', ''))
msg = c.decode('us-ascii')
parts = line.split('\u001B'.encode('iso-8859-1'))
p1 = parts[0].decode(encoding).strip()
p2 = parts[1].decode(encoding)[6:].strip()
p3 = parts[2].decode('iso-8859-1')[3:].strip()
if p3[:1] == ':':
p3 = p3[1:].strip()
items.append((p3, item, p1, p2, msg))
else:
footer.append(line.decode('us-ascii'))
for place, item, p1, p2, msg in sorted(items):
if p2:
sys.stdout.write('''
<span class="ipa2">{}<span class="featerr">{}</span></span><br>
<span class="ghide"><span class="black">{} — {}</span> — {}: {:04X} — {}</span><br>
<br>
'''.format(u.html.escape(p1),
u.html.escape(p2),
u.html.escape(place),
u.html.escape(item),
u.html.escape(msg),
ord(p2[0]),
unicodedata.name(p2[0], '')))
else:
sys.stdout.write('''
<span class="ipa2">{}</span><br>
<span class="ghide"><span class="black">{} — {}</span> — {}</span><br>
<br>
'''.format(u.html.escape(p1),
u.html.escape(place),
u.html.escape(item),
u.html.escape(msg)))
for f in footer:
sys.stdout.write(f + '<br>\n')
sys.stdout.write('</div>\n')
if not method.endswith('user'):
charcats = {}
fp = open('charcat.txt', 'rt')
for line in fp:
a, b = line.split()
charcats['{:c}'.format(int(a))] = b
fp.close()
fp = open('tokens-int.txt', 'rt', encoding=encoding)
items = {}
for line in fp:
line = line.strip()
if not line:
continue
if line.startswith('TOKEN'):
continue
i, c = line.split()
c = c.replace('[[SP]]', ' ')
if not c in items:
items[c] = 0
items[c] += int(i)
fp.close()
sys.stdout.write('<h3 id="tokens">Token list{}</h3>\n'.format(u.html.help('datatokens')))
if not method.endswith('user'):
sys.stdout.write('<p><small>V = vowel, C = consonant, S = semivowel, U = unknown, X = stress, P = punctuation, M = modifier</small>\n')
sys.stdout.write('<br><small>1 = opening bracket, 2 = closing bracket</small>\n')
sys.stdout.write('<table class="charcount" border="1" cellspacing="0" cellpadding="4">\n')
for c in sorted(items):
xx = ' '.join(['{:04X}'.format(ord(x)) for x in c])
cc = '-'.join([str(ord(x)) for x in c])
if method.endswith('user'):
cats = ''
else:
cats = '<td><small>' + ' '.join([charcats[x] for x in c]) + '</small></td>'
sys.stdout.write('''<tr>
<td><tt>{5}</tt></td>
<td align="center" class="ipa2"> {0} </td>
<td align="right"><a href="{2}bin/sample2?{3}-{4}" target="_blank">{1}</a></td>
{6}
'''.format(u.html.escape(c), items[c], u.config.appurl, pnum, cc, xx, cats))
sys.stdout.write('</table>\n')
elif os.access('QUEUED', os.F_OK):
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path[:-8])
if not os.path.isdir('refmaps'):
os.mkdir('refmaps')
os.chdir('refmaps')
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgrefmaps">
<h2>reference point maps</h2>
'''.format(crumbs))
if os.access('../diff/OK', os.F_OK):
pseudo = False
if os.access('../map/PSEUDOMAP', os.F_OK):
pseudo = True
if os.access('current', os.F_OK):
fp = open('current', 'rt')
current = fp.read().split()
curplace, curmethod = [int(x) for x in current[:2]]
if len(current) > 2:
currev = int(current[2])
else:
currev = 0
fp.close()
else:
curplace = curmethod = currev = 0
if currev:
_colors.reverse()
places = {}
truelabels = {}
truelbl = []
fp = open('../data/labels.txt', 'rt', encoding='iso-8859-1')
fp2 = open('../data/truelabels.txt', 'rt', encoding='utf-8')
for line in fp:
a, b = line.strip().split(None, 1)
places[b] = int(a)
l = fp2.readline().strip()
truelabels[b] = l
truelbl.append(l)
fp2.close()
fp.close()
sys.stdout.write('''
<form action="{}refmapsform" method="post">
<input type="hidden" name="p" value="project_{}">
<fieldset>
<legend></legend>
Place: <select name="pl">
<option value="0">--</option>
'''.format(u.config.binurls, pnum))
for place in sorted(places):
if places[place] == curplace:
sel = ' selected="selected"'
placename = place
else:
sel = ''
sys.stdout.write('<option value="{}"{}>{}</option>\n'.format(places[place],
sel,
u.html.escape(truelabels[place])))
sel = [''] * 4
sel[curmethod] = ' selected="selected"'
if currev:
checked = ' checked="checked"'
else:
checked = ''
sys.stdout.write('''
</select><br>
<br>
Method: <select name="m">
<option value="0"{0[0]}>linear distances</option>
<option value="1"{0[1]}>quadratic distances</option>
<option value="2"{0[2]}>zero-based linear distances</option>
<option value="3"{0[3]}>zero-based quadratic distances</option>
</select><br>
<br>
<input type="checkbox" name="revcol"{1}> Reverse colours<br>
<br>
<input type="submit" value="Show map">
</fieldset>
</form>
'''.format(sel, checked))
if curplace:
if not os.access('curmap.eps', os.F_OK):
n, lbls, dif = _difread('../diff/diff.txt')
idx = lbls.index(placename)
diffs = [dif[idx][i] for i in range(n) if i != idx]
fmin = min(diffs)
fmax = max(diffs)
if curmethod == 2 or curmethod == 3:
fmin = 0.0
fp = open('current.rgb', 'wt', encoding='iso-8859-1')
fp.write('3\n{}\n1\n0\n0\n'.format(placename))
for i in range(n):
if i == idx:
continue
f = (dif[idx][i] - fmin) / (fmax - fmin)
if curmethod == 1 or curmethod == 3:
f = f * f
fi = int(f * _ncols)
if fi == _ncols:
fi = _ncols - 1
fp.write('{}\n{}\n'.format(lbls[i], _colors[fi]))
fp.close()
os.system('maprgb -r -o curmap.eps ../map/map.cfg current.rgb 2> /dev/null')
os.system('$PYTHON3 $APPDIR/util/smappost curmap.eps')
if currev:
os.system('ref2star -r curmap.eps')
else:
os.system('ref2star curmap.eps')
if not pseudo and not os.access('plot01.eps', os.F_OK):
if not os.access('geo.dst', os.F_OK):
os.system('difmodin ../data/labels.txt tmp.tbl')
os.system('difmod -o geo.dst ../map/map.geo tmp.tbl')
os.remove('tmp.tbl')
n2, lbls2, geo = _difread('geo.dst')
fp = open('curplot.data', 'wt')
assert n == n2
for i in range(n):
if i == idx:
continue
fp.write('{} {}\n'.format(geo[idx][i], dif[idx][i]))
fp.close()
fp = open('curplace.txt', 'wt', encoding='utf-8')
fp.write(truelabels[placename] + '\n')
fp.close()
os.system('R --no-save < {}util/refplot.R > plot.log 2>&1'.format(u.config.appdir))
state = 0
fpin = open('plot01.eps', 'rt', encoding='iso-8859-1')
fpout = open('plot01i.eps', 'wt', encoding='iso-8859-1')
for line in fpin:
if state == 0:
fpout.write(line)
if line.startswith('%%BeginProlog'):
fpout.write('''
/mys 10 string def
/myf (image.coo) (w) file def
/mylog {
2 copy transform
exch 2 {
round cvi mys cvs myf exch writestring myf ( ) writestring
} repeat
myf (\\n) writestring
} bind def
''')
state = 1
elif state == 1:
if line.strip().endswith(' c p1'):
a = line.split()
fpout.write('{0[0]} {0[1]} mylog {0[2]} c p1\n'.format(a))
elif line.startswith('%%EOF'):
fpout.write('myf closefile\n')
fpout.write(line)
state = 2
else:
fpout.write(line)
else:
fpout.write(line)
fpout.close()
fpin.close()
os.system('eps2png > eps2png.log 2>&1')
n = -1
fp = open('image.coo', 'rt')
lines = fp.readlines()
fp.close()
i = 0
fp = open('image.coo', 'wt', encoding='iso-8859-1')
for line in lines:
if i == idx:
i += 1
fp.write(line.strip() + ' ' + truelbl[i].encode('iso-8859-1', 'xmlcharrefreplace').decode('iso-8859-1') + '\n')
i += 1
fp.close()
os.system('mkmap map2')
sys.stdout.write(u.html.img('project_{}-refmaps-curmap'.format(pnum), True, usemap="map1", idx=1))
if not pseudo:
fp = open('image.html', 'rt', encoding='utf-8')
sys.stdout.write(fp.read())
fp.close()
sys.stdout.write('<p>\n' + u.html.img('project_{}-refmaps-plot01'.format(pnum), usemap="map2", noover=True, idx=2))
elif os.access('../diff/QUEUED', os.F_OK):
os.chdir('../diff/')
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]).replace('refmaps', 'diff'))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path[:-6])
if not os.path.isdir('align'):
os.mkdir('align')
os.chdir('align')
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.head(ltitle))
sys.stdout.write('''
{}
<div class="pgalign">
<h2>alignments</h2>
'''.format(crumbs))
if os.access('../data/OK', os.F_OK):
fp = open('../data/Method', 'rt')
m = methods[fp.read().strip()]
fp.close()
sys.stdout.write('Method: {}\n'.format(m))
if os.access('../data/tokens-int.txt', os.F_OK):
features = True
else:
features = False
if not os.access('tokenlist.txt', os.F_OK):
if features:
_tokens2list()
else:
_charcount2list()
if os.access('current', os.F_OK):
fp = open('current', 'rt', encoding='utf-8')
current = fp.readline().strip()
try:
curplace = fp.readline().strip()
except:
curplace = '0'
fp.close()
else:
current = ''
curplace = '0'
sys.stdout.write('''
<form action="{}bin/alignform" method="post">
<input type="hidden" name="p" value="project_{}">
<fieldset>
<legend></legend>
Item: <select name="n">
<option>--</option>
'''.format(u.config.appurl, pnum))
filenames = [
x[:-5] for x in os.listdir('../data/_') if x.endswith('.data')
]
lines = []
for filename in sorted(filenames):
itemname = re.sub('_([0-9]+)_', _num2chr, filename)
if itemname == current:
sel = ' selected="selected"'
else:
sel = ''
lines.append(
(itemname.lower(), '<option value="{}"{}>{}</option>\n'.format(
filename, sel, u.html.escape(itemname))))
for a, b in sorted(lines):
sys.stdout.write(b)
sys.stdout.write('''
</select><br>
<br>
Place: <select name="l">
<option value="0"> -- random places --</option>
''')
lines = []
truelabels = {}
pseudolabels = {}
fp = open('../data/labels.txt', 'rt', encoding='iso-8859-1')
fp2 = open('../data/truelabels.txt', 'rt', encoding='utf-8')
for line in fp:
a, b = line.split(None, 1)
b = b.strip()
lines.append('{}\t{}'.format(b, a))
l = fp2.readline().strip()
truelabels[b] = l
pseudolabels[l.encode(
'iso-8859-1', 'xmlcharrefreplace').decode('iso-8859-1')] = l
fp2.close()
fp.close()
lines.sort()
for line in lines:
b, a = line.split('\t', 1)
if a == curplace:
sel = ' selected="selected"'
else:
sel = ''
sys.stdout.write('<option value="{}"{}>{}</option>\n'.format(
a, sel, u.html.escape(truelabels[b])))
sys.stdout.write(
'</select>\n<br> <br>\n<input type="submit" value="Show alignments">\n</fieldset>\n</form>\n'
)
if current:
sys.stdout.write('<h3>{}</h3>\n'.format(u.html.escape(current)))
try:
fp = open('page', 'rt')
except:
page = 0
pages = 0
fp = open('alignments.txt', 'rb')
else:
page = int(fp.read())
fp.close()
fp = open('pages', 'rt')
pages = int(fp.read())
fp.close()
fp = open('alignments{}.txt'.format(page), 'rb')
if page:
sys.stdout.write('Page: ')
if page == 1:
sys.stdout.write(' «')
else:
sys.stdout.write(
' <a href="alignpage?{1}-{0}">«</a>'.format(
page - 1, pnum))
for i in range(1, pages + 1):
if i == page:
sys.stdout.write(' <b>{}</b>'.format(i))
else:
sys.stdout.write(
' <a href="alignpage?{1}-{0}">{0}</a>'.format(
i, pnum))
if page == pages:
sys.stdout.write(' »')
else:
sys.stdout.write(
' <a href="alignpage?{1}-{0}">»</a>'.format(
page + 1, pnum))
sys.stdout.write('''<br>→
<a href="aligndownload?p=project_{}" target="_blank">download as a text file </a><br>
<p>\n'''.format(pnum))
intab = False
inItem = False
for line in fp:
line = line.strip(b'\n')
if not line:
if intab:
sys.stdout.write('</table>\n')
intab = False
elif line[:1] == b'[':
line = line.decode('iso-8859-1')
lbl = pseudolabels[line.partition(']')[2].strip()]
if inItem:
sys.stdout.write(' — ')
inItem = False
else:
inItem = True
sys.stdout.write(u.html.escape(lbl) + '\n')
else:
line = line.decode('utf-8')
if not intab:
sys.stdout.write('<table class="align">\n')
rownum = 0
intab = True
rownum += 1
if rownum < 3:
sys.stdout.write('<tr class="ipa2">')
else:
sys.stdout.write('<tr>')
prev = '0'
for i in line[1:].split('\t'):
if rownum == 3:
if i == prev:
i = ''
else:
ii = i
i = '{:g}'.format(float(i) - float(prev))
prev = ii
if i == '[[SP]]':
ii = '<span class="space">SP</span>'
else:
ii = u.html.escape(i)
sys.stdout.write('<td> {} \n'.format(ii))
if rownum == 3:
sys.stdout.write(
'<td class="total"> {} \n'.format(
line.split('\t')[-1]))
else:
sys.stdout.write('<td class="white"> \n')
fp.close()
if page:
sys.stdout.write('<p>Page: ')
if page == 1:
sys.stdout.write(' «')
else:
sys.stdout.write(
' <a href="alignpage?{1}-{0}">«</a>'.format(
page - 1, pnum))
for i in range(1, pages + 1):
if i == page:
sys.stdout.write(' <b>{}</b>'.format(i))
else:
sys.stdout.write(
' <a href="alignpage?{1}-{0}">{0}</a>'.format(
i, pnum))
if page == pages:
sys.stdout.write(' »')
else:
sys.stdout.write(
' <a href="alignpage?{1}-{0}">»</a>'.format(
page + 1, pnum))
sys.stdout.write('<p>\n')
elif os.access('../data/QUEUED', os.F_OK):
os.chdir('../data/')
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(
u.html.makeError(path.split('-', 1)[1]).replace('align', 'data'))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
crumbs = u.path.breadcrumbs(path)
ltitle = path.split("-")[1].replace("_", " ") + " / " + title
pnum = path.split("-")[-2].split("_")[-1]
p = path.split("-", 1)[1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write(
"""
{}
<div class="pgdiff">
<h2>{}</h2>
""".format(
crumbs, title
)
)
if os.access("OK", os.F_OK):
fp = open("ca.txt", "r")
ca = fp.read().strip()
fp.close()
if os.access("../map/PSEUDOMAP", os.F_OK):
linc = "undefined for pseudomap"
else:
fp = open("linc.txt", "r")
linc = fp.read()
fp.close()
try:
fp = open("../data/Method", "rt")
m = fp.readline().strip()
fp.close()
except:
pass
else:
mt = "Method: {}\n<p>".format(_methods[m])
sys.stdout.write(mt + "\n")
sys.stdout.write("Local incoherence: {}{}\n<p>\n".format(_number(linc), u.html.help("linc")))
sys.stdout.write("Cronbach's alpha: {}{}\n<p>\n".format(_number(ca), u.html.help("ca")))
sys.stdout.write(
"""
→ <a href="{0}bin/getdiff?p=project_{1}&f=L04" target="_blank">download differences</a> (RuG/L04 format)
<p>
→ <a href="{0}bin/getdiff?p=project_{1}&f=tab" target="_blank">download differences</a> (table format)
""".format(
u.config.appurl, pnum
)
)
if m.startswith("levfeat"):
sys.stdout.write(
"""
<p>
→ <a href="{}bin/getfeat?p=project_{}" target="_blank">download feature definition</a>
""".format(
u.config.appurl, pnum
)
)
sys.stdout.write(
"""
<p>
{}
<p>
{}
<p>
{}
""".format(
u.html.img(p + "-diff01"),
u.html.img(p + "-diff1", usemap="map1", idx=1),
u.html.img(p + "-diff", usemap="map1", idx=2),
)
)
elif os.access("QUEUED", os.F_OK):
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split("-", 1)[1]))
sys.stdout.write("\n</div>\n")
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
pnum = int(path.split('-')[1].split('_')[-1])
sys.stdout.write(u.html.head(ltitle))
sys.stdout.write('''
{}
<div class="pgprob">
<h2>linguistic difference ↔ geographic distance</h2>
'''.format(crumbs))
if os.access('OK', os.F_OK):
sys.stdout.write('''
→ <a href="getplotdata?{}">download R data</a>{}
<p>
'''.format(pnum, u.html.help("plotr")))
try:
fp = open('plot01.log', 'rt', encoding='utf-8')
txt = fp.read()
fp.close()
except:
txt = ''
else:
txt = '<pre class="log">\n' + u.html.escape(txt) + '</pre>\n'
try:
fp = open('plot02.log', 'rt', encoding='utf-8')
txt2 = fp.read()
fp.close()
except:
txt2 = ''
else:
txt2 = '<pre class="log">\n' + u.html.escape(txt2) + '</pre>\n'
sys.stdout.write('''
{}
<p>
{}
<p>
<hr>
'''.format(u.html.img(p + '-plot02'), txt2))
if (os.access('plot01.eps', os.F_OK)):
sys.stdout.write('''
<p>
{}
<p>
A plot with local regression (red) and asymptotic regression (blue).<br>
A large b/a ratio indicates a large signal/noise ratio in the data.<br>
A small value for c indicates that linguistic variation is measurable over a short geographic distance.
<p>
{}
<p>
<hr>
'''.format(u.html.img(p + '-plot01'), txt))
else:
# TO DO: link to help page on reason for failure
sys.stdout.write('''
<p>
A plot with asymptotic regression failed.
<p>
{}
<p>
<hr>
'''.format(txt))
sys.stdout.write('''
<p>
{}
'''.format(u.html.img(p + '-plot03')))
elif os.access('QUEUED', os.F_OK):
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path[:-9])
os.chdir("data")
crumbs = u.path.breadcrumbs(path)
ltitle = path.split("-")[1].replace("_", " ") + " / " + title
p = path.split("-", 1)[1]
pnum = path.split("-")[-2].split("_")[-1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write(
"""
{}
<div class="pgitems">
<h2>{}</h2>
""".format(
crumbs, title
)
)
if os.access("OK", os.F_OK) and os.access("../map/OK", os.F_OK) and os.access("../diff/OK", os.F_OK):
sys.stdout.write(
"""
<div class="info">
The map shows the total amount of data available for each location.
<br> <br>
Below the map is a list of all items — the column labels — in your data set.<br>
The number (if any) says how many values are missing for each item.
<br> <br>
Click on a number to get a map of missing values.
</div>
<table class="items" cellspacing="0" cellpadding="0" border="0">
"""
)
if not os.access("datacount.txt", os.F_OK):
truelabels = {}
fp1 = open("labels.txt", "rt", encoding="iso-8859-1")
fp2 = open("truelabels.txt", "rt", encoding="utf-8")
for line in fp1:
lbl = line.split(None, 1)[1].strip()
truelabels[lbl] = fp2.readline().strip()
fp2.close()
fp1.close()
p = {}
fp = open("table.txt", "rt", encoding="iso-8859-1")
fp.readline()
for line in fp:
i, j = line.rsplit('"', 1)
lbl = re.sub("\\\\(.)", "\\1", i[1:])
n = len([True for i in j.split() if i != "NA"])
p[lbl] = n
fp.close()
fp = open("datacount.txt", "wt", encoding="utf-8")
for i in sorted(p):
fp.write("{:6d}\t{}\n".format(p[i], truelabels[i]))
fp.close()
m = max(p.values())
m *= m
pp = {}
for i in p:
p[i] *= p[i]
pp[i] = m
os.chdir("..")
u.distribute.distmap(p, pp, "data/datacount")
os.chdir("data")
p = path.split("-", 1)[1].replace("numitems", "data")
pnum = path.split("-")[-2].split("_")[-1]
sys.stdout.write(u.html.img(p + "-datacount", usemap="map1", bw=True))
sys.stdout.write(
"""
→ <a href="{}bin/getdatacount?{}" target="_blank">download as list</a>
<p>
""".format(
u.config.appurl, pnum
)
)
lines = []
fp = open("NAs.txt", "rt", encoding="utf-8")
n = -1
for line in fp:
n += 1
a = line.split("\t")
lines.append((a[1].strip(), int(a[0]), n))
fp.close()
lines.sort()
for item, i, n in lines:
if i == 0:
i = ""
else:
i = '<a href="namap?{}-{}" target="_blank">{}</a>'.format(pnum, n, i)
sys.stdout.write('<tr><td align="right">{}<td>{}\n'.format(i, u.html.escape(item.strip())))
sys.stdout.write("</table>\n")
elif os.access("../diff/QUEUED", os.F_OK) or os.access("../map/QUEUED", os.F_OK):
os.chdir("../diff")
sys.stdout.write(u.html.busy())
else:
if os.access("../diff/QUEUED", os.F_OK):
p = "map"
else:
p = "diff"
sys.stdout.write(u.html.makeError(path.split("-", 1)[1].replace("numitems", p)))
sys.stdout.write("\n</div>\n")
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path[:-8])
os.chdir('data')
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
pnum = path.split('-')[-2].split('_')[-1]
sys.stdout.write(u.html.head(ltitle))
sys.stdout.write('''
{}
<div class="pgdata">
<h2>{}</h2>
'''.format(crumbs, title))
if os.access('OK', os.F_OK):
sys.stdout.write('''
Contents:
<ul>
<li><a href="#overview">Overview</a>
<li><a href="#boxplot">Box plots</a>
</ul>
''')
sys.stdout.write('<h3 id="overview">Overview{}</h3>\n'.format(u.html.help('numdataoverview')))
if os.access('comments.txt', os.F_OK):
sys.stdout.write('<pre class="log">\n')
fp = open('comments.txt', 'rt', encoding='utf-8')
for line in fp:
sys.stdout.write(u.html.escape(line))
fp.close()
sys.stdout.write('</pre>\n')
if os.access('stats.txt', os.F_OK):
fp = open('stats.txt', 'rt')
nPlaces, nItems, NAs = [int(x) for x in fp.read().split()]
Total = nPlaces * nItems
Values = Total - NAs
sys.stdout.write('''
<table class="stats" border="0" cellspacing="0" cellpadding="0">
<tr><td>Places:<td align="right">{0}
<tr><td>Items:<td align="right">{1}
<tr><td>Values:<td align="right">{2}<td>(of {4})<td align="right">{5:.3f}%
<tr><td>Missing:<td align="right">{3}<td>(of {4})<td align="right">{6:.3f}%
</table>
<p>
'''.format(nPlaces, nItems, Values, NAs, Total, Values * 100.0 / Total, NAs * 100.0 / Total))
sys.stdout.write('<h3 id="boxplot">Box plots{}</h3>\n'.format(u.html.help('numdataboxplot')))
sys.stdout.write(u.html.img(p.replace('numdata', 'data') + '-boxplot01'))
if os.access('boxplot02.eps', os.F_OK):
sys.stdout.write('''
un-normalised values
<p>
{}
normalised values
'''.format(u.html.img(p.replace('numdata', 'data') + '-boxplot02')))
elif os.access('../diff/QUEUED', os.F_OK):
os.chdir('../diff')
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1].replace('numitems', 'diff')))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
title = path.split('-', 1)[1].replace('-', '/').replace('_', ' ')
try:
fp = open('description', 'rt', encoding='utf-8')
description = fp.readline().strip()
fp.close()
except:
description = ''
fp = open('data/Method', 'rt')
method = fp.read().strip()
fp.close()
header = '''<script type="text/javascript"><!--
var visible = false;
function EDIT () {{
s = document.getElementById('edit').style;
if (visible) {{
s.visibility = 'hidden';
s.display = 'none';
visible = false;
}} else {{
s.visibility = 'visible';
s.display = 'block';
visible = true;
}}
s = document.getElementById('title').style;
if (visible) {{
s.visibility = 'hidden';
s.display = 'none';
}} else {{
s.visibility = 'visible';
s.display = 'block';
}}
}}
function CANCEL () {{
s = document.getElementById('edit').style;
s.visibility = 'hidden';
s.display = 'none';
s = document.getElementById('title').style;
s.visibility = 'visible';
s.display = 'block';
visible = false;
}}
function DEL() {{
var answer = confirm("Remove project {0}?\\n\\nThis will destroy all data and results for this project");
if (answer) {{
window.location = "{1}bin/projectremove?p={0}";
}}
}}
//--></script>
'''.format(path.split('_')[-1], u.config.appurl)
sys.stdout.write(u.html.head(title, headers=header))
sys.stdout.write('''
<div class="pgproject">
<h2>{0} <a href="javascript:DEL()" title="Remove"> × <!--✗--></a></h2>
<div id="title">
{1} <a href="javascript:EDIT()" title="Edit"> « </a>
</div>
<div id="edit" style="visibility:hidden;display:none">
<form action="{3}bin/editproject" method="post" enctype="multipart/form-data" accept-charset="utf-8" name="editproject">
<input type="hidden" name="hebci_auml" value="ä">
<input type="hidden" name="hebci_divide" value="÷">
<input type="hidden" name="hebci_euro" value="€">
<input type="hidden" name="hebci_middot" value="·">
<input type="hidden" name="hebci_oelig" value="œ">
<input type="hidden" name="hebci_oslash" value="ø">
<input type="hidden" name="hebci_Scaron" value="Š">
<input type="hidden" name="hebci_sect" value="§">
<input type="hidden" name="hebci_thorn" value="þ">
<input type="hidden" name="p" value="{2}">
Description:<br>
<input type="text" name="description" size="80" value="{1}"><br>
<input type="submit" value="Save">
<input onclick="CANCEL()" type="button" value="Cancel">
</form>
</div>
'''.format(title, description, path.split('_')[-1], u.config.appurl))
if method.startswith('num'):
num = 'num'
else:
num = ''
sys.stdout.write('''<p>
<table class="project">
<tr><td colspan="2"><hr>
<tr valign="top">
<td>Index
<td><ul>
<li><a href="{0}bin/goto?p={1}-map">places</a>
'''.format(u.config.appurl, path.split('-', 1)[1]))
if not method.startswith('dif'):
sys.stdout.write('''
<li><a href="{0}bin/goto?p={1}-{2}items">items</a>
'''.format(u.config.appurl, path.split('-', 1)[1], num))
sys.stdout.write('''
</ul>
<tr valign="top">
<td>Data inspection
<td><ul>
''')
if method.startswith('num'):
sys.stdout.write('''
<li><a href="{0}bin/goto?p={1}-numdata">data overview</a>
<li><a href="{0}bin/goto?p={1}-nummap">value maps</a>
'''.format(u.config.appurl, path.split('-', 1)[1]))
elif method.startswith('dif'):
sys.stdout.write('''
<li><a href="{0}bin/goto?p={1}-difdata">data overview</a>
'''.format(u.config.appurl, path.split('-', 1)[1]))
else:
sys.stdout.write('''
<li><a href="{0}bin/goto?p={1}-data">data overview</a>
'''.format(u.config.appurl, path.split('-', 1)[1]))
sys.stdout.write('<li><a href="{0}bin/goto?p={1}-distmap">distribution maps</a>\n'.format(
u.config.appurl, path.split('-', 1)[1]))
sys.stdout.write('</ul>\n')
if method.startswith('lev'):
sys.stdout.write('''
<tr valign="top">
<td>Measuring technique
<td><ul>
<li><a href="{0}bin/goto?p={1}-align">alignments</a>
</ul>
'''.format(u.config.appurl, path.split('-', 1)[1]))
sys.stdout.write('''
<tr valign="top">
<td>Differences
<td><ul>
<li><a href="{0}bin/goto?p={1}-diff">statistics and difference maps</a>
'''.format(u.config.appurl, path.split('-', 1)[1]))
if not os.access('map/PSEUDOMAP', os.F_OK):
sys.stdout.write('''
<li><a href="{0}bin/goto?p={1}-plot">linguistic difference ↔ geographic distance</a>
'''.format(u.config.appurl, path.split('-', 1)[1]))
sys.stdout.write('''
<li><a href="{0}bin/goto?p={1}-refmaps">reference point maps</a>
</ul>
<tr><td colspan="2"><hr>
<tr valign="top">
<td>Multidimensional Scaling
<td><ul>
<li><a href="{0}bin/goto?p={1}-mdsplots">mds plots</a>
<li><a href="{0}bin/goto?p={1}-mdsmaps">mds maps</a>
</ul>
<tr valign="top">
<td>Discrete clustering
<td><ul>
<li><a href="{0}bin/goto?p={1}-clusters">cluster maps and dendrograms</a>
<li><a href="{0}bin/goto?p={1}-clumaps">cluster validation</a>
</ul>
<tr valign="top">
<td>Fuzzy clustering
<td><ul>
<li><a href="{0}bin/goto?p={1}-prob">probabilistic dendrogram</a>
<li><a href="{0}bin/goto?p={1}-cccmaps">fuzzy cluster maps</a>
</ul>
'''.format(u.config.appurl, path.split('-', 1)[1]))
if not method.startswith('num') and not method.startswith('dif'):
sys.stdout.write('''
<tr><td colspan="2"><hr>
<tr valign="top">
<td>Data mining
<td><ul>
<li><a href="{0}bin/goto?p={1}-cludet">cluster determinants</a>
</ul>
'''.format(u.config.appurl, path.split('-', 1)[1]))
sys.stdout.write('<tr><td colspan="2"><hr>\n</table>\n')
sys.stdout.write('</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
pnum = path.split('-')[-2].split('_')[-1]
fp = open('Method', 'rt')
method = fp.read().strip()
fp.close()
sys.stdout.write(u.html.head(ltitle))
sys.stdout.write('''
{}
<div class="pgdata">
<h2>{}</h2>
'''.format(crumbs, title))
if os.access('OK', os.F_OK):
if method.startswith('lev'):
sys.stdout.write('''
Contents:
<ul>
<li><a href="#overview">Overview</a>
<li><a href="#charlist">Character list</a>
<li><a href="#search">Sample search</a>
''')
if method.startswith('levfeat'):
sys.stdout.write('''
<li><a href="#errors">Errors</a>
<li><a href="#tokens">Token list</a>
''')
sys.stdout.write('</ul>\n')
sys.stdout.write('<h3 id="overview">Overview{}</h3>\n'.format(
u.html.help('dataoverview')))
if os.access('comments.txt', os.F_OK):
sys.stdout.write('<pre class="log">\n')
fp = open('comments.txt', 'rt', encoding='utf-8')
for line in fp:
sys.stdout.write(u.html.escape(line))
fp.close()
sys.stdout.write('</pre>\n')
if method.startswith('levfeat'):
if not os.access('tokenstats.txt', os.F_OK):
sumtokens = 0
uniqtokens = 0
fp = open('tokens-int.txt', 'rt')
for line in fp:
if line.startswith('TOKEN'):
uniqtokens += 1
sumtokens += int(line.split()[-1])
fp.close()
fp = open('tokenstats.txt', 'wt')
fp.write('{} {}\n'.format(sumtokens, uniqtokens))
fp.close()
fp = open('tokenstats.txt', 'rt')
sumtokens, uniqtokens = fp.readline().split()
fp.close()
tokens = '\n<tr><td>Tokens:<td align="right">{}<tr><td>\nUnique tokens:<td align="right">{}\n'.format(
sumtokens, uniqtokens)
else:
tokens = ''
fp = open('stats.txt', 'rt')
f = fp.readline().split()
fp.close()
if method.startswith('lev'):
charcount = []
charsum = 0
fp = open('charcount.txt', 'rt')
for line in fp:
c, n = [int(x) for x in line.split()]
charsum += 1
charcount.append((c, n))
fp.close()
characters = '\n<tr><td>Characters:<td align="right">{}\n<tr><td>Unique characters:<td align="right">{}'.format(
f[3], charsum)
else:
characters = ''
sys.stdout.write('''
<table class="stats" border="0" cellspacing="0" cellpadding="0">
<tr><td>Places:<td align="right">{0[0]}
<tr><td>Items:<td align="right">{0[1]}
<tr><td>Instances:<td align="right">{0[2]}{1}{2}
</table>
'''.format(f, characters, tokens))
if method.startswith('lev'):
sys.stdout.write(
'<h3 id="charlist">Character list{}</h3>\n'.format(
u.html.help('datacharlist')))
sys.stdout.write(
'<p>\nClick on number in third column for a distribution map and a list of samples<p>\n'
)
sys.stdout.write(
'<table class="charcount" border="1" cellspacing="0" cellpadding="4">\n'
)
for c, n in charcount:
if c > 255:
f1 = '''
<form method="post" action="http://www.unicode.org/cgi-bin/Code2Chart.pl"
enctype="application/x-www-form-urlencoded" target="_blank">
<input type="hidden" name="HexCode" value="{:04X}">
<input value="?" type="submit">
'''.format(c)
f2 = '</form>'
else:
f1 = f2 = ''
sys.stdout.write('''
<tr><!-- <td align="right"><tt>{0}</tt></td> -->
<td><tt>{0:04X}</tt></td>
<td align="center" class="ipa2"> &#{0}; </td>
<td align="right"><a href="{3}bin/sample?{4}-{0}" target="_blank">{1}</a></td>
<td>{5}<small>{2}</small>{6}</td>
</tr>
'''.format(c, n, unicodedata.name('%c' % c, ''),
u.config.appurl, pnum, f1, f2))
sys.stdout.write('</table>\n')
sys.stdout.write('''
<h3 id="search">Sample search{}</h3>
<form action="{}bin/samplerx" method="post" enctype="multipart/form-data" accept-charset="utf-8" target="_blank">
<input type="hidden" name="hebci_auml" value="ä">
<input type="hidden" name="hebci_divide" value="÷">
<input type="hidden" name="hebci_euro" value="€">
<input type="hidden" name="hebci_middot" value="·">
<input type="hidden" name="hebci_oelig" value="œ">
<input type="hidden" name="hebci_oslash" value="ø">
<input type="hidden" name="hebci_Scaron" value="Š">
<input type="hidden" name="hebci_sect" value="§">
<input type="hidden" name="hebci_thorn" value="þ">
<input type="hidden" name="p" value="{}">
Regular expression:{}<br>
<input type="text" name="regex" size="40">
<input type="submit" value="Get samples">
'''.format(u.html.help('datasearch'), u.config.appurl, pnum,
u.html.help('regex')))
if method.startswith('levfeat'):
sys.stdout.write('<h3 id="errors">Errors{}</h3>\n'.format(
u.html.help('dataerrors')))
if os.access('UTF', os.F_OK):
encoding = 'utf-8'
else:
encoding = 'iso-8859-1'
lines = []
fp = open('errors-float.txt', 'rb')
for line in fp:
if not line.startswith(b'No mapping from real'):
lines.append(line)
fp.close()
if not lines:
sys.stdout.write('No errors\n')
else:
items = []
footer = []
sys.stdout.write('<p>\n<div class="log">\n')
for line in lines:
if line[:2] == b'_/':
a, b, c, line = line.split(b':', 3)
item = re.sub(
'_([0-9]+)_', _num2chr,
a.decode('us-ascii')[2:].replace('.sp',
'').replace(
'.data', ''))
msg = c.decode('us-ascii')
parts = line.split('\u001B'.encode('iso-8859-1'))
p1 = parts[0].decode(encoding).strip()
p2 = parts[1].decode(encoding)[6:].strip()
p3 = parts[2].decode('iso-8859-1')[3:].strip()
if p3[:1] == ':':
p3 = p3[1:].strip()
items.append((p3, item, p1, p2, msg))
else:
footer.append(line.decode('us-ascii'))
for place, item, p1, p2, msg in sorted(items):
if p2:
sys.stdout.write('''
<span class="ipa2">{}<span class="featerr">{}</span></span><br>
<span class="ghide"><span class="black">{} — {}</span> — {}: {:04X} — {}</span><br>
<br>
'''.format(u.html.escape(p1), u.html.escape(p2),
u.html.escape(place), u.html.escape(item),
u.html.escape(msg), ord(p2[0]),
unicodedata.name(p2[0], '')))
else:
sys.stdout.write('''
<span class="ipa2">{}</span><br>
<span class="ghide"><span class="black">{} — {}</span> — {}</span><br>
<br>
'''.format(u.html.escape(p1), u.html.escape(place),
u.html.escape(item), u.html.escape(msg)))
for f in footer:
sys.stdout.write(f + '<br>\n')
sys.stdout.write('</div>\n')
if not method.endswith('user'):
charcats = {}
fp = open('charcat.txt', 'rt')
for line in fp:
a, b = line.split()
charcats['{:c}'.format(int(a))] = b
fp.close()
fp = open('tokens-int.txt', 'rt', encoding=encoding)
items = {}
for line in fp:
line = line.strip()
if not line:
continue
if line.startswith('TOKEN'):
continue
i, c = line.split()
c = c.replace('[[SP]]', ' ')
if not c in items:
items[c] = 0
items[c] += int(i)
fp.close()
sys.stdout.write('<h3 id="tokens">Token list{}</h3>\n'.format(
u.html.help('datatokens')))
if not method.endswith('user'):
sys.stdout.write(
'<p><small>V = vowel, C = consonant, S = semivowel, U = unknown, X = stress, P = punctuation, M = modifier</small>\n'
)
sys.stdout.write(
'<br><small>1 = opening bracket, 2 = closing bracket</small>\n'
)
sys.stdout.write(
'<table class="charcount" border="1" cellspacing="0" cellpadding="4">\n'
)
for c in sorted(items):
xx = ' '.join(['{:04X}'.format(ord(x)) for x in c])
cc = '-'.join([str(ord(x)) for x in c])
if method.endswith('user'):
cats = ''
else:
cats = '<td><small>' + ' '.join([charcats[x] for x in c
]) + '</small></td>'
sys.stdout.write('''<tr>
<td><tt>{5}</tt></td>
<td align="center" class="ipa2"> {0} </td>
<td align="right"><a href="{2}bin/sample2?{3}-{4}" target="_blank">{1}</a></td>
{6}
'''.format(u.html.escape(c), items[c], u.config.appurl, pnum,
cc, xx, cats))
sys.stdout.write('</table>\n')
elif os.access('QUEUED', os.F_OK):
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
project = path.split('-')[1]
pnum = path.split('-')[-2].split('_')[-1]
if not os.access('items.txt', os.F_OK):
items = {}
for filename in os.listdir('../data/_'):
if not filename.endswith('.data'):
continue
fname = filename.replace('.data', '')
iname = re.sub('_([0-9]+)_', _num2chr, fname)
items[fname] = iname
fp = open('items.txt', 'wt', encoding='utf-8')
for i in sorted(items):
fp.write('{}\t{}\n'.format(i, items[i]))
fp.close()
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgcludet">
<h2>cluster determinants</h2>
'''.format(crumbs))
if os.access('OK', os.F_OK):
_setup()
isPseudo = os.access('../map/PSEUDOMAP', os.F_OK)
fp = open('version', 'rt')
mtd = fp.read().strip()
fp.close()
try:
fp = open('currentparms', 'rt')
Beta = fp.read().split()[0]
fp.close()
except:
Beta = 'β'
if os.access('../data/UTF', os.F_OK):
encoding = 'utf-8'
else:
encoding = 'iso-8859-1'
sys.stdout.write('''<div class="info">
Here you can discover what linguistic features are characteristic for certain areas.<br>
<br>
For an introduction, read <a href="{}doc/ClusterDeterminants" target="_blank">this demonstration</a>
</div>'''.format(u.config.appurls))
accents = {}
if encoding == 'utf-8':
if not os.access('accents.txt' ,os.F_OK):
fpin = open('../data/charcount.txt', 'rt')
fpout = open('accents.txt', 'wt')
for line in fpin:
i = int(line.split()[0])
c = u.setChar.ci(i)
if c != 'V' and c != 'S' and c != 'C':
fpout.write('{}\n'.format(i))
fpout.close()
fpin.close()
fp = open('accents.txt', 'rt')
for line in fp:
accents[int(line)] = False
fp.close()
if not os.access('accentscurrent.txt' ,os.F_OK):
fp = open('accentscurrent.txt', 'wt')
for j in defaults:
i = int(j.split()[0], 16)
if i in accents:
fp.write('{}\n'.format(i))
fp.close()
fp = open('accentscurrent.txt', 'rt')
for line in fp:
accents[int(line)] = True
fp.close
sys.stdout.write('<h3 id="s1">Step 1: select number of clusters</h3>\n' + u.html.img(
p + '-clmap', usemap="map1", idx=1, pseudoforce=True) + '\n')
fp = open('current', 'rt')
current = fp.read().split()
fp.close()
sys.stdout.write('''
<p>
<form action="{}cludetform" method="post" enctype="multipart/form-data">
<input type="hidden" name="p" value="{}">
<input type="hidden" name="action" value="number">
Number of clusters:
<select name="n">
'''.format(u.config.binurls, project))
n = int(current[0])
for i in range(2, 13):
if i == n:
sys.stdout.write('<option selected="selected">{}</option>\n'.format(i))
else:
sys.stdout.write('<option>{}</option>\n'.format(i))
sys.stdout.write('''
</select>
<input type="submit" value="Change number">
</form>
<p>
''')
if len(current) > 1:
curclnum = int(current[1])
else:
curclnum = 0
sys.stdout.write('''
<h3 id="s2">Step 2: select cluster</h3>
<form action="{}cludetform" method="post" enctype="multipart/form-data">
<input type="hidden" name="p" value="{}">
<input type="hidden" name="action" value="cluster">
'''.format(u.config.binurls, project))
if accents:
sys.stdout.write('''
<div class="accents">
These characters are ignored, unless checked:
<div class="ipa2">
''')
for i in sorted(accents):
if i == 32:
s = 'SPACE'
else:
s = ' &#{}; '.format(i)
if accents[i]:
v = ' checked="checked"'
else:
v = ''
nm = unicodedata.name('{:c}'.format(i), '')
if nm:
a1 = '<abbr title="{}">'.format(u.html.escape(nm))
a2 = '</abbr>'
else:
a1 = a2 = ''
sys.stdout.write('<span class="cdc">{}<input type="checkbox" name="chr{}"{}> {}{}</span>\n'.format(a1, i, v, s, a2))
sys.stdout.write('''
</div>
</div>
<br>
''')
if isPseudo:
sys.stdout.write('<input type="hidden" name="method" value="fast">\n')
else:
s1 = s2 = ''
s = ' selected="selected"'
if mtd == 'fast':
s1 = s
elif mtd == 'slow':
s2 = s
sys.stdout.write('''
Method:
<select name="method">
<option value="fast"{}>raw data</option>
<option value="slow"{}>localised data</option>
</select>{}
<p>
'''.format(s1, s2, u.html.help('cludetfastslow')))
sys.stdout.write('''
Clusters in plot:
'''.format(u.config.appurl, project))
for i in range(1, n + 1):
if i == curclnum:
c = ' checked="checked"'
else:
c = ''
sys.stdout.write('<span class="s{0}"><input type="radio" name="c" value="{0}"{1}></span>\n'.format(i, c))
sys.stdout.write('''
<input type="submit" value="Select cluster">
</form>
<p>
''')
if os.access('score.txt', os.F_OK):
if len(current) > 2:
curitem = current[2]
else:
curitem = ''
sys.stdout.write('''
<h3 id="s3">step 3: select item</h3>
<form action="{}cludetform" method="post" enctype="multipart/form-data">
<input type="hidden" name="p" value="{}">
<input type="hidden" name="action" value="item">
Items sorted by value:
<select name="item">
'''.format(u.config.binurls, project))
fp = open('score.txt', 'rt')
for line in fp:
a, b, c, f, g = line.split()
if f.split(':')[0] == '0':
continue
gg = g[2:-5]
if gg == curitem:
sel = ' selected="selected"'
else:
sel = ''
sys.stdout.write('<option value="{}"{}>{} {} {}</option>\n'.format(
gg, sel, a, _toStrHtml(gg), f))
sys.stdout.write('''
</select>
<input type="submit" value="Select item">
<br>→ <a href="cludetlist?p={}" target="_blank">download as list</a>
</form>
<p>
'''.format(project))
if curitem:
if not os.access('currentlist.txt', os.F_OK):
partition = set()
fp = open('clgroups.txt', 'rt', encoding='iso-8859-1')
for line in fp:
a, b = line.split(None, 1)
if int(a) == curclnum:
partition.add(_unquote(b))
fp.close()
variants = {}
variantsin = {}
fp = open('../data/_/' + curitem + '.data', 'rb')
for line in fp:
if line[:1] == b':':
lbl = line[1:].strip().decode('iso-8859-1')
elif line[:1] == b'-':
v = line[1:].strip().decode(encoding)
if not v in variants:
variants[v] = 0
variantsin[v] = 0
variants[v] += 1
if lbl in partition:
variantsin[v] += 1
fp.close()
fp = open('currentlist.txt', 'wt', encoding='utf-8')
for v in sorted(variants):
fp.write('{}:{}\t{}\n'.format(variantsin[v], variants[v], v))
fp.close()
if curitem:
if not os.access('currentselect.txt', os.F_OK):
fpin = open('_/' + curitem + '.utxt', 'rt')
fpout = open('currentselect.txt', 'wt')
fpout2 = open('currentreject.txt', 'wt')
for line in fpin:
line = line.strip()
if not line:
continue
elif line[0] == '[':
fpout2.write(line[1:-1] + '\n')
continue
elif line[-1] != ']':
continue
for a in line[:-2].split('[ ')[1].split(' | '):
fpout.write(a + '\n')
fpout2.close()
fpout.close()
fpin.close()
if curitem:
wrdcount = {}
fp = open('currentlist.txt', 'rt', encoding='utf-8')
for line in fp:
a, b = line.split(None, 1)
wrdcount[b.strip()] = a
fp.close()
fp = open('_/' + curitem + '.utxt', 'rt')
lines = fp.readlines()
fp.close()
sys.stdout.write(('''
<table style="margin:1em 0px;padding:0px;border:0px" cellpadding="0" cellspacing="0" border="0">
<tr valign="top"><td style="padding-right:4em">
Current item: {0}
<table cellspacing="0" cellpadding="0" border="0">
<tr><td>Adjusted F<sub>{3}</sub> score: <td>{1[0]}{2}
<tr><td>— Adjusted Precision: <td>{1[1]}
<tr><td>— Adjusted Recall: <td>{1[2]}
</table>
Patterns with forms:
<ul>
''').format(_toStrHtml(curitem), lines[-1].split(), u.html.help('adjustedfscore'), Beta))
for line in lines[:-1]:
if line[0] == '[':
continue
if line.strip():
a, b, c, d, e, f = line.split(None, 5)
sys.stdout.write('''<li><span class="ipa2">{}</span> {}<br>
{} - {} - {}<br>
'''.format( _toStrHtml(d, True), e, a, b, c))
wrds = [re.sub('_([0-9]+)_', _num2chr, w) for w in f.split() if w != '[' and w != '|' and w != ']']
if len(wrds) > 1 or _toStrHtml(d) != u.html.escape(wrds[0]):
sys.stdout.write('<ul>\n')
for wrd in sorted(wrds):
sys.stdout.write('<li><span class="ipa2">{}</span> {}\n'.format(
u.html.escape(wrd), wrdcount[wrd]))
sys.stdout.write('</ul>\n')
sys.stdout.write('''
</ul>
</td>
''')
if os.access('currentreject.txt', os.F_OK):
sys.stdout.write('''
<td style="padding-left:2em;border-left:1px solid #808080">
Rejected patterns:<br>
<br>
''')
fp = open('currentreject.txt', 'rt')
for line in fp:
line = line.strip()
a, b = line.rsplit(None, 1)
sys.stdout.write('<span class="ipa2">{}</span> {}<br>\n'.format(_toStrHtml(a, True), b))
fp.close()
sys.stdout.write('</td>\n')
sys.stdout.write('</tr>\n</table>\n')
if curitem:
if not os.access('distmap.ex', os.F_OK):
fpout = open('distmap.ex', 'wt', encoding='iso-8859-1')
fpin = open('clgroups.txt', 'rt', encoding='iso-8859-1')
for line in fpin:
a, b = line.split(None, 1)
if a == current[1]:
fpout.write('1 ' + b)
fpin.close()
if os.access('../map/map.ex', os.F_OK):
fpin = open('../map/map.ex', 'rt', encoding='iso-8859-1')
for line in fpin:
fpout.write(line)
fpin.close()
fpout.close()
if not os.access('distmap.eps', os.F_OK):
variantset = set()
fp = open('currentselect.txt', 'rt')
for line in fp:
variantset.add(re.sub('_([0-9]+)_', _num2chr, line.strip()))
fp.close()
placen = {}
placeall = {}
fp = open('../data/_/{}.data'.format(curitem), 'rb')
for line in fp:
if line[:1] == b':':
place = line[1:].decode('iso-8859-1').strip()
if not place in placen:
placen[place] = 0
placeall[place] = 0
elif line[:1] == b'-':
if line[1:].strip().decode(encoding) in variantset:
placen[place] += 1
placeall[place] += 1
fp.close()
os.chdir('..')
u.distribute.distmap(placen, placeall, 'cludet/distmap', exfile='cludet/distmap.ex', normalise=False)
os.chdir('cludet')
if curitem:
sys.stdout.write(u.html.img('project_{}-cludet-distmap'.format(pnum), True, usemap="map1", idx=2))
if os.access('currentregex.txt', os.F_OK):
fp = open('currentregex.txt', 'rt', encoding='utf-8')
regex = fp.read().strip()
fp.close()
else:
regex = ''
sys.stdout.write('''
<h3 id="s4">Step 4: try for determinant feature</h3>
<form action="{}cludetform" method="post" enctype="multipart/form-data">
<input type="hidden" name="hebci_auml" value="ä">
<input type="hidden" name="hebci_divide" value="÷">
<input type="hidden" name="hebci_euro" value="€">
<input type="hidden" name="hebci_middot" value="·">
<input type="hidden" name="hebci_oelig" value="œ">
<input type="hidden" name="hebci_oslash" value="ø">
<input type="hidden" name="hebci_Scaron" value="Š">
<input type="hidden" name="hebci_sect" value="§">
<input type="hidden" name="hebci_thorn" value="þ">
<input type="hidden" name="p" value="{}">
<input type="hidden" name="action" value="regex">
Regular expression:{}<br><input type="text" name="regex" size="60" value="{}" class="ipa2">
<input type="submit" value="Try feature">
</form>
'''.format(u.config.binurls, project, u.html.help('regex'), regex))
if regex:
fp = open('reresults.txt', 'rt')
results = fp.read().split()
fp.close()
sys.stdout.write(('''
<br>
Current regular expression: <span class="ipa2">{0}</span><br>
<table cellspacing="0" cellpadding="0" border="0">
<tr><td>Adjusted F<sub>{2}</sub> score: <td>{1[0]}
<tr><td>— Adjusted Precision: <td>{1[1]}
<tr><td>— Adjusted Recall: <td>{1[2]}
</table>
Matching forms:
''').format(regex, results, Beta))
found = False
fp = open('rematches.txt', 'rt', encoding='utf-8')
for line in fp:
if not found:
sys.stdout.write('<ul>\n')
found = True
a, b = line.split(None, 1)
sys.stdout.write('<li><span class="ipa2">{}</span> {}\n'.format(u.html.escape(b.strip()), a))
fp.close()
if found:
sys.stdout.write('</ul>\n')
else:
sys.stdout.write('none\n')
if regex and not os.access('redistmap.eps', os.F_OK):
variantset = set()
fp = open('rematches.txt', 'rt', encoding='utf-8')
for line in fp:
variantset.add(line.split(None, 1)[1].strip())
fp.close()
placen = {}
placeall = {}
fp = open('../data/_/{}.data'.format(curitem), 'rb')
for line in fp:
if line[:1] == b':':
place = line[1:].decode('iso-8859-1').strip()
if not place in placen:
placen[place] = 0
placeall[place] = 0
elif line[:1] == b'-':
if line[1:].strip().decode(encoding) in variantset:
placen[place] += 1
placeall[place] += 1
fp.close()
os.chdir('..')
u.distribute.distmap(placen, placeall, 'cludet/redistmap', exfile='cludet/distmap.ex', normalise=False)
os.chdir('cludet')
if regex:
sys.stdout.write(u.html.img('project_{}-cludet-redistmap'.format(pnum), True, usemap="map1", idx=3))
elif os.access('QUEUED', os.F_OK):
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
p = path.split('-', 1)[1]
project = path.split('-')[1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgclumaps">
<h2>{}</h2>
'''.format(crumbs, title))
if os.access('OK', os.F_OK):
sys.stdout.write('''
<div class="info">
The results of clustering can be deceptive. So,
the purpose of this page is not just to devide your data into dialects groups, but to test the validity of each group.<br>
<br>
For an introduction, read <a href="../doc/ClusterValidation" target="_blank">this demonstration</a>
</div>
''')
open('reset', 'wt').close()
current2 = ['wm']
try:
fp = open('current', 'rt')
lines = fp.readlines()
fp.close()
current = lines[0].split()
if len(lines)> 1:
current2 = lines[1].split()
except:
current = ['c', '6', 'all']
if current[0] == 'c':
bw = ''
else:
bw = 'bw'
try:
with open('../data/stats.txt', 'r') as fp:
nPlaces = int(fp.read().strip().split()[0])
except:
nPlaces = 2
method = current2[0]
n = min(int(current[1]), nPlaces)
current = current[2:]
logfile = 'plot_{}_{}{}_{}.log'.format(method, n, bw, '_'.join(current))
if os.access(logfile, os.F_OK):
corr = '<small><i>r</i> = ' + _number(open(logfile, 'rt').read().strip().split()[-1]) + '</small>'
else:
corr = ''
sys.stdout.write('''
{}
<p>
Above: Clustering using <b>{}</b>.
<p>
Below: An MDS-plot of the clusters from the map shows what major cluster divisions are "real".
You can omit one major group of clusters to inspect the divisions of the remaining clusters.
{}
{}
<p>
'''.format(u.html.img(p + '-{}{}{}'.format(method, n, bw), usemap="map1", pseudoforce=True),
methods[method],
u.html.img((p + '-plot_{}_{}{}_{}'.format(method, n, bw, '_'.join(current)))),
corr))
sys.stdout.write('''
<form action="{}bin/clumapsform" method="post">
<input type="hidden" name="p" value="{}">
<input type="hidden" name="action" value="method">
<em>Note: each of the following buttons changes a single option</em>
<p>
Clustering method:
<select name="m">
'''.format(u.config.appurl, project))
for i in sorted(methods):
if i == method:
sys.stdout.write('<option selected="selected" value="{}">{}</option>\n'.format(i, methods[i]))
else:
sys.stdout.write('<option value="{}">{}</option>\n'.format(i, methods[i]))
sys.stdout.write('''
</select>
<input type="submit" value="Change method">
</form>
<p>
''')
sys.stdout.write('''
<form action="{}bin/clumapsform" method="post">
<input type="hidden" name="p" value="{}">
<input type="hidden" name="action" value="number">
Number of clusters:
<select name="n">
'''.format(u.config.appurl, project))
for i in range(2, 13):
if i == n:
sys.stdout.write('<option selected="selected">{}</option>\n'.format(i))
else:
sys.stdout.write('<option>{}</option>\n'.format(i))
sys.stdout.write('''
</select>
<input type="submit" value="Change number">
</form>
<p>
''')
sys.stdout.write('''
<form action="{}bin/clumapsform" method="post">
<input type="hidden" name="p" value="{}">
<input type="hidden" name="action" value="subset">
Clusters in plot:
'''.format(u.config.appurl, project))
if current[0] == 'all':
subset = set(range(1, n + 1))
else:
subset = set(int(x) for x in current)
if bw == 'bw':
for i in range(1, n + 1):
if i in subset:
sys.stdout.write('<span class="sym"><input type="checkbox" name="s{0}" checked="checked"><img src="{1}img/sym{0:02d}.png"></span>\n'.format(i, u.config.appurl))
else:
sys.stdout.write('<span class="sym"><input type="checkbox" name="s{0}"<img src="{1}img/sym{0:02d}.png"></span>\n'.format(i, u.config.appurl))
else:
for i in range(1, n + 1):
if i in subset:
sys.stdout.write('<span class="s{0}"><input type="checkbox" name="s{0}" checked="checked"></span>\n'.format(i))
else:
sys.stdout.write('<span class="s{0}"><input type="checkbox" name="s{0}"></span>\n'.format(i))
sys.stdout.write('''
<input type="submit" value="Change subset">
</form>
<p>
''')
if bw == 'bw':
sys.stdout.write('''
<form action="{}bin/clumapsform" method="post">
<input type="hidden" name="p" value="{}">
<input type="hidden" name="action" value="col">
<input type="submit" value="Switch to colour">
</form>
'''.format(u.config.appurl, project))
else:
sys.stdout.write('''
<form action="{}bin/clumapsform" method="post">
<input type="hidden" name="p" value="{}">
<input type="hidden" name="action" value="bw">
<input type="submit" value="Switch to black/white">
</form>
'''.format(u.config.appurl, project))
elif os.access('QUEUED', os.F_OK):
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]))
if os.access('reset', os.F_OK):
for filename in os.listdir('.'):
if filename.startswith('current') or filename.startswith('tmp'):
os.remove(filename)
open('OK', 'wt').close()
sys.stdout.write('<p>\n<a href="">Continue</a>\n')
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())
def makepage(path):
u.path.chdir(path)
crumbs = u.path.breadcrumbs(path)
ltitle = path.split('-')[1].replace('_', ' ') + ' / ' + title
pnum = path.split('-')[-2].split('_')[-1]
p = path.split('-', 1)[1]
sys.stdout.write(u.html.head(ltitle, tip=True, maptip=True))
sys.stdout.write('''
{}
<div class="pgdiff">
<h2>{}</h2>
'''.format(crumbs, title))
if os.access('OK', os.F_OK):
fp = open('ca.txt', 'r')
ca = fp.read().strip()
fp.close()
if os.access('../map/PSEUDOMAP', os.F_OK):
linc = 'undefined for pseudomap'
else:
fp = open('linc.txt', 'r')
linc = fp.read()
fp.close()
try:
fp = open('../data/Method', 'rt')
m = fp.readline().strip()
fp.close()
except:
pass
else:
mt = 'Method: {}\n<p>'.format(_methods[m])
sys.stdout.write(mt + '\n')
sys.stdout.write('Local incoherence: {}{}\n<p>\n'.format(
_number(linc), u.html.help('linc')))
sys.stdout.write('Cronbach\'s alpha: {}{}\n<p>\n'.format(
_number(ca), u.html.help('ca')))
sys.stdout.write('''
→ <a href="{0}bin/getdiff?p=project_{1}&f=L04" target="_blank">download differences</a> (RuG/L04 format)
<p>
→ <a href="{0}bin/getdiff?p=project_{1}&f=tab" target="_blank">download differences</a> (table format)
'''.format(u.config.appurl, pnum))
if m.startswith('levfeat'):
sys.stdout.write('''
<p>
→ <a href="{}bin/getfeat?p=project_{}" target="_blank">download feature definition</a>
'''.format(u.config.appurl, pnum))
sys.stdout.write('''
<p>
{}
<p>
{}
<p>
{}
'''.format(u.html.img(p + '-diff01'),
u.html.img(p + '-diff1', usemap="map1", idx=1),
u.html.img(p + '-diff', usemap="map1", idx=2)))
elif os.access('QUEUED', os.F_OK):
sys.stdout.write(u.html.busy())
else:
sys.stdout.write(u.html.makeError(path.split('-', 1)[1]))
sys.stdout.write('\n</div>\n')
sys.stdout.write(u.html.foot())