def create_hits(trips_set,db,n):
r = etree.Element("hitlist")
for t in trips_set:
head = etree.SubElement(r,"hit")
etree.SubElement(head,"reward").text = reward
etree.SubElement(head,"num_copies").text = str(n)
etree.SubElement(head,"db_name").text = db
data = etree.SubElement(head,"hit_data")
for i in t:
etree.SubElement(data,"triple").text = str(i)
i = 1
while True:
fname = "c:/vision/hitset"+str(i)+".xml"
if not os.path.exists(fname):
break
i+=1
etree.ElementTree(r).write(fname)
print "Created file: "+fname
# f = open(block_list_xml,"w")
# f.write("<blocklist>\n")
# for wid in block_list+reject_list:
# f.write("<workerID>"+wid+"</workerID>\n")
# f.write("</blocklist>\n")
# f.close()
# print "Created file: "+block_list_xml
# pipe =os.popen("ImageSimilarityMechanicalTurk.exe -create "+fname+" "+block_list_xml+" 2>&1",'r')
text = tools.my_run("ImageSimilarityMechanicalTurk.exe -create "+fname,ce_root)
print text
def oops(label=None):
if label==None:
label=get_last_label()
st = "deleteHits -successfile "+log_root+"/"+label+".success -force -approve -expire"
text = tools.my_run(st,turk_root).strip()
my_log("[DELETE] "+st+"\n"+text+"\n")
return text
def load_hit_helper(label,input2, question2, properties2, comment):
if comment!=None:
fname = log_root+"/"+label+"_"+comment+".comment"
tools.my_write(fname,"blah")
st = "loadHITs -input "+input2+" -question "+question2+" -properties "+properties2+" -label "+label
text = tools.my_run(st,turk_root)
my_log("[LOAD HIT] "+st+"\nLABEL: "+label+"\n"+text)
shutil.move(turk_root+"/"+label+".success",log_root+"/"+label+".success")
#now find errors, try to reload 10 times
# for i in range(10):
# etxt = text.split("[ERROR] Error creating HIT ")[1:]
# errors = [int(a[:a.find(" ")]) for a in etxt]
# n = len(errors)
# if n==0:
# return (label,text)
# for i in range(n):
# print "...trying to fix ",etxt[i][:30]
# input3=input2+"_temp.input"
# s = tools.my_read(input2).splitlines()
# s2 = [s[0]]+[s[i] for i in errors]
# print "new inputs:",s2
# s3 = string.join(s2,"\n")
# tools.my_write(input3,s3)
# st = "loadHITs -input "+input3+" -question "+question2+" -properties "+properties2+" -label "+label
# text = tools.my_run(st,turk_root)
# my_log("[LOAD HIT] "+st+"\nLABEL: "+label+"\n"+text)
# shutil.move(turk_root+"/"+label+".success",log_root+"/"+label+".success")
#
#
# for i in range(10):
# print "*** FAILED TO LOAD ALL HITS!"
return (label,text)
def load_hit_helper(label,input2, question2, properties2, comment):
if comment!=None:
fname = log_root+"/"+label+"_"+comment+".comment"
tools.my_write(fname,"blah")
st = "loadHITs -input "+input2+" -question "+question2+" -properties "+properties2+" -label "+label
text = tools.my_run(st,turk_root)
my_log("[LOAD HIT] "+st+"\nLABEL: "+label+"\n"+text)
shutil.move(turk_root+"/"+label+".success",log_root+"/"+label+".success")
return (label,text)
def get_results(label = None):
label = label or get_last_label()
label = str(label)
successfile = log_root+"/"+label+".success"
resultsfile = log_root+"/"+label+".results"
st = "getResults -successfile "+successfile+" -outputfile "+resultsfile
text = tools.my_run(st,turk_root)
my_log("[GET RESULTS] "+st+"\nLABEL: "+label+"\n"+text)
return text
def get_balance():
text = tools.my_run("getBalance",turk_root).strip()
s = text.find("$")
if s==-1:
my_log("[BALANCE] getbalance\nCould not get balance: "+text)
return -1
else:
b = float(text[s+1:])
my_log("[BALANCE] getbalance\nGot balance of: "+str(b))
return b
def classify(dir,att,info_file=None,fit_file = None):
global model_file, data_file,pred_file
if dir[-1]!="/" and dir[-1]!="\\":
dir+="/"
if info_file==None:
info_file = dir+"info.csv"
if fit_file==None:
fit_file = glob.glob(dir+"*fit.txt")[0]
(x,y) = create_data(dir,att,info_file,fit_file,data_file)
st2= tools.my_run("c:/users/adum/simexp/svmlight/svm_learn.exe -x 1 "+data_file+" "+model_file)
w, thresh = read_model(model_file)
print st2
print thresh
res = [r[0] for r in np.dot(x,w.T)-thresh]
res2 = [(res[i],chr(65+i)) for i in range(26)]
res2.sort()
print res2
st3= tools.my_run("c:/users/adum/simexp/svmlight/svm_classify.exe "+data_file+" "+model_file+" "+pred_file)
print st3
def reject_work(wids,label=None):
label = label or get_last_label()
file = open(log_root+"/"+label+".results","r")
blah = csv.DictReader(file,delimiter='\t')
aids = []
for r in blah:
wid = r['workerid']
if wid in wids:
aids.append(r['assignmentid'])
file.close()
ttext = ""
for aid in aids:
st = "rejectWork -assignment "+aid+' -force'
text = tools.my_run(st,turk_root).strip()
my_log("[REJECT WORK]"+st+text+"\n")
ttext+=st+"\n"+text+"\n"
return ttext
def reject_work(wids, label=None):
label = label or turk.get_last_label()
file = open(turk.log_root + "/" + label + ".results", "r")
blah = csv.DictReader(file, delimiter='\t')
aids = []
for r in blah:
wid = r['workerid']
if wid in wids:
aids.append(r['assignmentid'])
file.close()
ttext = ""
for aid in aids:
st = "rejectWork -assignment " + aid + ' -force'
text = tools.my_run(st, turk.turk_root).strip()
my_log("[REJECT WORK]" + st + text + "\n")
ttext += st + "\n" + text + "\n"
return ttext
def flush():
monitor = load_monitor()
print "swish..."
blocked = 0
failed = []
while monitor.block_buffer:
wid = monitor.block_buffer.pop()
res = tools.my_run(mturk_bin_dir+"/blockWorker -workerid "+wid+" -reason "+block_reason,mturk_bin_dir)
if ('Blocked '+wid) in res:
blocked+=1
else:
print "Failed to block:",wid
print "*"+res
failed.append(wid)
monitor.block_buffer = failed
if blocked>0:
print "Blocked "+str(blocked)+" workers"
save_monitor(monitor)
def flush():
monitor = load_monitor()
print "swish..."
blocked = 0
failed = []
while monitor.block_buffer:
wid = monitor.block_buffer.pop()
res = tools.my_run(
mturk_bin_dir + "/blockWorker -workerid " + wid + " -reason " +
block_reason, mturk_bin_dir)
if ('Blocked ' + wid) in res:
blocked += 1
else:
print "Failed to block:", wid
print "*" + res
failed.append(wid)
monitor.block_buffer = failed
if blocked > 0:
print "Blocked " + str(blocked) + " workers"
save_monitor(monitor)
def redo_work(wids=[],hitids=[],label=None):
label = label or get_last_label()
file = open(log_root+"/"+label+".results","r")
blah = csv.DictReader(file,delimiter='\t')
st = "hitid\thittypeid\n"
for r in blah:
wid = r['workerid']
if r['hitid'] in hitids:
st+=r['hitid']+"\t"+r['hittypeid']+"\n"
hitids.remove(r['hitid'])
else:
if wid in wids:
st+=r['hihttp://www.mturk.com/mturk/preview?groupId=1DS91QNJYJ9DKY1BZYKGZ13YD5OGVLtid']+"\t"+r['hittypeid']+"\n"
file.close()
sfname = log_root+"/"+label+"_temp.success"
tools.my_write(sfname,st)
st = "extendHITs -assignments 1 -hours 24 -successfile " + sfname
text = tools.my_run(st,turk_root).strip()
my_log("[EXTEND_HITS]"+st+text+"\n")
return st+"\n"+text+"\n"
def redo_work(wids=[], hitids=[], label=None):
label = label or get_last_label()
file = open(log_root + "/" + label + ".results", "r")
blah = csv.DictReader(file, delimiter='\t')
st = "hitid\thittypeid\n"
for r in blah:
wid = r['workerid']
if r['hitid'] in hitids:
st += r['hitid'] + "\t" + r['hittypeid'] + "\n"
hitids.remove(r['hitid'])
else:
if wid in wids:
st += r[
'hihttp://www.mturk.com/mturk/preview?groupId=1DS91QNJYJ9DKY1BZYKGZ13YD5OGVLtid'] + "\t" + r[
'hittypeid'] + "\n"
file.close()
sfname = log_root + "/" + label + "_temp.success"
tools.my_write(sfname, st)
st = "extendHITs -assignments 1 -hours 24 -successfile " + sfname
text = tools.my_run(st, turk_root).strip()
my_log("[EXTEND_HITS]" + st + text + "\n")
return st + "\n" + text + "\n"
def check_hits():
text2 = tools.my_run("ImageSimilarityMechanicalTurk.exe -check", ce_root)
print text2
m = re.search('(.*) out of (.*) are finished!', text2)
assert m
return m.group(1) == m.group(2)
def check_hits():
text2 = tools.my_run("ImageSimilarityMechanicalTurk.exe -check",ce_root)
print text2
m=re.search('(.*) out of (.*) are finished!',text2)
assert m
return m.group(1)==m.group(2)
# -*- coding: utf-8 -*-
"""
Created on Tue Oct 05 22:18:04 2010
@author: adum
"""
import tools
tools.my_run("tree2html.py c:/sim/trees/neckties.tree9 neckties 0.75",
"c:/sim/py")
tools.my_run("tree2html.py c:/sim/trees/animals.tree9 animals 1.1",
"c:/sim/py")
tools.my_run("tree2html.py c:/sim/trees/flags.tree9 flags 1.4", "c:/sim/py")
tools.my_run("tree2html.py c:/sim/trees/dogs.tree9 dogs 1", "c:/sim/py")
tools.my_run("tree2html.py c:/sim/trees/lfw.tree9 lfw 1", "c:/sim/py")
tools.my_run("tree2html.py c:/sim/trees/newtiles.tree9 newtiles 1",
"c:/sim/py")
tools.my_run("tree2html.py c:/sim/trees/signs.tree9 signs 1", "c:/sim/py")
k += 1
f.write("\n")
print >>f, "EOF"
f.close()
fonts = tools.my_read("c:/data/fonts/ids.txt").splitlines()
fonts2 = [i.split(".")[0] for i in fonts]
fonts2.sort()
TSPLIB_format()
tools.my_run("lkh MyFonts.par","c:/sim/tsp")
tour = tools.my_read("c:/sim/tsp/MyFonts.tour").strip().splitlines()
tour = tour[6:-2]
tour = [int(i)-1 for i in tour]
#real_tour = []
#for i in tour:
# if i<5:
# real_tour.append(i)
# if i>5:
# real_tour.append(i-1)
#tour=real_tour
print tour
fonts3 = []
for i in range(len(fonts2)):
# -*- coding: utf-8 -*-
"""
Created on Tue Oct 05 22:18:04 2010
@author: adum
"""
import tools
tools.my_run("tree2html.py c:/sim/trees/neckties.tree9 neckties 0.75","c:/sim/py")
tools.my_run("tree2html.py c:/sim/trees/animals.tree9 animals 1.1","c:/sim/py")
tools.my_run("tree2html.py c:/sim/trees/flags.tree9 flags 1.4","c:/sim/py")
tools.my_run("tree2html.py c:/sim/trees/dogs.tree9 dogs 1","c:/sim/py")
tools.my_run("tree2html.py c:/sim/trees/lfw.tree9 lfw 1","c:/sim/py")
tools.my_run("tree2html.py c:/sim/trees/newtiles.tree9 newtiles 1","c:/sim/py")
tools.my_run("tree2html.py c:/sim/trees/signs.tree9 signs 1","c:/sim/py")