def printPhaseTimes(db, study_videos_only, all_videos):
table = [[
"video", "phase list", "phase", "phase order", "width", "first go",
"crowd go", "first ping", "crowd ping", "agreement", "total",
"num workers", "on retainer"
]]
rows = []
for video in all_videos:
video_rows = parseVideo(video['pk'], study_videos_only, db)
rows.extend(video_rows)
if len(rows) > 0:
median_row = ["median", "median", "median", "median"]
for column in range(4, len(rows[0])):
numpy_data = filter(lambda x: x is not None,
list(numpy.array(rows)[:, column]))
medians = numpy.median(numpy.array([numpy_data]))
median_row.append(medians)
rows = [median_row] + rows
for row in rows:
table.append(map(stringOrNothing, row))
pprint_table(sys.stdout, table)
def __main__():
# I have no idea what p_cs is but it is needed for posting things like changes to records -
# seems to be some sort of session id. The p_cs you get from logging in is not the one we
# need. We need the p_cs that is from the links in the main menu.
# They are obtained via searching for p_cs=([0-9*)
# cs0 is obtained from the https://solss.uow.edu.au/sid/sols_logon.validate_sols_logon reply - it is unneeded
# cs1 is obtained from the https://solss.uow.edu.au/sid/sols_menu.display_sols_menu reply - it is needed for posts to things in the main menu
# cs2 is obtained from the printEnrolmentRecord reply - it is needed for posts activating/deactivating enrolment record copies
(sessionId, studentNum) = startNewSession()
cs1 = getCs1(sessionId, studentNum)
cs2 = getCs2(sessionId, studentNum, cs1)
# Print session information in a neat table
print "New session started."
table = [["Username", username],
["Student Number", studentNum],
["Session ID", sessionId],
["CS1", cs1],
["CS2", cs2]]
out = sys.stdout
pprint_table(out, table)
# enrolmentTables is a list of pretty print tables. The first element of the list is a table of the enrolments for the first lab; the second is a table of the enrolments for the second lab and so on.
# Each enrolment table consists of "Name", "First Day and Time toEnrol" and "Last Day and Time to Enrol".
enrolmentTables = getListOfEnrolmentTables(sessionId, studentNum, cs1)
print ""
for enrolmentTable in enrolmentTables:
pprint_table(out, enrolmentTable)
def summarize_experiment(dbfile, resultsd):
try:
kmeansf = os.path.join(resultsd, "kmeans.results.json")
kmeans_results = json.load(open(kmeansf))
data = kmeans_results["data"]
all_requests = []
for req in get_keys(data, "runs", "1", "firstView", "requests"):
all_requests.append(req["full_url"])
all_requests = set(all_requests)
with sqlite3.connect(dbfile) as db:
cursor = db.cursor()
cursor.execute(
"select ruri from moz_page_predictions, moz_pages "
"where moz_pages.id = moz_page_predictions.pid and "
"moz_pages.uri = ?", (data["testUrl"],))
prediction = [tp[0] for tp in cursor]
if not prediction:
host = get_host_for_uri(data["testUrl"])
cursor.execute(
"select ruri from moz_host_predictions, moz_hosts "
"where moz_hosts.id = moz_host_predictions.hid and "
"moz_hosts.origin = ?", (host,))
prediction = [tp[0] for tp in cursor]
cursor.execute(
"select moz_subresources.uri from moz_subresources, moz_pages "
"where moz_pages.id = moz_subresources.pid and "
"moz_pages.uri = ?", (data["testUrl"],))
known_resources = [tp[0] for tp in cursor]
requests_predicted = [r for r in prediction if r in all_requests]
known_predicted = [r for r in known_resources if r in prediction]
known_requested = [r for r in all_requests if r in known_resources]
correct_predictions_ratio = 100.0 * len(requests_predicted) / len(prediction)
predicted_requests_ratio = 100.0 * len(requests_predicted) / len(all_requests)
known_requests_ratio = 100.0 * len(known_requested) / len(all_requests)
print "At least %.2f %% predictions were correct" % correct_predictions_ratio
print "At least %.2f %% requests were predicted" % predicted_requests_ratio
print "At least %.2f %% requests were known" % known_requests_ratio
if known_resources:
known_requests_predicted_ratio = 100 * len(known_predicted) / float(len(known_resources))
print "Out of those, %.2f %% were predicted" % known_requests_predicted_ratio
print
except Exception as x:
print "Failed to load kmeans statistics: " + str(x)
speedindex_table = make_table_for_metric("SpeedIndex", resultsd)
print "SpeedIndex"
padnums.pprint_table(sys.stdout, speedindex_table)
def print_table():
# Update the displayed speed from parameter server
table[table_ids["speed"]] = getParam("joint_speed", "speed")
print "\x1B[2J"
print display_counter
pp.pprint_table(out, table)
def print_table():
# Update the displayed speed from parameter server
table[table_ids["speed"]] = getParam('joint_speed', 'speed')
print "\x1B[2J"
print display_counter
pp.pprint_table(out, table)
def printDeals(deals):
table = map(lambda x: x.desc(), deals)
for entry in table:
entry[4] = "%d (%.1f)" % (entry[4], entry[5])
entry[7] = "%s (%.1f)" % (entry[7], entry[8])
entry[9] = "%s (%.1f)" % (entry[9], entry[10])
del entry[5]
del entry[7]
del entry[8]
table.insert(0, ("TYPE", "AMOUNT", "VOLUME", "BUY PRICE", "PROFIT", "SELL PRICE", "BUY", "SELL", "JUMPS"))
padnums.pprint_table(sys.stdout, table)
def printAssignments(assignments):
""" Calculates relevant deltas and prints them out to the console """
table = [[
"workerId", "condition", "assignmentId", "accept-show", "show-go",
"go-answer"
]]
for click in assignments:
table.append([
click.workerid, click.condition, click.assignmentid,
str(click.showDeltaAccept()),
str(click.goDeltaShow()),
str(click.answerDeltaGo())
])
pprint_table(sys.stdout, table)
def printVideoTimes(db, study_videos_only):
sql = """SELECT pictures.videoid, MIN(start), MAX(end), MAX(end) - MIN(start) AS elapsed FROM pictures, phase_lists, phases"""
if study_videos_only:
sql += """, study_videos """
sql += """WHERE pictures.phase_list = phase_lists.pk
AND phase_lists.pk = phases.phase_list
AND phase_lists.is_historical = FALSE"""
if study_videos_only:
sql += """ AND study_videos.videoid = pictures.videoid"""
sql += """ GROUP BY phase_lists.pk"""
result = db.query_and_return_array(sql)
table = [["video", "elapsed"]]
for picture in result:
table.append([str(picture['videoid']), str(picture['elapsed'])])
print("Median: %s" % numpy.median([float(row[1]) for row in table[1:]]))
pprint_table(sys.stdout, table)
def __main__():
processEnrolmentLink = None
while not processEnrolmentLink:
(sessionId, studentNum) = startNewSession()
timetableUrl = getTimetableLink(studentNum, sessionId, getCs1(sessionId, studentNum))
# Print session information in a neat table
print "New session started."
table = [["Username", username],
["Student Number", studentNum],
["Session ID", sessionId],
["timetableUrl", timetableUrl]]
out = sys.stdout
pprint_table(out, table)
print ""
processEnrolmentLink = None
try:
while not processEnrolmentLink:
print "Attempting to find enrolment link..."
processEnrolmentLink = getProcessEnrolmentLink(timetableUrl, tut_string)
except SessionExpiredError, e:
print "Session expired."
continue
def printres(res, filepath='best.txt',all=True,sorting=2):
res.sort()
s = ""
best = []
good = []
ok = []
bad = []
for b in res:
fil = b.get('file')
pLeng = str(b.get('Pleng'))
caps = str(b.get('caps'))
if b.get('rank')==1:
best.append([fil,pLeng,caps])
elif b.get('rank')==2:
good.append([fil,str(b.get('Cleng')),caps,pLeng])
else:
s = [fil,str(b.get('Uleng')),caps,pLeng,str(b.get('Cleng'))]
if b.get('rank')==3:
ok.append(s)
else:
s = s+[str(b.get('tleng'))]
bad.append(s)
# sort 'best' by number of capitals after '.'
best = sorted(best, key=lambda x: float(x[sorting]),reverse=True)
if all:
# sort the others by sentence length
good = sorted(good, key=lambda x: float(x[1]),reverse=True)
ok = sorted(ok , key=lambda x: float(x[1]),reverse=True)
bad = sorted(bad , key=lambda x: float(x[1]),reverse=True)
goods = [["File","Sentence length for ','","Capital after ','"
,"Sentence length for '.'"]]+good
oks = [["File","Sentence length for capitals","Capital after '.'"
,"Sentence length for '.'","Sentence length for ','"]]+ok
bads = [["File","Sentence length for capitals","Capital after '.'"
,"Sentence length for '.'","Sentence length for ','","Sentence length for tab"]]+bad
pprint_table(open('good.txt' ,'w'),goods)
pprint_table(open('ok.txt' ,'w'),oks)
pprint_table(open('bad.txt' ,'w'),bads)
bests = [["File","Sentence length for '.'","Capital after punctuation"]]+best
pprint_table(open(filepath ,'w'),bests)
def printSummary(assignments,
assignments_including_incomplete,
condition=None):
# TODO?: WARNING: not removing first worker attempt to smooth
print("N = %d, %d unique workers" %
(len(assignments),
len(set([assignment.workerid for assignment in assignments]))))
if len(assignments) == 0:
return
table = [["metric", "10%", "25%", "50%", "75%", "90%", "mean", "std. dev"]]
accept_show = [
click.showDeltaAccept() for click in assignments
if click.showDeltaAccept() is not None
]
table.append([
"accept-show",
str(stats.scoreatpercentile(accept_show, 10)),
str(stats.scoreatpercentile(accept_show, 25)),
str(stats.scoreatpercentile(accept_show, 50)),
str(stats.scoreatpercentile(accept_show, 75)),
str(stats.scoreatpercentile(accept_show, 90)),
str(numpy.mean(accept_show)),
str(numpy.std(accept_show))
])
go_show = [
click.goDeltaShow() for click in assignments_including_incomplete
]
for i in range(len(go_show)):
if go_show[i] is None:
go_show[i] = sys.maxint
table.append([
"show-go",
str(stats.scoreatpercentile(go_show, 10)),
str(stats.scoreatpercentile(go_show, 25)),
str(stats.scoreatpercentile(go_show, 50)),
str(stats.scoreatpercentile(go_show, 75)),
str(stats.scoreatpercentile(go_show, 90)),
str(numpy.mean(go_show)),
str(numpy.std(go_show))
])
go_answer = [
click.answerDeltaGo() for click in assignments
if click.answerDeltaGo() is not None
]
table.append([
"go-answer",
str(stats.scoreatpercentile(go_answer, 10)),
str(stats.scoreatpercentile(go_answer, 25)),
str(stats.scoreatpercentile(go_answer, 50)),
str(stats.scoreatpercentile(go_answer, 75)),
str(stats.scoreatpercentile(go_answer, 90)),
str(numpy.mean(go_answer)),
str(numpy.std(go_answer))
])
pprint_table(sys.stdout, table)
print("UNDER TWO: %s" % stats.percentileofscore(go_show, 2.0))
print("UNDER THREE: %s" % stats.percentileofscore(go_show, 3.0))
# Correlation between wait-show and show-go
#(r, p_val) = stats.pearsonr(accept_show, go_show)
#print("Correlation between accept-show and show-go: %f, p<%f" % (r, p_val))
#(r_answer, p_val_answer) = stats.pearsonr(go_show, go_answer)
#print("Correlation between show-go and go-answer: %f, p<%f" % (r_answer, p_val_answer))
# bounce rate
mortality = 1 - float(
len(assignments)) / len(assignments_including_incomplete)
print("Mortality Rate: " + str(mortality))
# preview to accept ratio: we can look for a given HIT id
# (which will only have one assignment) how many unique IPs previewed it
# it's possible that lots of people previewed and wanted, but only one
# was fast enough to grab it?
if condition is not None:
db = DBConnection()
if condition == 'tetris':
experiments_string = ','.join([str(ex) for ex in EXPERIMENTS])
result = db.query_and_return_array(
""" SELECT COUNT(DISTINCT assignmentid) FROM logging WHERE event = 'tetris_row_clear' AND experiment IN (%s) """,
(experiments_string, ))[0]
num_playing = result['COUNT(DISTINCT assignmentid)']
print(
str(num_playing) + " assignments out of " +
str(len(assignments)) + " (" +
str(float(num_playing) / len(assignments) * 100) +
"%) cleared a row in Tetris ")
table.append(("Nonlinear least squares (unweighted)",
model.params[0], model.params[1], model.residual,
get_error(model.params)))
model = NonLinearLeastSquaresModel(weighted = True)
model.fit(data)
table.append(("Nonlinear least squares (weighted)",
model.params[0], model.params[1], model.residual,
get_error(model.params)))
model = NonLinearLeastSquaresModel()
(latitude, longitude), inliers, residual = ransac.ransac(
data, model, iterations = 50, min_samples = 3,
min_inliers = 0.6, eps = 1e-3)
table.append(("RANSAC (unweighted, %d/%d points)" % (len(inliers), len(data)),
latitude, longitude, residual, get_error(model.params)))
model = NonLinearLeastSquaresModel(weighted = True)
(latitude, longitude), inliers, residual = ransac.ransac(
data, model, iterations = 50, min_samples = 3,
min_inliers = 0.6, eps = 1e-3)
table.append(("RANSAC (weighted, %d/%d points)" % (len(inliers), len(data)),
latitude, longitude, residual, get_error(model.params)))
padnums.pprint_table(sys.stdout, table)
def getDealsAroundSS(self, SS, radius):
solarSystems = self.getSSAroundSS(SS, radius)
print "Getting Sell Orders"
sqlSellOrders = (
"SELECT `orderID`,`typeID`,`price`,`volRemaining`, `solarSystemID`\
FROM `emdrOrders`\
WHERE `solarSystemID` IN (%s) AND `BID` = 0 AND `generatedAt` > ADDDATE(NOW(), INTERVAL -9 HOUR)\
ORDER BY `typeID`,`price`"
% arrayList(solarSystems)
)
sellOrders = list(self.runQueryR(sqlSellOrders))
sellOrders = map(lambda x: list(x), sellOrders)
print "Done in %s. %d" % (self.qdt, len(sellOrders))
print "Getting Buy Orders"
sqlBuyOrders = (
" SELECT `orderID`,`typeID`,`price`,`volRemaining`, `solarSystemID`\
FROM `emdrOrders`\
WHERE `solarSystemID` IN (%s) AND `BID` = 1 AND `generatedAt` > ADDDATE(NOW(), INTERVAL -9 HOUR)\
ORDER BY `typeID`,`price` DESC"
% arrayList(solarSystems)
)
buyOrders = list(self.runQueryR(sqlBuyOrders))
buyOrders = map(lambda x: list(x), buyOrders)
print "Done in %s. %d" % (self.qdt, len(buyOrders))
print "Algorithm begin."
deals = []
log = []
header = ["OrderID", "TypeID", "Price", "Amount", "", "OrderID", "TypeID", "Price", "Amount", "RESULT"]
log.append(header)
while len(sellOrders) and len(buyOrders):
so = sellOrders[0]
bo = buyOrders[0]
if len(sellOrders) % 30 == 0:
log.append(["", "", "", "", "", "", "", "", "", ""])
log.append(header)
logE = [so[0], so[1], so[2], so[3], "", bo[0], bo[1], bo[2], bo[3]]
if so[1] < bo[1]:
del sellOrders[0]
res = "sItem < bItem"
logE.append(res)
log.append(logE)
continue
if so[1] > bo[1]:
del buyOrders[0]
res = "sItem > bItem"
logE.append(res)
log.append(logE)
continue
profit = bo[2] - so[2]
if profit < so[2] * 0.05:
del sellOrders[0]
del buyOrders[0]
res = "unprofitable"
logE.append(res)
log.append(logE)
continue
res = "OK!"
logE.append(res)
log.append(logE)
amount = min(so[3], bo[3])
tmp = so[3] - bo[3]
if tmp < 0:
del sellOrders[0]
bo[3] -= amount
elif tmp > 0:
del buyOrders[0]
so[3] -= amount
else:
del sellOrders[0]
del buyOrders[0]
# typeID, sellOrderID, buyOrderID, iSellSSID, iBuySSID, profit, amount
myDeal = Deal([so[1], so[0], bo[0], bo[4], so[4], profit, amount], self)
deals.append(myDeal)
padnums.pprint_table(sys.stdout, log)
print "Here we go, %d deals!" % len(deals)
Deal.printDeals(deals)
