-
Notifications
You must be signed in to change notification settings - Fork 0
/
assignments.py
150 lines (131 loc) · 5.73 KB
/
assignments.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
from pyquery import PyQuery as pq
import requests
import os
from session import session as s
from utils import save, stripQuery, jsondump, mkdir, genHtml
def getAssignment(url):
data = s.get(url)
doc = pq(data.content)
tables = doc(".ruContentPage>center>form>table")
host = stripQuery(url)
ret = {}
ret["description"] = getAssignmentDescription(tables.eq(0), host)
if len(tables) >= 3:
ret["grade"] = getAssignmentGrade(tables.eq(1), host)
ret["handin"] = getAssignmentHandin(tables.eq(2), host)
ret["statistics"] = getAssignmentStatistics(doc(".ruContentPage>table"), host)
return ret
def getAssignmentHandin(table, host):
rows = list(table.children("tr").items())
ret = {}
ret["comment"] = rows[-2].children("td").eq(2).html().replace("<br/>","\n").strip()
ret["files"] = []
files = rows[-1].children("td").eq(2)
if len(files.children()) == 1 and len(files.find("table")) == 1:
ret["files"] = readFileTable(files.find("table"), host)
return ret
def getAssignmentGrade(table, host):
items = ["grade", "ranking", "comments"]
grade = {}
for id, i in zip(items,[1,2,3]):
grade[id] = table.children("tr").eq(i).children("td").eq(2).html().replace("<br/>","\n").strip()
grade["files"] = []
files = table.children("tr").eq(4).children("td").eq(2)
if len(files.children()) == 1 and len(files.find("table")) == 1:
grade["files"] = readFileTable(files.find("table"), host)
return grade
def getAssignmentDescription(desctable, host):
# The assignment description
desc = {}
items = ["due", "description", "percent", "filetype", "filenumber"]
for id, i in zip(items,[1,2,4,5,6]):
desc[id] = desctable.children("tr").eq(i).children("td").eq(2).html().strip()
desc["title"] = desctable.find("tr").eq(0).text()
# Files
desc["files"] = []
files = desctable.children("tr").eq(3).children("td").eq(2)
if len(files.children()) == 1 and len(files.find("table")) == 1:
desc["files"] = readFileTable(files.find("table"), host)
return desc
def readFileTable(table, host):
files = []
for item in list(table.find("tr").items())[1:]:
files.append({
"filename": item.find("td").eq(1).text(),
"url": host + item.find("td").eq(1).find("a").attr("href"),
"size": item.find("td").eq(2).text(),
"date": item.find("td").eq(3).text()
})
return files
def getAssignmentStatistics(table, host):
ret = {}
try:
ret["image"] = host + table.children("tr").eq(0).find("img").attr("src")
except:
ret["image"] = None
stats = table.children("tr").eq(1).find("table tr").not_(".ruTableTitle").children("td").items()
stats = list(stats)
print(stats)
if not stats:
return ret
ret["average"] = stats[1].text()
ret["middle"] = stats[4].text()
ret["standarddev"] = stats[6].text()
ret["mostcommon"] = stats[9].text()
ret["handins"] = stats[11].text()
return ret
def getAssignments(url):
host = stripQuery(url)
data = s.get(url)
doc = pq(data.content)
tableQ = ".ruContentPage > center > .ruTable"
titles = [el.text() for el in doc(tableQ + " > .ruTableTitle").items() if el.text() != ""]
tables = doc(tableQ + " .ruTable").items()
assignments = []
for table in tables:
title = [el.text() for el in table.find("tr.ruTableTitle th").items()]
assign = []
for el in table.find("tr").not_(".ruTableTitle").items():
if el.text().strip() == "": continue
d = []
for item in el.find("td").items():
if len(item.children()) == 0:
d.append(item.text())
else:
# Assert that this is a link if it has children
assert len(item.children()) == 1 and len(item.find("a")) == 1
d.append((item.find("a").text(), host + item.find("a").attr("href")))
assign.append(dict(zip(title, d)))
assignments.append(assign)
return dict(zip(titles,assignments))
def downloadAssignments(url, handin=False):
assignments = getAssignments(url)
os.mkdir("assignments")
for k, v in assignments.items():
dirpath = mkdir("assignments", k)
for item in v:
print("Processing %s/%s" % (k, item["Nafn"][0]))
path = mkdir(dirpath, item["Nafn"][0])
assignment = getAssignment(item["Nafn"][1])
# Description
descr = os.path.join(path, "description")
os.mkdir(descr)
jsondump(os.path.join(descr,"description.json"), assignment["description"])
genHtml(os.path.join(descr, "description.html"), assignment["description"]["description"], assignment["description"]["title"])
for item in assignment["description"]["files"]:
save(descr, item["url"])
if handin:
if "grade" in assignment:
grade = os.path.join(path, "grade")
os.mkdir(grade)
jsondump(os.path.join(grade, "grade.json"), assignment["grade"])
for item in assignment["grade"]["files"]:
save(grade, item["url"])
if "handin" in assignment:
handin = os.path.join(path, "handin")
os.mkdir(handin)
jsondump(os.path.join(handin, "handin.json"), assignment["handin"])
if "statistics" in assignment:
jsondump(os.path.join(path, "stats.json"), assignment["statistics"])
if assignment["statistics"]["image"] is not None:
save(os.path.join(path, "stats.jpg"), assignment["statistics"]["image"])