def tsne(data, featureDict, x, prop):
"""t-SNE methods
Args:
data (TYPE): Description
x (TYPE): Description
queryrate (TYPE): Description
plotsize (TYPE): Description
Returns:
TYPE: Description
"""
x = 'tsne-%s' % x
print "Computing t-SNE embedding"
t0 = time.clock()
X = np.array(data['data'])
model = TSNE(n_components=2, random_state=0)
np.set_printoptions(suppress=True)
# list, each of them is formatted as (x,y)
Y = model.fit_transform(X)
print "t-SNE embedding of the digits (time %.2fs)" % (time.clock() - t0)
func.matrixtofile(
Y, '%s/PointsPosition-1-in-%s-%s.csv' %
(prop['dic'], str(prop['queryrate']), str(x)))
result = {'data': Y, 'id': data['id']}
prop['text'] = x
drawFigure(result, featureDict, prop)
def pca(data, featureDict, x, prop):
"""PCA methods
Args:
data (TYPE): Description
x (TYPE): Description
queryrate (TYPE): Description
plotsize (TYPE): Description
Returns:
TYPE: Description
"""
x = 'PCA-%s' % x
print("Computing PCA projection")
t0 = time.clock()
pca = PCA(n_components=2)
X_pca = pca.fit_transform(data['data'])
print "Principal Components projection of the digits (time %.2fs)" % (
time.clock() - t0)
func.matrixtofile(
X_pca, '%s/PointsPosition-1-in-%s-%s.csv' %
(prop['dic'], str(prop['queryrate']), str(x)))
result = {'data': X_pca, 'id': data['id']}
prop['text'] = x
drawFigure(result, featureDict, prop)
def query(self, query):
self.connect()
self.cur = self.dbh.cursor()
CommonFunc.debug("Executing query: %s" % (query))
self.res = self.cur.execute(query)
#CommonFunc.debug( "Returned rows: %d" % (int(self.numrows())) )
return self.res
def decompose(data, featureDict, rowstring, deapproaches, prop):
collists = {
'total': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
'poi1': [0, 1, 2, 4, 5, 6, 7],
'poi2': [0, 1, 5, 6, 7]
}
rowlists = {
'total': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
'workday': [0, 1, 2, 3, 4, 5],
'weekend': [6, 7, 8, 9, 10, 11],
'daytime': [1, 2, 3, 7, 8, 9],
'evening': [4, 10],
'wodaytime': [1, 2, 3],
'woevening': [4],
'wedaytime': [7, 8, 9],
'weevening': [10]
}
decomposelists = ['t-SNE', 'PCA', 'MDS']
filterqdata = func.matrixtoarray(data, rowlists[rowstring],
collists['total'])
if '1' in deapproaches:
tsne(filterqdata, featureDict, rowstring, prop)
if '2' in deapproaches:
pca(filterqdata, featureDict, rowstring, prop)
if '3' in deapproaches:
mds(filterqdata, featureDict, rowstring, prop)
def mds(data, featureDict, x, prop):
x = 'MDS-%s' % x
print "Computing MDS embedding"
t0 = time.clock()
clf = MDS(n_components=2, n_init=1, max_iter=100)
X_mds = clf.fit_transform(data['data'])
print("Done. Stress: %f" % clf.stress_)
print "MDS embedding of the digits (time %.2fs)" % (time.clock() - t0)
func.matrixtofile(
X_mds, '%s/PointsPosition-1-in-%s-%s.csv' %
(prop['dic'], str(prop['queryrate']), str(x)))
result = {'data': X_mds, 'id': data['id']}
prop['text'] = x
drawFigure(result, featureDict, prop)
def connect(self):
#CommonFunc.debug("Creating (or reusing) DB connection")
#CommonFunc.debug("--- DB ---")
CommonFunc.debug("Connectiong to DB")
if self.dbh == None:
CommonFunc.debug("Creating DB hablder DB handler")
self.dbh = MySQLdb.connect(host=self.db_host, user=self.db_user, passwd=self.db_pass, db=self.db_name)
CommonFunc.debug("Using DB hablder: %s" % (self.dbh))
else:
CommonFunc.debug("Reusing DB handler: %s" % (self.dbh))
def DB_get_user_data(self):
import appmysqldb, CommonFunc
user_id = "None"
course_id = "None"
user_name = "None"
user_email = "None"
db = appmysqldb.mysql('localhost', 3306, 'edxapp', 'root', '')
q = "SELECT id, user_id, course_id FROM student_anonymoususerid WHERE anonymous_user_id='" + self.n_user_id + "'"
CommonFunc.debug("QUERY: %s" %(q))
db.query(q)
res = db.fetchall()
for row in res:
user_id = row[1]
course_id = row[2]
q = "SELECT name FROM auth_userprofile WHERE user_id='%s' " % (user_id)
CommonFunc.debug("QUERY: %s" %(q))
db.query(q)
res = db.fetchall()
for row in res:
user_name = row[0]
q = "SELECT username FROM auth_user WHERE id='%s' " % (user_id)
CommonFunc.debug("QUERY: %s" %(q))
db.query(q)
res = db.fetchall()
for row in res:
user_email = row[0]
results = [user_id,course_id,user_name,user_email]
return results
def DB_get_user_data(self):
import appmysqldb, CommonFunc
user_id = "None"
course_id = "None"
user_name = "None"
user_email = "None"
db = appmysqldb.mysql('localhost', 3306, 'edxapp', 'root', '')
q = "SELECT id, user_id, course_id FROM student_anonymoususerid WHERE anonymous_user_id='" + self.n_user_id + "'"
CommonFunc.debug("QUERY: %s" %(q))
db.query(q)
res = db.fetchall()
for row in res:
user_id = row[1]
course_id = row[2]
q = "SELECT name FROM auth_userprofile WHERE user_id='%s' " % (user_id)
CommonFunc.debug("QUERY: %s" %(q))
db.query(q)
res = db.fetchall()
for row in res:
user_name = row[0]
q = "SELECT username FROM auth_user WHERE id='%s' " % (user_id)
CommonFunc.debug("QUERY: %s" %(q))
db.query(q)
res = db.fetchall()
for row in res:
user_email = row[0]
results = [user_id,course_id,user_name,user_email]
return results
def get_resp(self):
resp_tmp = b''
while not self.end_thread:
resp = self.tcp_get_resp()
if resp:
resp_tmp += resp
if resp_tmp:
if resp_tmp[4] < 0xfc:
daq = resp_tmp[5]
if len(resp_tmp) < self.daq_pack_len_list[resp_tmp[5]]:
pass
else:
daq_tmp = resp_tmp[0:self.daq_pack_len_list[resp_tmp[5]]]
if len(resp_tmp) > self.daq_pack_len_list[resp_tmp[5]]:
resp_tmp = resp_tmp[self.daq_pack_len_list[resp_tmp[5]]:]
else:
resp_tmp = b''
i = 0
daq_data = b''
while i < len(daq_tmp):
if len(daq_data) == 0:
start_ptr = 10
else:
start_ptr = 6
daq_data += daq_tmp[i + start_ptr:daq_tmp[i] + i + 4]
i += daq_tmp[i] + 4
if len(daq_data) == self.daq_pack_len[daq]:
if daq > 0:
pointer = 0
for j in range(daq):
pointer += self.daq_pack_len[j]
if daq < len(self.daq_pack_len) - 1:
pointer_end = pointer + self.daq_pack_len[daq]
self.daq = self.daq[0:pointer] + daq_data + self.daq[pointer_end:]
else:
self.daq = self.daq[0:pointer] + daq_data
else:
if len(self.daq_pack_len) >= 2:
pointer_end = self.daq_pack_len[daq]
self.daq = daq_data + self.daq[pointer_end:]
else:
self.daq = daq_data
self.coll_rl_prob, self.coll_rr_prob = \
CommonFunc.analyze_daq(self.coll_rl_prob, self.coll_rr_prob, self.daq,
self.redis_dict, self.redis_dict_raw)
elif len(daq_data) > self.daq_pack_len[daq]:
print('expected: ' + str(self.daq_pack_len[daq]))
print('actual: ' + str(len(daq_tmp)))
daq_data = b''
else:
single_resp = resp_tmp[0:resp_tmp[0]+4]
if len(resp_tmp) > resp_tmp[0] + 4:
resp_tmp = resp_tmp[resp_tmp[0]+4:]
else:
resp_tmp = b''
if single_resp[4] == 0xff:
self.resp_recvd = True
self.pos_resp = True
elif single_resp[4] == 0xfc:
pass
elif single_resp[4] == 0xfe:
self.resp_recvd = True
self.end_thread = False
def DB_get_user_data(self):
import appmysqldb, CommonFunc
user_id = "None"
course_id = "None"
user_name = "None"
user_email = "None"
user_score = "0"
#ids : user and course
db = appmysqldb.mysql('localhost', 3306, 'edxapp', 'root', '')
q = "SELECT id, user_id, course_id FROM student_anonymoususerid WHERE anonymous_user_id='" + self.n_user_id + "'"
CommonFunc.debug("QUERY: %s" %(q))
db.query(q)
res = db.fetchall()
for row in res:
user_id = row[1]
course_id = row[2]
#username
q = "SELECT name FROM auth_userprofile WHERE user_id='%s' " % (user_id)
CommonFunc.debug("QUERY: %s" %(q))
db.query(q)
res = db.fetchall()
for row in res:
user_name = row[0]
#email
q = "SELECT email FROM auth_user WHERE id='%s' " % (user_id)
CommonFunc.debug("QUERY: %s" %(q))
db.query(q)
res = db.fetchall()
for row in res:
user_email = row[0]
""" getting course data from mongodb """
# Mongo DB Connect
from pymongo import Connection
xmoduledb = "edxapp"
connection = Connection()
db_mongo = connection[xmoduledb]
mongo_modulestore = db_mongo['modulestore']
badge_list_problems = edxappCourseData.getListProblemsFromBadgeId(mongo_modulestore,self.bg_id,course_id, self.xblock_name_field)
badge_problems_score = edxappCourseData.getScoreFromBadgeId(mongo_modulestore,self.bg_id,course_id, self.xblock_name_field)
""" """
#calculate badge_score
user_score = 0
partial_user_score = []
badge_partial_user_score = 0
badge_percent_user_score = 0
#calculate user partials
if badge_problems_score>0:
if len(badge_list_problems)>0:
for problem in badge_list_problems:
if 'problem_score' in problem:
problem_score = problem['problem_score']
problem_id = problem['problem_id']
#getting partial values
if int(problem_score)>0:
q = "SELECT ((%s/max_grade)*grade) FROM courseware_studentmodule WHERE course_id='%s' AND student_id='%s' AND module_id='%s'" % (problem_score,course_id,user_id,problem_id)
CommonFunc.debug("QUERY: %s" %(q))
db.query(q)
res = db.fetchall()
for row in res:
if row[0]>0:
partial_user_score.append( float(row[0]) )
badge_partial_user_score = sum(partial_user_score)
#calculate total percent
if round(badge_partial_user_score,2)>0 and int(badge_problems_score)>0:
badge_percent_user_score = ( badge_partial_user_score * 100.0 ) / badge_problems_score
badge_percent_user_score = round(badge_percent_user_score,2)
if int(badge_percent_user_score)>0:
user_score = badge_percent_user_score
#show results
results = [user_id,course_id,user_name,user_email,user_score,badge_list_problems,badge_problems_score,badge_partial_user_score,badge_percent_user_score,badge_problems_score]
return results
def queryUserMatrix(dbname,
collectname,
queryrate,
recordsthre=0,
entropytype='col',
entropymin=0,
entropymax=100):
"""Query user matrix from database and return it as a matrix
Args:
dbname (string): Database name
collectname (string): Collection name
queryrate (int): Query rate, for example: 5 means 1/5 in whole data
Returns:
TYPE: Description
"""
conn, db = func.connectMongo(dbname)
entropyprop = 'entropy.%s' % entropytype
if recordsthre != 0:
# bug
qresult = list(db[collectname].find(
{
'$and': [{
'_id': {
'$mod': [queryrate, 0]
}
}, {
'totalNum': {
'$gt': recordsthre
}
}, {
entropyprop: {
'$gte': float(entropymin),
'$lte': float(entropymax)
}
}]
}, {
'pVec': 1,
'gt11sim': 1,
'whlsim': 1,
'totalNum': 1
}).sort([("_id", 1)]))
else:
qresult = list(db[collectname].find(
{
'$and': [{
'_id': {
'$mod': [queryrate, 0]
}
}, {
entropyprop: {
'$gte': float(entropymin),
'$lte': float(entropymax)
}
}]
}, {
'pVec': 1,
'gt11sim': 1,
'whlsim': 1,
'totalNum': 1
}).sort([("_id", 1)]))
print "User query result: %s people." % str(len(qresult))
data = {'id': [], 'data': []}
featureDict = {}
for x in qresult:
data['data'].append(x['pVec'])
data['id'].append(str(x['_id']))
featureDict[str(x['_id'])] = {
'_id': int(x['_id']),
'gt11sim': float(x['gt11sim']),
'whlsim': float(x['whlsim']),
'totalNum': int(x['totalNum'])
}
conn.close()
return data, featureDict
def drawFigure(data, featureDict, prop):
"""Figures' drawing and saving
Args:
data (list): Dots list
plotsize (TYPE): Description
queryrate (TYPE): Description
Returns:
TYPE: Description
"""
# Init transformation
plotsize, queryrate, fthre = prop['plotsize'], str(prop['queryrate']), str(
prop['text'])
# matrixData 用于存储降维后点值的二维位置与对应 ID 信息
matrixDataRaw = np.asarray(np.array(data['data'])[:, 0:2])
matrixDataRes = []
# 相似度对应的 colormap 取值表
simcolormap = {
'4': '#1924B1',
'5': '#37B6CE',
'6': '#25D500',
'7': '#FFC700',
'8': '#FF8E00',
'9': '#FF1300'
}
# Scatter plot
X, Y = data['data'][:, 0], data['data'][:, 1]
whlC, gt11C, numC = [], [], []
for x in xrange(0, len(data['id'])):
whlSimVal = int(
math.floor(featureDict[str(data['id'][x])]['whlsim'] * 10))
gt11SimVal = int(
math.floor(featureDict[str(data['id'][x])]['gt11sim'] * 10))
whlC.append(simcolormap[str(whlSimVal)])
gt11C.append(simcolormap[str(gt11SimVal)])
totalNum = featureDict[str(data['id'][x])]['totalNum']
numC.append(func.calColorbyNum(int(totalNum)))
matrixDataRes.append([
data['id'][x], matrixDataRaw[x][0], matrixDataRaw[x][1], totalNum,
whlSimVal, gt11SimVal
])
textRecNum = '2D-ScatterData_1-in-%s_%s(byRecNum)' % (queryrate, fthre)
textAveSim = '2D-ScatterData_1-in-%s_%s(byAveSim)' % (queryrate, fthre)
func.matrixtofile(
matrixDataRes,
'%s/2D-ScatterData_1-in-%s_%s.csv' % (prop['dic'], queryrate, fthre))
scatterTC = TimeConsuming(textRecNum)
plt.figure()
plt.title(textRecNum)
plt.scatter(X, Y, s=plotsize, c=numC, lw=0)
classes = ['5', '10', '200', '800', '2000', '4000', '8500']
class_colours = [
'#000000', '#1924B1', '#37B6CE', '#25D500', '#FFC700', '#FF8E00',
'#FF1300'
]
recs = []
for i in range(0, len(class_colours)):
recs.append(mpatches.Rectangle((0, 0), 1, 1, fc=class_colours[i]))
plt.legend(recs,
classes,
scatterpoints=1,
loc='lower left',
ncol=4,
fontsize=5)
img = plt.gcf()
img.savefig('%s/%s.png' % (prop['imgdic'], textRecNum), dpi=400)
plt.close()
scatterTC.end()
scatterTC2 = TimeConsuming(textAveSim)
plt.figure()
plt.title(textAveSim)
plt.scatter(X, Y, s=plotsize, c=whlC, lw=0)
whlClasses = [float(x) / 10 for x in simcolormap]
whlClassColours = [simcolormap[x] for x in simcolormap]
whlRecs = []
for i in range(0, len(whlClassColours)):
whlRecs.append(mpatches.Rectangle((0, 0), 1, 1, fc=whlClassColours[i]))
plt.legend(whlRecs,
whlClasses,
scatterpoints=1,
loc='lower left',
ncol=3,
fontsize=5)
img2 = plt.gcf()
img2.savefig('%s/%s.png' % (prop['imgdic'], textAveSim), dpi=400)
plt.close()
scatterTC2.end()
def disconnect(self):
CommonFunc.debug("Closing DB connection")
self.dbh.close()
CommonFunc.debug("Disconnected")
def showConfig(self): CommonFunc.debug( "db_host : %s" % (self.db_host) ) CommonFunc.debug( "db_name : %s" % (self.db_name) ) CommonFunc.debug( "db_user : %s" % (self.db_user) ) CommonFunc.debug( "db_pass : %s" % (self.db_pass) )
def fetchall(self):
CommonFunc.debug("Fetching all results")
return self.cur.fetchall()