def boredRecommend(user_id, num = DEF_REC_NUM):
recommendations = gae_cache.cache_get(user_id + 'boredrmd')
if recommendations:
return recommendations
expertise = ExUserInfo.gql('WHERE user_id = :1', user_id).get().expertise
diff_expertise = set(data_models.EXPERTISES.keys())
diff_expertise.difference_update(set(expertise))
if not diff_expertise:
return None
query1 = URLStorage.gql('WHERE topic IN :1', list(diff_expertise))
# url_recs = [item for item in query1]
# logging.info('number of bored rmds: ' + str(len(url_recs)))
# li = [random.randint(0, len(url_recs)-1) for i in range(2*num)]
query2 = Rating.gql('WHERE user_id = :1', user_id)
rated = [item.url for item in query2]
recommendations = []
for url_rec in query1:
if url_rec.url not in rated:
recommendations.append(url_rec)
if len(recommendations) >= num:
break
gae_cache.cache_put(user_id+'boredrmd', recommendations)
return recommendations
def boredRecommend(user_id, num=DEF_REC_NUM):
recommendations = gae_cache.cache_get(user_id + 'boredrmd')
if recommendations:
return recommendations
expertise = ExUserInfo.gql('WHERE user_id = :1', user_id).get().expertise
diff_expertise = set(data_models.EXPERTISES.keys())
diff_expertise.difference_update(set(expertise))
if not diff_expertise:
return None
query1 = URLStorage.gql('WHERE topic IN :1', list(diff_expertise))
# url_recs = [item for item in query1]
# logging.info('number of bored rmds: ' + str(len(url_recs)))
# li = [random.randint(0, len(url_recs)-1) for i in range(2*num)]
query2 = Rating.gql('WHERE user_id = :1', user_id)
rated = [item.url for item in query2]
recommendations = []
for url_rec in query1:
if url_rec.url not in rated:
recommendations.append(url_rec)
if len(recommendations) >= num:
break
gae_cache.cache_put(user_id + 'boredrmd', recommendations)
return recommendations
def worth_points(url, user, update_cache = False):
points = gae_cache.cache_get(user.user_id+url+'points')
if points and not update_cache:
return points
points = None
m_rating = Rating.gql('WHERE user_id = :1 AND url = :2', user.user_id, url).get()
#if less than one hour between two attempts, no points are awarded
current_time = datetime.datetime.now()
if m_rating and (current_time - m_rating.time).total_seconds() < 3600:
points = 0
if points is None:
points = BASIC_RATING_POINTS
#punish multiple attempts to a specific url
punish_coefficient = 1
if m_rating:
punish_coefficient = math.pow(0.5, float(m_rating.attempts))
sparse_points = 0
diff_coefficient = 1
url_stat = URLStorage.gql('WHERE url = :1', url).get()
if url_stat:
for i in range(6):
if url_stat.total < SPARSITY_LEVEL[i]:
sparse_points = SPARSITY_POINTS[i]
break
r = url_stat.ratings
diff_coefficient = 1.0 / (((abs(r[4] + r[3] - r[1] - r[0])) / (r[4] + r[3] + r[1] + r[0] + 10.0)) + 1)
points = int(math.ceil(sparse_points * diff_coefficient * punish_coefficient))
gae_cache.cache_put(user.user_id+url+'points', points)
return points
def get(self):
url = self.request.get('url')
status = []
avg = 0
total = 0
m_stat = URLStorage.gql('WHERE url = :1', url).get()
if m_stat:
status = m_stat.ratings
total = m_stat.total
avg = 0
if total != 0:
avg = sum([(i + 1) * status[i]
for i in range(5)]) * 1.0 / total
status.reverse()
(truthfulness, unbiased, security, design) = self.get_details(url)
self.render('status_table.html',
status=status,
url=url,
avg=avg,
truthfulness=truthfulness,
unbiased=unbiased,
security=security,
design=design,
total=total)
def randomAlg(num = DEF_REC_NUM):
# m_stat = stats.KindStat.gql('WHERE kind_name = :1', 'URLStorage').get()
m_stat = data_models.get_stat()
li = [random.randint(1, m_stat.url_count) for i in range(num)]
# li = [1]
query = URLStorage.gql('WHERE index IN :1', li)
recommendations = [item for item in query]
return recommendations
def randomAlg(num=DEF_REC_NUM):
# m_stat = stats.KindStat.gql('WHERE kind_name = :1', 'URLStorage').get()
m_stat = data_models.get_stat()
li = [random.randint(1, m_stat.url_count) for i in range(num)]
# li = [1]
query = URLStorage.gql('WHERE index IN :1', li)
recommendations = [item for item in query]
return recommendations
def txn():
rec = URLStorage.get(key)
if rec:
rec.topic = self.get_topic(rec.url)
img = self.get_snapshot(rec.url)
ss = SnapshotStorage(url = rec.url)
ss.snapshot = db.Blob(img)
ss.put()
rec.snapshot = '/img/blob_snapshot?key=%s' % ss.key()
rec.title = self.get_title(rec.url)
rec.put()
def txn():
rec = URLStorage.get(key)
if rec:
rec.topic = self.get_topic(rec.url)
img = self.get_snapshot(rec.url)
ss = SnapshotStorage(url=rec.url)
ss.snapshot = db.Blob(img)
ss.put()
rec.snapshot = '/img/blob_snapshot?key=%s' % ss.key()
rec.title = self.get_title(rec.url)
rec.put()
def get(self):
if self.user:
url = self.request.get('url')
permit = self.check_X_Frame_Options(url)
points = gmp_rules.worth_points(url, self.user)
url_info = URLStorage.gql('WHERE url = :1', url).get()
expertise_rmds = recommendAlg.expertiseRecommend(self.user.user_id)
sparse_rmds = recommendAlg.sparseRecommend(self.user.user_id)
diff_rmds = recommendAlg.diffRecommend(self.user.user_id)
self.render('evaluate.html',
url = url,
permit = permit,
points = points,
url_info = url_info,
expertise_rmds = expertise_rmds,
sparse_rmds = sparse_rmds,
diff_rmds = diff_rmds)
else:
self.render('welcome.html')
def get(self):
if self.user:
url = self.request.get('url')
permit = self.check_X_Frame_Options(url)
points = gmp_rules.worth_points(url, self.user)
url_info = URLStorage.gql('WHERE url = :1', url).get()
expertise_rmds = recommendAlg.expertiseRecommend(self.user.user_id)
sparse_rmds = recommendAlg.sparseRecommend(self.user.user_id)
diff_rmds = recommendAlg.diffRecommend(self.user.user_id)
self.render('evaluate.html',
url=url,
permit=permit,
points=points,
url_info=url_info,
expertise_rmds=expertise_rmds,
sparse_rmds=sparse_rmds,
diff_rmds=diff_rmds)
else:
self.render('welcome.html')
def sparseRecommend(user_id, num = DEF_REC_NUM):
recommendations = gae_cache.cache_get(user_id + 'sparsermd')
if recommendations:
return recommendations
query1 = URLStorage.gql('ORDER BY total ASC')
query2 = Rating.gql('WHERE user_id = :1', user_id)
rated = [item.url for item in query2]
recommendations = []
tmp = 0
for url_rec in query1:
tmp += 1
if url_rec.url not in rated:
recommendations.append(url_rec)
if len(recommendations) >= num:
break
gae_cache.cache_put(user_id+'sparsermd', recommendations)
return recommendations
def sparseRecommend(user_id, num=DEF_REC_NUM):
recommendations = gae_cache.cache_get(user_id + 'sparsermd')
if recommendations:
return recommendations
query1 = URLStorage.gql('ORDER BY total ASC')
query2 = Rating.gql('WHERE user_id = :1', user_id)
rated = [item.url for item in query2]
recommendations = []
tmp = 0
for url_rec in query1:
tmp += 1
if url_rec.url not in rated:
recommendations.append(url_rec)
if len(recommendations) >= num:
break
gae_cache.cache_put(user_id + 'sparsermd', recommendations)
return recommendations
def worth_points(url, user, update_cache=False):
points = gae_cache.cache_get(user.user_id + url + 'points')
if points and not update_cache:
return points
points = None
m_rating = Rating.gql('WHERE user_id = :1 AND url = :2', user.user_id,
url).get()
#if less than one hour between two attempts, no points are awarded
current_time = datetime.datetime.now()
if m_rating and (current_time - m_rating.time).total_seconds() < 3600:
points = 0
if points is None:
points = BASIC_RATING_POINTS
#punish multiple attempts to a specific url
punish_coefficient = 1
if m_rating:
punish_coefficient = math.pow(0.5, float(m_rating.attempts))
sparse_points = 0
diff_coefficient = 1
url_stat = URLStorage.gql('WHERE url = :1', url).get()
if url_stat:
for i in range(6):
if url_stat.total < SPARSITY_LEVEL[i]:
sparse_points = SPARSITY_POINTS[i]
break
r = url_stat.ratings
diff_coefficient = 1.0 / (((abs(r[4] + r[3] - r[1] - r[0])) /
(r[4] + r[3] + r[1] + r[0] + 10.0)) + 1)
points = int(
math.ceil(sparse_points * diff_coefficient *
punish_coefficient))
gae_cache.cache_put(user.user_id + url + 'points', points)
return points
def expertiseRecommend(user_id, num=DEF_REC_NUM):
recommendations = gae_cache.cache_get(user_id + 'expertisermd')
if recommendations:
return recommendations
expertise = ExUserInfo.gql('WHERE user_id = :1', user_id).get().expertise
if not expertise:
return None
query1 = URLStorage.gql('WHERE topic IN :1', expertise)
# url_keys = [item for item in query1.run(keys_only=True)]
# logging.info('number of expertise rmds: ' + str(len(url_keys)))
# li = [random.randint(0, len(url_keys)-1) for i in range(2*num)]
query2 = db.GqlQuery('SELECT url from Rating WHERE user_id = :1', user_id)
rated = [item.url for item in query2]
recommendations = []
for url_rec in query1:
# url_rec = URLStorage.get(url_keys[i])
if url_rec.url not in rated:
recommendations.append(url_rec)
if len(recommendations) >= num:
break
gae_cache.cache_put(user_id + 'expertisermd', recommendations)
return recommendations
def expertiseRecommend(user_id, num = DEF_REC_NUM):
recommendations = gae_cache.cache_get(user_id + 'expertisermd')
if recommendations:
return recommendations
expertise = ExUserInfo.gql('WHERE user_id = :1', user_id).get().expertise
if not expertise:
return None
query1 = URLStorage.gql('WHERE topic IN :1', expertise)
# url_keys = [item for item in query1.run(keys_only=True)]
# logging.info('number of expertise rmds: ' + str(len(url_keys)))
# li = [random.randint(0, len(url_keys)-1) for i in range(2*num)]
query2 = db.GqlQuery('SELECT url from Rating WHERE user_id = :1', user_id)
rated = [item.url for item in query2]
recommendations = []
for url_rec in query1:
# url_rec = URLStorage.get(url_keys[i])
if url_rec.url not in rated:
recommendations.append(url_rec)
if len(recommendations) >= num:
break
gae_cache.cache_put(user_id+'expertisermd', recommendations)
return recommendations
def get_url_info(url):
info = URLStorage.gql('WHERE url = :1', url).get()
return info
def get_url_info(url):
info = URLStorage.gql("WHERE url = :1", url).get()
return info
def test_import():
all = URLStorage.all()
def compute_diff():
urls = URLStorage.all()
for url_rec in urls:
data_models.update_url_diff(url_rec = url_rec)
