def boredRecommend(user_id, num = DEF_REC_NUM): recommendations = gae_cache.cache_get(user_id + 'boredrmd') if recommendations: return recommendations expertise = ExUserInfo.gql('WHERE user_id = :1', user_id).get().expertise diff_expertise = set(data_models.EXPERTISES.keys()) diff_expertise.difference_update(set(expertise)) if not diff_expertise: return None query1 = URLStorage.gql('WHERE topic IN :1', list(diff_expertise)) # url_recs = [item for item in query1] # logging.info('number of bored rmds: ' + str(len(url_recs))) # li = [random.randint(0, len(url_recs)-1) for i in range(2*num)] query2 = Rating.gql('WHERE user_id = :1', user_id) rated = [item.url for item in query2] recommendations = [] for url_rec in query1: if url_rec.url not in rated: recommendations.append(url_rec) if len(recommendations) >= num: break gae_cache.cache_put(user_id+'boredrmd', recommendations) return recommendations
def boredRecommend(user_id, num=DEF_REC_NUM): recommendations = gae_cache.cache_get(user_id + 'boredrmd') if recommendations: return recommendations expertise = ExUserInfo.gql('WHERE user_id = :1', user_id).get().expertise diff_expertise = set(data_models.EXPERTISES.keys()) diff_expertise.difference_update(set(expertise)) if not diff_expertise: return None query1 = URLStorage.gql('WHERE topic IN :1', list(diff_expertise)) # url_recs = [item for item in query1] # logging.info('number of bored rmds: ' + str(len(url_recs))) # li = [random.randint(0, len(url_recs)-1) for i in range(2*num)] query2 = Rating.gql('WHERE user_id = :1', user_id) rated = [item.url for item in query2] recommendations = [] for url_rec in query1: if url_rec.url not in rated: recommendations.append(url_rec) if len(recommendations) >= num: break gae_cache.cache_put(user_id + 'boredrmd', recommendations) return recommendations
def worth_points(url, user, update_cache = False): points = gae_cache.cache_get(user.user_id+url+'points') if points and not update_cache: return points points = None m_rating = Rating.gql('WHERE user_id = :1 AND url = :2', user.user_id, url).get() #if less than one hour between two attempts, no points are awarded current_time = datetime.datetime.now() if m_rating and (current_time - m_rating.time).total_seconds() < 3600: points = 0 if points is None: points = BASIC_RATING_POINTS #punish multiple attempts to a specific url punish_coefficient = 1 if m_rating: punish_coefficient = math.pow(0.5, float(m_rating.attempts)) sparse_points = 0 diff_coefficient = 1 url_stat = URLStorage.gql('WHERE url = :1', url).get() if url_stat: for i in range(6): if url_stat.total < SPARSITY_LEVEL[i]: sparse_points = SPARSITY_POINTS[i] break r = url_stat.ratings diff_coefficient = 1.0 / (((abs(r[4] + r[3] - r[1] - r[0])) / (r[4] + r[3] + r[1] + r[0] + 10.0)) + 1) points = int(math.ceil(sparse_points * diff_coefficient * punish_coefficient)) gae_cache.cache_put(user.user_id+url+'points', points) return points
def get(self): url = self.request.get('url') status = [] avg = 0 total = 0 m_stat = URLStorage.gql('WHERE url = :1', url).get() if m_stat: status = m_stat.ratings total = m_stat.total avg = 0 if total != 0: avg = sum([(i + 1) * status[i] for i in range(5)]) * 1.0 / total status.reverse() (truthfulness, unbiased, security, design) = self.get_details(url) self.render('status_table.html', status=status, url=url, avg=avg, truthfulness=truthfulness, unbiased=unbiased, security=security, design=design, total=total)
def randomAlg(num = DEF_REC_NUM): # m_stat = stats.KindStat.gql('WHERE kind_name = :1', 'URLStorage').get() m_stat = data_models.get_stat() li = [random.randint(1, m_stat.url_count) for i in range(num)] # li = [1] query = URLStorage.gql('WHERE index IN :1', li) recommendations = [item for item in query] return recommendations
def randomAlg(num=DEF_REC_NUM): # m_stat = stats.KindStat.gql('WHERE kind_name = :1', 'URLStorage').get() m_stat = data_models.get_stat() li = [random.randint(1, m_stat.url_count) for i in range(num)] # li = [1] query = URLStorage.gql('WHERE index IN :1', li) recommendations = [item for item in query] return recommendations
def txn(): rec = URLStorage.get(key) if rec: rec.topic = self.get_topic(rec.url) img = self.get_snapshot(rec.url) ss = SnapshotStorage(url = rec.url) ss.snapshot = db.Blob(img) ss.put() rec.snapshot = '/img/blob_snapshot?key=%s' % ss.key() rec.title = self.get_title(rec.url) rec.put()
def txn(): rec = URLStorage.get(key) if rec: rec.topic = self.get_topic(rec.url) img = self.get_snapshot(rec.url) ss = SnapshotStorage(url=rec.url) ss.snapshot = db.Blob(img) ss.put() rec.snapshot = '/img/blob_snapshot?key=%s' % ss.key() rec.title = self.get_title(rec.url) rec.put()
def get(self): if self.user: url = self.request.get('url') permit = self.check_X_Frame_Options(url) points = gmp_rules.worth_points(url, self.user) url_info = URLStorage.gql('WHERE url = :1', url).get() expertise_rmds = recommendAlg.expertiseRecommend(self.user.user_id) sparse_rmds = recommendAlg.sparseRecommend(self.user.user_id) diff_rmds = recommendAlg.diffRecommend(self.user.user_id) self.render('evaluate.html', url = url, permit = permit, points = points, url_info = url_info, expertise_rmds = expertise_rmds, sparse_rmds = sparse_rmds, diff_rmds = diff_rmds) else: self.render('welcome.html')
def get(self): if self.user: url = self.request.get('url') permit = self.check_X_Frame_Options(url) points = gmp_rules.worth_points(url, self.user) url_info = URLStorage.gql('WHERE url = :1', url).get() expertise_rmds = recommendAlg.expertiseRecommend(self.user.user_id) sparse_rmds = recommendAlg.sparseRecommend(self.user.user_id) diff_rmds = recommendAlg.diffRecommend(self.user.user_id) self.render('evaluate.html', url=url, permit=permit, points=points, url_info=url_info, expertise_rmds=expertise_rmds, sparse_rmds=sparse_rmds, diff_rmds=diff_rmds) else: self.render('welcome.html')
def sparseRecommend(user_id, num = DEF_REC_NUM): recommendations = gae_cache.cache_get(user_id + 'sparsermd') if recommendations: return recommendations query1 = URLStorage.gql('ORDER BY total ASC') query2 = Rating.gql('WHERE user_id = :1', user_id) rated = [item.url for item in query2] recommendations = [] tmp = 0 for url_rec in query1: tmp += 1 if url_rec.url not in rated: recommendations.append(url_rec) if len(recommendations) >= num: break gae_cache.cache_put(user_id+'sparsermd', recommendations) return recommendations
def sparseRecommend(user_id, num=DEF_REC_NUM): recommendations = gae_cache.cache_get(user_id + 'sparsermd') if recommendations: return recommendations query1 = URLStorage.gql('ORDER BY total ASC') query2 = Rating.gql('WHERE user_id = :1', user_id) rated = [item.url for item in query2] recommendations = [] tmp = 0 for url_rec in query1: tmp += 1 if url_rec.url not in rated: recommendations.append(url_rec) if len(recommendations) >= num: break gae_cache.cache_put(user_id + 'sparsermd', recommendations) return recommendations
def worth_points(url, user, update_cache=False): points = gae_cache.cache_get(user.user_id + url + 'points') if points and not update_cache: return points points = None m_rating = Rating.gql('WHERE user_id = :1 AND url = :2', user.user_id, url).get() #if less than one hour between two attempts, no points are awarded current_time = datetime.datetime.now() if m_rating and (current_time - m_rating.time).total_seconds() < 3600: points = 0 if points is None: points = BASIC_RATING_POINTS #punish multiple attempts to a specific url punish_coefficient = 1 if m_rating: punish_coefficient = math.pow(0.5, float(m_rating.attempts)) sparse_points = 0 diff_coefficient = 1 url_stat = URLStorage.gql('WHERE url = :1', url).get() if url_stat: for i in range(6): if url_stat.total < SPARSITY_LEVEL[i]: sparse_points = SPARSITY_POINTS[i] break r = url_stat.ratings diff_coefficient = 1.0 / (((abs(r[4] + r[3] - r[1] - r[0])) / (r[4] + r[3] + r[1] + r[0] + 10.0)) + 1) points = int( math.ceil(sparse_points * diff_coefficient * punish_coefficient)) gae_cache.cache_put(user.user_id + url + 'points', points) return points
def expertiseRecommend(user_id, num=DEF_REC_NUM): recommendations = gae_cache.cache_get(user_id + 'expertisermd') if recommendations: return recommendations expertise = ExUserInfo.gql('WHERE user_id = :1', user_id).get().expertise if not expertise: return None query1 = URLStorage.gql('WHERE topic IN :1', expertise) # url_keys = [item for item in query1.run(keys_only=True)] # logging.info('number of expertise rmds: ' + str(len(url_keys))) # li = [random.randint(0, len(url_keys)-1) for i in range(2*num)] query2 = db.GqlQuery('SELECT url from Rating WHERE user_id = :1', user_id) rated = [item.url for item in query2] recommendations = [] for url_rec in query1: # url_rec = URLStorage.get(url_keys[i]) if url_rec.url not in rated: recommendations.append(url_rec) if len(recommendations) >= num: break gae_cache.cache_put(user_id + 'expertisermd', recommendations) return recommendations
def expertiseRecommend(user_id, num = DEF_REC_NUM): recommendations = gae_cache.cache_get(user_id + 'expertisermd') if recommendations: return recommendations expertise = ExUserInfo.gql('WHERE user_id = :1', user_id).get().expertise if not expertise: return None query1 = URLStorage.gql('WHERE topic IN :1', expertise) # url_keys = [item for item in query1.run(keys_only=True)] # logging.info('number of expertise rmds: ' + str(len(url_keys))) # li = [random.randint(0, len(url_keys)-1) for i in range(2*num)] query2 = db.GqlQuery('SELECT url from Rating WHERE user_id = :1', user_id) rated = [item.url for item in query2] recommendations = [] for url_rec in query1: # url_rec = URLStorage.get(url_keys[i]) if url_rec.url not in rated: recommendations.append(url_rec) if len(recommendations) >= num: break gae_cache.cache_put(user_id+'expertisermd', recommendations) return recommendations
def get_url_info(url): info = URLStorage.gql('WHERE url = :1', url).get() return info
def get_url_info(url): info = URLStorage.gql("WHERE url = :1", url).get() return info
def test_import(): all = URLStorage.all()
def compute_diff(): urls = URLStorage.all() for url_rec in urls: data_models.update_url_diff(url_rec = url_rec)