def get_article_samples(config_):
#Connect to mongo database
logger.info("Connect to database...")
connect(config_['database']['db-name'],
username= config_['database']['user'],
password= config_['database']['passwd'],
port = config_['database']['port'])
#get user
user = User.objects(email=u"*****@*****.**").first()
ranked_article_ids = (a.article.id
for a
in RankedArticle.objects(user_id = user.id).only("article"))
all_article_ids = Set(a.id
for a
in Article.objects(id__in = ranked_article_ids).only("id"))
read_article_ids = Set(a.article.id
for a
in ReadArticleFeedback.objects(user_id = user.id).only("article"))
unread_article_ids = all_article_ids - read_article_ids
#sample test articles
X, y = get_samples(extractor, read_article_ids, unread_article_ids)
return X, y
def ajax_add_user():
'''
Called remotely to add a new user.
'''
if not current_user.is_authenticated():
abort(403)
name = request.form['name']
email = request.form['email'].lower()
new_password = request.form['new_password']
new_password_repeat = request.form['new_password_repeat']
if current_user.mongodb_user.email != "*****@*****.**":
abort(403)
#check passwords
if new_password != new_password_repeat:
abort(400)
if new_password == "":
abort(400)
#hash password
m = hashlib.sha256()
m.update(new_password.encode("UTF-8"))
m.update(SALT.encode("UTF-8"))
#check if user with email address already exists
users_with_same_email = User.objects(email = email)
if len(users_with_same_email) > 0:
abort(400)
try:
app.logger.debug("Adding new user %s" % name)
#just pick the first article as feedback
first_article = Article.objects().first()
first_profile = LearnedProfile(features = first_article.features)
new_user = User(name = name, password = m.hexdigest(),
email = email,
learned_profile = [first_profile])
new_user.save(safe=True)
first_feedback = ReadArticleFeedback(user_id = new_user.id,
article = first_article,
score = 1.0)
first_feedback.save()
app.logger.debug("...done.")
except Exception as inst:
app.logger.error("Could not add new user: %s: %s" % (type(inst), type))
abort(500)
return ""
def load_user(user_id):
'''
Loads user from Database
'''
try:
user = User.objects(id = user_id).first()
except Exception as inst:
app.logger.error("Could not login user %s: %s" % (type(inst), type))
return None
if user == None:
app.logger.error("No user found for %s" % user_id)
return None
return AppUser(user)
def setUp(self):
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.DEBUG)
config = load_config(file_path = "/home/karten/Programmierung/frontend/config.yaml",
logger = logger,
exit_with_error = True)
#Connect to test database
connect("nyan_test", port = 27017)
fill_database()
#connect(config['database']['db-name'],
# username= config['database']['user'],
# password= config['database']['passwd'],
# port = config['database']['port'])
self.user_id = User.objects(email = u'*****@*****.**').first().id
#feature_extractor = EsaFeatureExtractor(prefix = config['prefix'])
feature_extractor = TfidfFeatureExtractor(prefix = config['prefix'])
self.trainer = UserModelBayes(self.user_id, extractor = feature_extractor)
def setUp(self):
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.DEBUG)
config = load_config(file_path = ("/media/sdc1/Aptana Studio 3 Workspace"
"/configs/config.yaml"),
logger = logger,
exit_with_error = True)
#Connect to test database
connect("nyan_test", port = 20545)
fill_database()
#connect(config['database']['db-name'],
# username= config['database']['user'],
# password= config['database']['passwd'],
# port = config['database']['port'])
self.user_id = User.objects(email = u'*****@*****.**').first().id
#feature_extractor = EsaFeatureExtractor(prefix = config['prefix'])
feature_extractor = TfidfFeatureExtractor(prefix = config['prefix'])
self.trainer = UserModelSVM(self.user_id, extractor = feature_extractor)
def login():
if current_user.is_authenticated():
return redirect(request.args.get("next") or url_for('top'))
if request.method == 'POST':
e_mail = request.form['e_mail'].lower()
password = request.form['password']
if e_mail != None and password!=None:
#get user from database
try:
start = time.time()
users = User.objects(email = e_mail)
user = users.first()
end = time.time()
app.logger.info("Getting user took %f.5 seconds." % (end-start))
except Exception as inst:
app.logger.error("Could not login user %s: %s" %
(type(inst), type))
raise abort(500)
if user == None:
app.logger.error("No user found for %s" % e_mail)
flash('Username or password are not correct.', 'error')
else:
#check password
m = hashlib.sha256()
m.update(password.encode("UTF-8"))
m.update(SALT.encode("UTF-8"))
if m.hexdigest() == user.password:
app.logger.debug("Login %s" % e_mail)
login_user(AppUser(user))
return redirect(request.args.get("next") or url_for('top'))
else:
flash('Username or password are not correct.', 'error')
return render_template('login.html')
def __init__(self, user_id, extractor):
self.user = User.objects(id = user_id).first()
self.extractor = extractor
self.num_features_ = self.extractor.get_feature_number()
connect(
config_["database"]["db-name"],
username=config_["database"]["user"],
password=config_["database"]["passwd"],
port=config_["database"]["port"],
)
# Load feature extractor
# feature_extractor = EsaFeatureExtractor(prefix = config_['prefix'])
# feature_extractor = TfidfFeatureExtractor(prefix = config_['prefix'])
# feature_extractor = LdaFeatureExtractor(prefix = config_['prefix'])
# feature_extractor = LdaBowFeatureExtractor(prefix = config_['prefix'])
feature_extractor = cEsaFeatureExtractor(prefix=config_["prefix"])
# get user
user = User.objects(email=u"*****@*****.**").first()
ranked_article_ids = (a.article.id for a in RankedArticle.objects(user_id=user.id).only("article"))
all_article_ids = set(a.id for a in Article.objects(id__in=ranked_article_ids).only("id"))
read_article_ids = set(a.article.id for a in ReadArticleFeedback.objects(user_id=user.id).only("article"))
unread_article_ids = all_article_ids - read_article_ids
for p_synthetic in xrange(100, 700, 100):
for p_majority in xrange(100, 700, 100):
logger.info("Synthetic over-sampling %d and majority undersampling %d" % (p_synthetic, p_majority))
# run test N_ITERATIONS
precisions_read = np.zeros((N_ITERATIONS))
#Connect to mongo database
connect(config_['database']['db-name'],
username= config_['database']['user'],
password= config_['database']['passwd'],
port = config_['database']['port'])
#Load feature extractor
#feature_extractor = EsaFeatureExtractor(prefix = config_['prefix'])
#feature_extractor = TfidfFeatureExtractor(prefix = config_['prefix'])
#feature_extractor = LdaFeatureExtractor(prefix = config_['prefix'])
#feature_extractor = LdaBowFeatureExtractor(prefix = config_['prefix'])
feature_extractor = cEsaFeatureExtractor(prefix = config_['prefix'])
#get user
user = User.objects(email=u"*****@*****.**").first()
ranked_article_ids = (a.article.id
for a
in RankedArticle.objects(user_id = user.id).only("article"))
all_article_ids = Set(a.id
for a
in Article.objects(id__in = ranked_article_ids).only("id"))
read_article_ids = Set(a.article.id
for a
in ReadArticleFeedback.objects(user_id = user.id).only("article"))
unread_article_ids = all_article_ids - read_article_ids
for p_synthetic in xrange(100, 700, 100):
def set_password(self, new_password):
'''
Sets the new password. Password should already be hashed etc.
'''
#do atomic set
User.objects(id = self.mongodb_user.id).update_one(set__password = new_password)
def remove_vendor_from_subscriptions(self, vendor):
'''
Removes vendor form subscriptions.
vendor should be a mongodb_model object.
'''
User.objects(id=self.mongodb_user.id).update_one(pull__subscriptions=vendor)
def add_vendor_to_subscriptions(self, vendor):
'''
Adds a new vendor to subscriptions.
vendor should be a mongodb_model object.
'''
User.objects(id=self.mongodb_user.id).update_one(add_to_set__subscriptions=vendor)
def test_constructor_with_file_wikicorpus(self):
#load tf-idf model
tfidf_model = tfidfmodel.TfidfModel.load("/media/sdc1/test_dump/result/test_tfidf.model")
extractor = TfidfFeatureExtractor("/media/sdc1/test_dump/result/test")
#load tf-idf corpus
tfidf_corpus = MmCorpus('/media/sdc1/test_dump/result/test_tfidf_corpus.mm')
#load lda corpus
#lda_corpus = MmCorpus('/media/sdc1/test_dump/result/test_lda_corpus.mm')
#load dictionary
id2token = Dictionary.load("/media/sdc1/test_dump/result/test_wordids.dict")
#load article titles
document_titles = DocumentTitles.load("/media/sdc1/test_dump/result/test_articles.txt")
#Connect to mongo database
connect(self.config_['database']['db-name'],
username= self.config_['database']['user'],
password= self.config_['database']['passwd'],
port = self.config_['database']['port'])
#Load articles as test corpus
user = User.objects(email=u"*****@*****.**").first()
ranked_article_ids = (a.article.id
for a
in RankedArticle.objects(user_id = user.id).only("article"))
all_article_ids = Set(a.id
for a
in Article.objects(id__in = ranked_article_ids).only("id"))
read_article_ids = Set(a.article.id
for a
in ReadArticleFeedback.objects(user_id = user.id).only("article"))
unread_article_ids = all_article_ids - read_article_ids
#sample test articles
X, y = get_samples(extractor, read_article_ids, unread_article_ids)
s,f = X.shape
logger.debug("Traning with %d samples, %d features, %d marks" %
(s,f, len(y)))
#train esa model
esa_model = CosineEsaModel(tfidf_corpus,
document_titles = document_titles,
test_corpus = X,
test_corpus_targets = y,
num_test_corpus = len(y),
num_best_features = 15,
num_features = len(id2token))
print esa_model
esa_model.save('/media/sdc1/test_dump/result/test_cesa.model')
tmp_esa = CosineEsaModel.load('/media/sdc1/test_dump/result/test_cesa.model')
print tmp_esa
level=logging.DEBUG,
filename=options.log)
#load config
logger = logging.getLogger("main")
logging.info("Load config")
config_ = load_config(options.config, logger, exit_with_error = True)
if config_ == None:
logger.error("No config. Exit.")
sys.exit(1)
#Connect to mongo database
connect(config_['database']['db-name'],
username= config_['database']['user'],
password= config_['database']['passwd'],
port = config_['database']['port'])
feature_extractor = EsaFeatureExtractor(prefix = config_['prefix'])
logger.info("Learn user model...")
users = User.objects()
for u in users:
logger.info("for %s" % u.name)
trainer = UserModelCentroid(user_id = u.id,
extractor = feature_extractor)
trainer.train()
trainer.save()
logger.info("...done.")