def test_cluster_same_feed(self): article = ArticleController().read(category_id__ne=None).first() cluster = article.cluster # all is enabled, article in cluster update_on_all_objs(articles=cluster.articles, cluster_enabled=True, cluster_same_feed=True) article = self.create_article_from(cluster, cluster.main_article.feed) self.assertInCluster(article, cluster) # feed's disabled, won't cluster FeedController().update( {'id__in': [a.feed_id for a in cluster.articles]}, {'cluster_same_feed': False}) article = self.create_article_from(cluster, cluster.main_article.feed) self.assertNotInCluster(article, cluster) # category's disabled, won't cluster FeedController().update( {'id__in': [a.feed_id for a in cluster.articles]}, {'cluster_same_feed': None}) CategoryController().update({'id': cluster.main_article.category.id}, {'cluster_same_feed': False}) article = self.create_article_from(cluster, cluster.main_article.feed) self.assertNotInCluster(article, cluster) # user's disable, won't cluster CategoryController().update({'id': cluster.main_article.category.id}, {'cluster_same_feed': None}) UserController().update({'id': cluster.user_id}, {'cluster_same_feed': False}) article = self.create_article_from(cluster, cluster.main_article.feed) self.assertNotInCluster(article, cluster) # reenabling user, will cluster UserController().update({'id': cluster.user_id}, {'cluster_same_feed': True}) article = self.create_article_from(cluster, cluster.main_article.feed) self.assertInCluster(article, cluster)
def setUp(self): super().setUp() login = '******' self.user = UserController().get(login=login) self.user2 = UserController().get(login='******') self.fctrl = FeedController(self.user.id) self.cctrl = CategoryController(self.user.id) self.uctrl = UserController()
def populate_db(): fcontr = FeedController() ccontr = CategoryController() UserController().create( **{ 'is_admin': True, 'is_api': True, 'cluster_enabled': False, 'login': '******', 'password': '******' }) user1, user2 = [ UserController().create(login=name, cluster_enabled=False, email="*****@*****.**" % name, password=name) for name in ["user1", "user2"] ] for iteration in range(2): article_total = 0 for user in (user1, user2): for iter_cat in range(3): cat_id = None if iter_cat: cat_id = ccontr.create(user_id=user.id, name=to_name( user, iteration, iter_cat)).id feed_id = fcontr.create( link="feed%d%d" % (iteration, iter_cat), user_id=user.id, category_id=cat_id, title=to_name(user, iteration, iter_cat, iter_cat)).id for iter_art in range(3): entry = to_name(user, iteration, iter_cat, iter_cat, iter_art) tags = [ to_name(user, iteration, iter_cat, iter_cat, iter_art, str(i)) for i in range(2) ] article_total += 1 ArticleController().create( entry_id=entry, link='http://test.te/%d' % article_total, feed_id=feed_id, user_id=user.id, tags=tags, category_id=cat_id, title=entry, date=utc_now() + timedelta(seconds=iteration), content="content %d" % article_total) session.commit() session.flush() ClusterController().clusterize_pending_articles()
def get(): """Given valid credentials, will provide a token to request the API.""" jwt = current_app.extensions["jwt"] user = UserController(current_identity.id).get(id=current_identity.id) access_token = jwt.jwt_encode_callback(user).decode("utf8") UserController(user.id).update({"id": user.id}, {"last_connection": utc_now(), "renew_password_token": ""}) SERVER.labels(method="get", uri="/auth/refresh", result='2XX').inc() return {"access_token": "%s %s" % (conf.auth.jwt_header_prefix, access_token)}, 200
def post(): """Initialize password recovery. Creates a uniq token and sending a mail with link to recovery page. """ attrs = login_init_recovery_parser.parse_args() token = str(random.getrandbits(128)) changed = UserController().update( { "login": attrs["login"], "email": attrs["email"] }, {"renew_password_token": token}) if not changed: SERVER.labels(method="post", uri="/auth/recovery", result='4XX').inc() raise BadRequest("No user with %r was found" % attrs) BASE_PATH = 'auth/recovery/%s/%s/%s' landing_url = get_ui_url(BASE_PATH % (attrs["login"], attrs["email"], token)) plaintext = render_template("mail_password_recovery.txt", plateform=conf.app.url, landing_url=landing_url) emails.send(to=attrs["email"], bcc=conf.notification.email, subject="[jarr] Password renew", plaintext=plaintext) SERVER.labels(method="post", uri="/auth/recovery", result='2XX').inc() return None, 204
def process_ids(cls, social_id, username, email): # pragma: no cover labels = {"method": "get", "uri": "/oauth/callback/" + cls.provider} if social_id is None: SERVER.labels(result="4XX", **labels).inc() raise UnprocessableEntity('No social id, authentication failed') ucontr = UserController() try: user = ucontr.get(**{'%s_identity' % cls.provider: social_id}) except NotFound: user = None if not user and not conf.oauth.allow_signup: SERVER.labels(result="4XX", **labels).inc() raise BadRequest('Account creation is not allowed through OAuth.') if not user: if username and not ucontr.read(login=username).count(): login = username else: login = '******' % (cls.provider, username or social_id) user = ucontr.create( **{ '%s_identity' % cls.provider: social_id, 'login': login, 'email': email }) jwt_ext = current_app.extensions['jwt'] access_token = jwt_ext.jwt_encode_callback(user).decode('utf8') SERVER.labels(result="2XX", **labels).inc() return { "access_token": "%s %s" % (conf.auth.jwt_header_prefix, access_token) }, 200
def test_articles_with_enclosure_and_fetched_content(self, truncated_cnt, get_vector): self._clean_objs() get_vector.return_value = None truncated_cnt.return_value = {'type': 'fetched', 'title': 'holy grail', 'content': 'blue, no read, aaah', 'link': 'https://monthy.python/brian'} feed = FeedController().read().first() FeedController().update({'id': feed.id}, {'truncated_content': True, 'cluster_enabled': True}) UserController().update({'id': feed.user_id}, {'cluster_enabled': True}) builder = ClassicArticleBuilder(feed, self.entry_w_enclosure, {}) self.assertIsNone(builder.article.get('article_type')) raw_articles = list(builder.enhance()) self.assertEqual(2, len(raw_articles)) self.assertEqual('audio', raw_articles[1]['article_type'].value) articles = [] for raw_article in raw_articles: articles.append( ArticleController(feed.user_id).create(**raw_article)) ClusterController(feed.user_id).clusterize_pending_articles() a1 = ArticleController().get(id=articles[0].id) a2 = ArticleController().get(id=articles[1].id) self.assertEqual(a1.cluster_id, a2.cluster_id) cluster = ClusterController().get(id=a1.cluster_id) self.assertEqual(2, cluster.content['v']) self.assertEqual(1, len(cluster.content['contents'])) self.assertEqual('fetched', cluster.content['contents'][0]['type'])
def scheduler(): logger.warning("Running scheduler") start = datetime.now() fctrl = FeedController() # browsing feeds to fetch feeds = list(fctrl.list_fetchable(conf.crawler.batch_size)) WORKER_BATCH.labels(worker_type='fetch-feed').observe(len(feeds)) logger.info('%d to enqueue', len(feeds)) for feed in feeds: logger.debug("%r: scheduling to be fetched", feed) process_feed.apply_async(args=[feed.id]) # browsing feeds to delete feeds_to_delete = list(fctrl.read(status=FeedStatus.to_delete)) if feeds_to_delete and REDIS_CONN.setnx(JARR_FEED_DEL_KEY, 'true'): REDIS_CONN.expire(JARR_FEED_DEL_KEY, LOCK_EXPIRE) logger.info('%d to delete, deleting one', len(feeds_to_delete)) for feed in feeds_to_delete: logger.debug("%r: scheduling to be delete", feed) feed_cleaner.apply_async(args=[feed.id]) break # only one at a time # applying clusterizer for user_id in ArticleController.get_user_id_with_pending_articles(): if not UserController().get(id=user_id).effectivly_active: continue if REDIS_CONN.setnx(JARR_CLUSTERIZER_KEY % user_id, 'true'): REDIS_CONN.expire(JARR_CLUSTERIZER_KEY % user_id, conf.crawler.clusterizer_delay) clusterizer.apply_async(args=[user_id]) scheduler.apply_async(countdown=conf.crawler.idle_delay) WORKER.labels(method='scheduler').observe( (datetime.now() - start).total_seconds()) update_slow_metrics.apply_async()
def metrics_users_long_term(): logger.debug('Counting long term users') threshold_connection = utc_now() - timedelta(days=conf.feed.stop_fetch) threshold_connection = utc_now() - timedelta(days=conf.feed.stop_fetch) threshold_created = utc_now() - timedelta(days=conf.feed.stop_fetch + 1) long_term = UserController().read(is_active=True, last_connection__ge=threshold_connection, date_created__lt=threshold_created) USER.labels(status='long_term').set(long_term.count())
def put(): user_id = current_identity.id attrs = parse_meaningful_params(parser_edit) if not attrs: raise BadRequest() return UserController(user_id).update({ "id": user_id }, attrs, return_objs=True).first(), 200
def put(): """Sending new password with recovery token.""" labels = {"method": "put", "uri": "/auth/recovery"} attrs = login_recovery_parser.parse_args() try: user = UserController().get(login=attrs["login"], email=attrs["email"]) except Exception: SERVER.labels(result="4XX", **labels).inc() raise if (not user.renew_password_token or not attrs["token"] or user.renew_password_token != attrs["token"]): SERVER.labels(result="4XX", **labels).inc() raise Forbidden() result = UserController().update({"id": user.id}, {"renew_password_token": "", "password": attrs["password"]}) SERVER.labels(result="2XX" if result else "4XX", **labels).inc() return None, 204 if result else 403
def db_create(login='******', password='******'): """Will create the database from conf parameters.""" admin = { 'is_admin': True, 'is_api': True, 'login': login, 'password': password } Base.metadata.create_all() UserController().create(**admin)
def test_cluster_enabled(self): ccontr = ClusterController() cluster = ccontr.read().first() feed = FeedController(cluster.user_id).read( category_id__ne=None, id__nin=[art.feed_id for art in cluster.articles]).first() category = feed.category # clustering works when all is true update_on_all_objs(articles=cluster.articles, feeds=[feed], cluster_enabled=True) article = self.create_article_from(cluster, feed) self.assertInCluster(article, cluster) # disabling on user desactivate all clustering by default update_on_all_objs(articles=cluster.articles, feeds=[feed], cluster_enabled=None) UserController().update({'id': cluster.user_id}, {'cluster_enabled': False}) article = self.create_article_from(cluster, feed) self.assertNotInCluster(article, cluster) # disabling on article's feed prevents from clustering update_on_all_objs(articles=cluster.articles, feeds=[feed], cluster_enabled=True) FeedController().update({'id': feed.id}, {'cluster_enabled': False}) article = self.create_article_from(cluster, feed) self.assertNotInCluster(article, cluster) # disabling on feed from cluster's articles prevents from clustering update_on_all_objs(articles=cluster.articles, feeds=[feed], cluster_enabled=True) FeedController().update( {'id__in': [a.feed_id for a in cluster.articles]}, {'cluster_enabled': False}) article = self.create_article_from(cluster, feed) self.assertNotInCluster(article, cluster) # disabling on article's category prevents from clustering CategoryController(cluster.user_id).update({'id': category.id}, {'cluster_enabled': False}) article = self.create_article_from(cluster, feed) self.assertNotInCluster(article, cluster) update_on_all_objs(articles=cluster.articles, feeds=[feed], cluster_enabled=True) article = self.create_article_from(cluster, feed) self.assertInCluster(article, cluster)
def test_ClusterResource_get(self): user = UserController().get(login='******') cluster = ClusterController(user.id).read().first() resp = self.jarr_client('get', 'cluster', cluster.id) self.assertStatusCode(401, resp) resp = self.jarr_client('get', 'cluster', cluster.id, user='******') self.assertStatusCode(403, resp) resp = self.jarr_client('get', 'cluster', cluster.id, user=user.login) self.assertStatusCode(226, resp) self.assertEqual(1, len(resp.json['articles'])) resp = self.jarr_client('get', 'cluster', cluster.id, user=user.login) self.assertStatusCode(200, resp)
def update_slow_metrics(): uctrl = UserController() USER.labels(status='any').set(uctrl.read().count()) threshold_connection = utc_now() - timedelta(days=conf.feed.stop_fetch) threshold_created = utc_now() - timedelta(days=conf.feed.stop_fetch + 1) active = uctrl.read(is_active=True, last_connection__ge=threshold_connection) USER.labels(status='active').set(active.count()) long_term = uctrl.read(is_active=True, last_connection__ge=threshold_connection, date_created__lt=threshold_created) USER.labels(status='long_term').set(long_term.count())
def test_password(self): login = '******' passwd = 'test_password' ucontr = UserController() user = ucontr.create(login=login, password=passwd) self.assertNotEqual(passwd, user.password) self.assertEqual(user, ucontr.check_password(login, passwd)) self.assertIsNone(ucontr.check_password(login, passwd * 2)) passwd *= 2 ucontr.update({'id': user.id}, {'password': passwd}) user = ucontr.get(id=user.id) self.assertNotEqual(passwd, user.password) self.assertEqual(user, ucontr.check_password(login, passwd)) self.assertIsNone(ucontr.check_password(login, passwd * 2))
def test_ClusterResource_delete(self): cluster = ClusterController().read().first() user = UserController().get(id=cluster.user_id) resp = self.jarr_client('delete', 'cluster', cluster.id) self.assertStatusCode(401, resp) resp = self.jarr_client('delete', 'cluster', cluster.id, user='******') self.assertStatusCode(403, resp) resp = self.jarr_client('delete', 'cluster', cluster.id, user=user.login) self.assertStatusCode(204, resp) self.assertEqual(0, ClusterController().read(id=cluster.id).count()) self.assertEqual(0, ArticleController().read(cluster_id=cluster.id).count())
def post(): """Given valid credentials, will provide a token to request the API.""" attrs = login_parser.parse_args() jwt = current_app.extensions["jwt"] user = jwt.authentication_callback(attrs["login"], attrs["password"]) if not user: SERVER.labels(method="post", uri="auth", result='4XX').inc() raise Forbidden() access_token = jwt.jwt_encode_callback(user).decode("utf8") UserController(user.id).update({"id": user.id}, {"last_connection": utc_now(), "renew_password_token": ""}) SERVER.labels(method="post", uri="/auth", result='2XX').inc() return {"access_token": "%s %s" % (conf.auth.jwt_header_prefix, access_token)}, 200
def get(): user_id = current_identity.id user = UserController(user_id).get(id=user_id) categories = { cat.id: cat for cat in CategoryController(user_id).read() } response = make_response( render_template('opml.xml', user=user, categories=categories, feeds=FeedController(user_id).read(), now=utc_now())) for key, value in OK_GET_HEADERS.items(): response.headers[key] = value return response
def test_articles_with_enclosure(self): self._clean_objs() feed = FeedController().read().first() UserController().update({'id': feed.user_id}, {'cluster_enabled': True}) builder = ClassicArticleBuilder(feed, self.entry_w_enclosure, {}) self.assertIsNone(builder.article.get('article_type')) raw_articles = list(builder.enhance()) self.assertEqual(2, len(raw_articles)) self.assertEqual('audio', raw_articles[1]['article_type'].value) articles = [] for raw_article in raw_articles: articles.append( ArticleController(feed.user_id).create(**raw_article)) ClusterController(feed.user_id).clusterize_pending_articles() a1 = ArticleController().get(id=articles[0].id) a2 = ArticleController().get(id=articles[1].id) cluster = ClusterController().get(id=a1.cluster_id) self.assertEqual(a1.cluster_id, a2.cluster_id) self.assertEqual(2, cluster.content['v']) self.assertEqual(0, len(cluster.content['contents']))
def test_ClusterResource_put(self): cluster = ClusterController().read().first() user = UserController().get(id=cluster.user_id) resp = self.jarr_client('put', 'cluster', cluster.id, data={'read': True}) self.assertStatusCode(401, resp) resp = self.jarr_client('put', 'cluster', cluster.id, data={'read': True}, user='******') self.assertStatusCode(403, resp) # marking as read resp = self.jarr_client('put', 'cluster', cluster.id, data={'read': True}, user=user.login) self.assertStatusCode(204, resp) cluster = ClusterController().get(id=cluster.id) self.assertTrue(cluster.read) self.assertFalse(cluster.liked) self.assertEqual('marked', cluster.read_reason.value) # marking as read / consulted resp = self.jarr_client('put', 'cluster', cluster.id, data={'read_reason': 'consulted', 'read': True}, user=user.login) self.assertStatusCode(204, resp) cluster = ClusterController().get(id=cluster.id) self.assertTrue(cluster.read) self.assertFalse(cluster.liked) self.assertEqual('consulted', cluster.read_reason.value) # marking as liked resp = self.jarr_client('put', 'cluster', cluster.id, data={'liked': True}, user=user.login) self.assertStatusCode(204, resp) self.assertTrue(ClusterController().get(id=cluster.id).read) self.assertTrue(ClusterController().get(id=cluster.id).liked) resp = self.jarr_client('put', 'cluster', cluster.id, data={'liked': False, 'read': False}, user=user.login) self.assertStatusCode(204, resp) self.assertFalse(ClusterController().get(id=cluster.id).read) self.assertFalse(ClusterController().get(id=cluster.id).liked) self.assertIsNone(ClusterController().get(id=cluster.id).read_reason)
def update_on_all_objs(articles=None, feeds=None, categories=None, users=None, **kwargs): articles = articles or [] feeds = feeds or [] categories = categories or [] users = users or [] feed_ids = {a.feed_id for a in articles}.union({f.id for f in feeds}) category_ids = {a.category_id for a in articles if a.category_id}\ .union({f.category_id for f in feeds if f.category_id})\ .union({c.id for c in categories}) user_ids = {a.user_id for a in articles}\ .union({f.user_id for f in feeds})\ .union({c.user_id for c in categories})\ .union({u.id for u in users}) FeedController().update( { '__or__': [{ 'id__in': feed_ids }, { 'category_id__in': category_ids }, { 'user_id__in': user_ids }] }, kwargs) CategoryController().update( {'__or__': [{ 'id__in': category_ids }, { 'user_id__in': user_ids }]}, kwargs) UserController().update( {'id__in': user_ids}, # pushing default to user if not specifically false {k: v if v is not None else True for k, v in kwargs.items()})
def test_scheduler(self): scheduler() UserController().update({}, {'last_connection': utc_now()}) fctrl = FeedController() epoch = datetime(1970, 1, 1, tzinfo=timezone.utc) self.assertEqual(fctrl.read().count(), self.process_feed_patch.apply_async.call_count) self.assertEqual(0, self.clusteriser_patch.apply_async.call_count) self.assertEqual(0, self.feed_cleaner_patch.apply_async.call_count) feed1, feed2, feed3 = list(FeedController().read().limit(3)) FeedController().update({'id__in': [feed1.id, feed3.id]}, {'status': 'to_delete'}) FeedController().update({'id': feed2.id}, { 'last_retrieved': epoch, 'expires': epoch }) self.assertEqual(1, len(list(fctrl.list_fetchable()))) scheduler() self.assertEqual(fctrl.read().count(), self.process_feed_patch.apply_async.call_count) self.assertEqual(0, self.clusteriser_patch.apply_async.call_count) self.assertEqual(1, self.feed_cleaner_patch.apply_async.call_count)
def metrics_users_any(): logger.debug('Counting users') USER.labels(status='any').set(UserController().read().count())
def test_feed_rights(self): feed = FeedController(2).read()[0] self.assertEqual(3, ArticleController().read(feed_id=feed.id).count()) self._test_controller_rights(feed, UserController().get(id=feed.user_id))
def metrics_users_active(): logger.debug('Counting active users') threshold_connection = utc_now() - timedelta(days=conf.feed.stop_fetch) active = UserController().read(is_active=True, last_connection__ge=threshold_connection) USER.labels(status='active').set(active.count())
def setUp(self): super().setUp() login = '******' self.user = UserController().get(login=login) self.user2 = UserController().get(login='******') self.uctrl = UserController()
def test_feed_rights(self): cat = CategoryController(2).read().first() self.assertEqual(3, ArticleController().read(category_id=cat.id).count()) self.assertEqual(1, FeedController().read(category_id=cat.id).count()) self._test_controller_rights(cat, UserController().get(id=cat.user_id))
def update_slow_metrics(): uctrl = UserController() USER.labels(status='any').set(uctrl.read().count()) USER.labels(status='active').set(uctrl.list_active().count())
def test_article_rights(self): article = ArticleController(USER_ID).read().first() self._test_controller_rights(article, UserController().get(id=article.user_id))