def fetch_asyncio(user_id=None, feed_id=None): "Crawl the feeds with asyncio." import asyncio with application.app_context(): from crawler import default_crawler filters = {} filters['is_active'] = True filters['automatic_crawling'] = True if None is not user_id: filters['id'] = user_id users = UserController().read(**filters).all() try: feed_id = int(feed_id) except: feed_id = None logger.info('Starting crawler.') start = datetime.now() loop = asyncio.get_event_loop() for user in users: default_crawler.retrieve_feed(loop, user, feed_id) loop.close() end = datetime.now() logger.info('Crawler finished in {} seconds.' \ .format((end - start).seconds))
def fetch_asyncio(user_id, feed_id): "Crawl the feeds with asyncio." import asyncio with application.app_context(): populate_g() from flask import g from web.models import User from crawler import classic_crawler users = [] try: users = User.query.filter(User.id == int(user_id)).all() except: users = User.query.all() finally: if users == []: users = User.query.all() try: feed_id = int(feed_id) except: feed_id = None loop = asyncio.get_event_loop() for user in users: if user.activation_key == "": print("Fetching articles for " + user.nickname) g.user = user classic_crawler.retrieve_feed(loop, g.user, feed_id) loop.close()
def fetch_asyncio(user_id, feed_id): "Crawl the feeds with asyncio." import asyncio with application.app_context(): from flask.ext.login import current_user from crawler import classic_crawler ucontr = UserController() users = [] try: users = [ucontr.get(user_id)] except: users = ucontr.read() finally: if users == []: users = ucontr.read() try: feed_id = int(feed_id) except: feed_id = None loop = asyncio.get_event_loop() for user in users: if user.is_active: logger.warn("Fetching articles for " + user.login) classic_crawler.retrieve_feed(loop, current_user, feed_id) loop.close()
def db_create(login='******', password='******'): "Will create the database from conf parameters." admin = {'is_admin': True, 'is_api': True, 'login': login, 'password': password} with application.app_context(): db.create_all() UserController(ignore_context=True).create(**admin)
def fetch_asyncio(user_id=None, feed_id=None): "Crawl the feeds with asyncio." import asyncio with application.app_context(): from crawler import default_crawler filters = {} filters['is_active'] = True filters['automatic_crawling'] = True if None is not user_id: filters['id'] = user_id users = UserController().read(**filters).all() try: feed_id = int(feed_id) except: feed_id = None loop = asyncio.get_event_loop() queue = asyncio.Queue(maxsize=3, loop=loop) producer_coro = default_crawler.retrieve_feed(queue, users, feed_id) consumer_coro = default_crawler.insert_articles(queue, 1) logger.info('Starting crawler.') start = datetime.now() loop.run_until_complete(asyncio.gather(producer_coro, consumer_coro)) end = datetime.now() loop.close() logger.info('Crawler finished in {} seconds.' \ .format((end - start).seconds))
def create_admin(nickname, password): "Will create an admin user." admin = {'is_admin': True, 'is_api': True, 'is_active': True, 'nickname': nickname, 'pwdhash': generate_password_hash(password)} with application.app_context(): UserController(ignore_context=True).create(**admin)
def db_create(): "Will create the database from conf parameters." admin = {'is_admin': True, 'is_api': True, 'is_active': True, 'nickname': 'admin', 'pwdhash': generate_password_hash( os.environ.get("ADMIN_PASSWORD", "password"))} with application.app_context(): db.create_all() UserController(ignore_context=True).create(**admin)
return request.accept_languages.best_match(conf.LANGUAGES.keys()) @babel.timezoneselector def get_timezone(): try: return conf.TIME_ZONE[get_locale()] except: return conf.TIME_ZONE["en"] # Jinja filters application.jinja_env.filters['month_name'] = lambda n: calendar.month_name[n] application.jinja_env.globals['conf'] = conf # Views with application.app_context(): from web import views application.register_blueprint(views.articles_bp) application.register_blueprint(views.article_bp) application.register_blueprint(views.feeds_bp) application.register_blueprint(views.feed_bp) application.register_blueprint(views.categories_bp) application.register_blueprint(views.category_bp) application.register_blueprint(views.icon_bp) application.register_blueprint(views.admin_bp) application.register_blueprint(views.users_bp) application.register_blueprint(views.user_bp) if __name__ == '__main__': # pragma: no cover application.run(host=conf.WEBSERVER_HOST,
def db_empty(): "Will drop every datas stocked in db." with application.app_context(): web.models.db_empty(db)
def db_create(): "Will create the database from conf parameters." with application.app_context(): populate_g() web.models.db_create(db)
babel = Babel(application) # Jinja filters def month_name(month_number): return calendar.month_name[month_number] application.jinja_env.filters['month_name'] = month_name application.jinja_env.filters['datetime'] = format_datetime application.jinja_env.globals['conf'] = conf # Views from flask_restful import Api from flask import g with application.app_context(): populate_g() g.api = Api(application, prefix='/api/v2.0') g.babel = babel from web import views application.register_blueprint(views.articles_bp) application.register_blueprint(views.article_bp) application.register_blueprint(views.feeds_bp) application.register_blueprint(views.feed_bp) application.register_blueprint(views.categories_bp) application.register_blueprint(views.category_bp) application.register_blueprint(views.icon_bp) application.register_blueprint(views.admin_bp) application.register_blueprint(views.users_bp) application.register_blueprint(views.user_bp)