def warm_cache(): # pragma: no cover """Background job to warm cache.""" from pybossa.core import create_app app = create_app(run_as_server=False) # Cache 3 pages projects_cached = [] pages = range(1, 4) import pybossa.cache.projects as cached_projects import pybossa.cache.categories as cached_cat import pybossa.cache.users as cached_users import pybossa.cache.project_stats as stats def warm_project(_id, short_name, featured=False): if _id not in projects_cached: cached_projects.get_project(short_name) cached_projects.n_tasks(_id) n_task_runs = cached_projects.n_task_runs(_id) cached_projects.overall_progress(_id) cached_projects.last_activity(_id) cached_projects.n_completed_tasks(_id) cached_projects.n_volunteers(_id) if n_task_runs >= 1000 or featured: # print ("Getting stats for %s as it has %s task runs" % # (short_name, n_task_runs)) stats.get_stats(_id, app.config.get('GEO')) projects_cached.append(_id) # Cache top projects projects = cached_projects.get_top() for p in projects: warm_project(p['id'], p['short_name']) for page in pages: projects = cached_projects.get_featured('featured', page, app.config['APPS_PER_PAGE']) for p in projects: warm_project(p['id'], p['short_name'], featured=True) # Categories categories = cached_cat.get_used() for c in categories: for page in pages: projects = cached_projects.get(c['short_name'], page, app.config['APPS_PER_PAGE']) for p in projects: warm_project(p['id'], p['short_name']) # Users users = cached_users.get_leaderboard(app.config['LEADERBOARD'], 'anonymous') for user in users: # print "Getting stats for %s" % user['name'] cached_users.get_user_summary(user['name']) cached_users.projects_contributed_cached(user['id']) cached_users.published_projects_cached(user['id']) cached_users.draft_projects_cached(user['id']) cached_users.get_top() return True
def warm_cache(): # pragma: no cover """Background job to warm cache.""" from pybossa.core import create_app app = create_app(run_as_server=False) projects_cached = [] import pybossa.cache.projects as cached_projects import pybossa.cache.categories as cached_cat import pybossa.cache.users as cached_users import pybossa.cache.project_stats as stats from pybossa.util import rank from pybossa.core import user_repo def warm_project(_id, short_name, featured=False): if _id not in projects_cached: #cached_projects.get_project(short_name) #cached_projects.n_tasks(_id) #n_task_runs = cached_projects.n_task_runs(_id) #cached_projects.overall_progress(_id) #cached_projects.last_activity(_id) #cached_projects.n_completed_tasks(_id) #cached_projects.n_volunteers(_id) #cached_projects.browse_tasks(_id) #if n_task_runs >= 1000 or featured: # # print ("Getting stats for %s as it has %s task runs" % # # (short_name, n_task_runs)) stats.update_stats(_id, app.config.get('GEO')) projects_cached.append(_id) # Cache top projects projects = cached_projects.get_top() for p in projects: warm_project(p['id'], p['short_name']) # Cache 3 pages to_cache = 3 * app.config['APPS_PER_PAGE'] projects = rank(cached_projects.get_all_featured('featured'))[:to_cache] for p in projects: warm_project(p['id'], p['short_name'], featured=True) # Categories categories = cached_cat.get_used() for c in categories: projects = rank(cached_projects.get_all(c['short_name']))[:to_cache] for p in projects: warm_project(p['id'], p['short_name']) # Users users = cached_users.get_leaderboard(app.config['LEADERBOARD']) for user in users: # print "Getting stats for %s" % user['name'] print user_repo u = user_repo.get_by_name(user['name']) cached_users.get_user_summary(user['name']) cached_users.projects_contributed_cached(u.id) cached_users.published_projects_cached(u.id) cached_users.draft_projects_cached(u.id) return True
class TestIsReservedName(object): from flask import Flask from pybossa.core import setup_blueprints, create_app app = create_app(run_as_server=False) def test_returns_true_for_reserved_name_for_app_blueprint(self): with self.app.app_context(): reserved = util.is_reserved_name('project', 'new') assert reserved is True, reserved reserved = util.is_reserved_name('project', 'category') assert reserved is True, reserved reserved = util.is_reserved_name('project', 'page') assert reserved is True, reserved reserved = util.is_reserved_name('project', 'draft') assert reserved is True, reserved reserved = util.is_reserved_name('project', 'published') assert reserved is True, reserved def test_returns_false_for_valid_name_for_app_blueprint(self): with self.app.app_context(): reserved = util.is_reserved_name('project', 'test_project') assert reserved is False, reserved reserved = util.is_reserved_name('project', 'newProject') assert reserved is False, reserved def test_returns_true_for_reserved_name_for_account_blueprint(self): with self.app.app_context(): reserved = util.is_reserved_name('account', 'register') assert reserved is True, reserved reserved = util.is_reserved_name('account', 'forgot-password') assert reserved is True, reserved reserved = util.is_reserved_name('account', 'profile') assert reserved is True, reserved reserved = util.is_reserved_name('account', 'signin') assert reserved is True, reserved reserved = util.is_reserved_name('account', 'reset-password') assert reserved is True, reserved def test_returns_false_for_valid_name_for_account_blueprint(self): with self.app.app_context(): reserved = util.is_reserved_name('account', 'fulanito') assert reserved is False, reserved reserved = util.is_reserved_name('acount', 'profileFulanito') assert reserved is False, reserved def test_returns_false_for_empty_name_string(self): with self.app.app_context(): reserved = util.is_reserved_name('account', '') assert reserved is False, reserved
def warm_cache(): # pragma: no cover """Background job to warm cache.""" from pybossa.core import create_app app = create_app(run_as_server=False) # Cache 3 pages apps_cached = [] pages = range(1, 4) import pybossa.cache.apps as cached_apps import pybossa.cache.categories as cached_cat import pybossa.cache.users as cached_users import pybossa.cache.project_stats as stats def warm_app(id, short_name, featured=False): if id not in apps_cached: cached_apps.get_app(short_name) cached_apps.n_tasks(id) n_task_runs = cached_apps.n_task_runs(id) cached_apps.overall_progress(id) cached_apps.last_activity(id) cached_apps.n_completed_tasks(id) cached_apps.n_volunteers(id) if n_task_runs >= 1000 or featured: print ("Getting stats for %s as it has %s task runs" % (short_name, n_task_runs)) stats.get_stats(id, app.config.get('GEO')) apps_cached.append(id) # Cache top projects apps = cached_apps.get_top() for a in apps: warm_app(a['id'], a['short_name']) for page in pages: apps = cached_apps.get_featured('featured', page, app.config['APPS_PER_PAGE']) for a in apps: warm_app(a['id'], a['short_name'], featured=True) # Categories categories = cached_cat.get_used() for c in categories: for page in pages: apps = cached_apps.get(c['short_name'], page, app.config['APPS_PER_PAGE']) for a in apps: warm_app(a['id'], a['short_name']) # Users cached_users.get_leaderboard(app.config['LEADERBOARD'], 'anonymous') cached_users.get_top() return True
def main(): app = create_app(run_as_server=False) with app.app_context(): def timer_function(): #Continue code here image_source = sentinel.slave.lpop("GEOTAGX-SOURCERER-QUEUE") if image_source: split_image_source = image_source.split(DELIMITER) base64Data = split_image_source[-1] decodedJSONString = base64.b64decode(base64Data) parsedJSONObject = json.loads(decodedJSONString) if parsedJSONObject['source'] == 'geotagx-chrome-sourcerer': #Handle Chrome Sourcerer #TODO : Refactor into an OOP based implementation SOURCE_URI = parsedJSONObject['source_uri'] IMAGE_URL = parsedJSONObject['image_url'] for category in parsedJSONObject['categories']: category_objects = Category.query.filter( Category.short_name == category) for category_object in category_objects: related_projects = Project.query.filter( Project.category == category_object) for related_project in related_projects: # Start building Task Object _task_object = Task() _task_object.project_id = related_project.id # Build Info Object from whatever data we have _info_object = {} _info_object['image_url'] = IMAGE_URL _info_object['source_uri'] = SOURCE_URI _info_object['id'] = parsedJSONObject['source'] + "_" + \ ''.join(random.choice('0123456789ABCDEF') for i in range(16)) _task_object.info = _info_object db.session.add(_task_object) db.session.commit() print _now(), _task_object else: print _now(), "GEOTAGX-SOURCERER-QUEUE Empty....." while True: timer_function() time.sleep(TIME_DELAY)
def main(): app = create_app(run_as_server=False) with app.app_context(): def timer_function(): #Continue code here image_source = sentinel.slave.lpop("GEOTAGX-SOURCERER-QUEUE") if image_source: split_image_source = image_source.split(DELIMITER) base64Data = split_image_source[-1] decodedJSONString = base64.b64decode(base64Data) parsedJSONObject = json.loads(decodedJSONString) if parsedJSONObject['source'] == 'geotagx-chrome-sourcerer': #Handle Chrome Sourcerer #TODO : Refactor into an OOP based implementation SOURCE_URI = parsedJSONObject['source_uri'] IMAGE_URL = parsedJSONObject['image_url'] for category in parsedJSONObject['categories']: category_objects = Category.query.filter(Category.short_name == category) for category_object in category_objects: related_projects = Project.query.filter(Project.category == category_object) for related_project in related_projects: # Start building Task Object _task_object = Task() _task_object.project_id = related_project.id # Build Info Object from whatever data we have _info_object = {} _info_object['image_url'] = IMAGE_URL _info_object['source_uri'] = SOURCE_URI _info_object['id'] = parsedJSONObject['source'] + "_" + \ ''.join(random.choice('0123456789ABCDEF') for i in range(16)) _task_object.info = _info_object db.session.add(_task_object) db.session.commit() print _now(), _task_object else: print _now(), "GEOTAGX-SOURCERER-QUEUE Empty....." while True: timer_function() time.sleep(TIME_DELAY)
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with PyBossa. If not, see <http://www.gnu.org/licenses/>. #!/usr/bin/env python import os import sys import optparse import inspect #import pybossa.model as model from pybossa.core import create_app app = create_app() def warm_cache(): '''Warm cache''' # Disable cache, so we can refresh the data in Redis os.environ['PYBOSSA_REDIS_CACHE_DISABLED'] = '1' # Cache 3 pages apps_cached = [] pages = range(1, 4) with app.app_context(): import pybossa.cache.projects as cached_apps import pybossa.cache.categories as cached_cat import pybossa.cache.users as cached_users import pybossa.cache.project_stats as stats
# GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with PyBossa. If not, see <http://www.gnu.org/licenses/>. from pybossa.model import db, rebuild_db from pybossa.core import create_app, sentinel from pybossa.model.app import App from pybossa.model.category import Category from pybossa.model.task import Task from pybossa.model.task_run import TaskRun from pybossa.model.user import User from functools import wraps import random flask_app = create_app() def with_context(f): @wraps(f) def decorated_function(*args, **kwargs): with flask_app.app_context(): return f(*args, **kwargs) return decorated_function class Test(object): def setUp(self): self.flask_app = flask_app self.app = flask_app.test_client() with self.flask_app.app_context(): rebuild_db()
import sys import optparse import inspect #import pybossa.model as model from pybossa.core import db, create_app from pybossa.model.app import App from pybossa.model.user import User from pybossa.model.category import Category from alembic.config import Config from alembic import command from html2text import html2text from sqlalchemy.sql import text app = create_app() def setup_alembic_config(): if "DATABASE_URL" not in os.environ: alembic_cfg = Config("alembic.ini") else: dynamic_filename = "alembic-heroku.ini" with file("alembic.ini.template") as f: with file(dynamic_filename, "w") as conf: for line in f.readlines(): if line.startswith("sqlalchemy.url"): conf.write("sqlalchemy.url = %s\n" % os.environ['DATABASE_URL']) else: conf.write(line)
from pybossa.core import create_app, sentinel from pybossa.model.project import Project from pybossa.model.category import Category from pybossa.model.task import Task from pybossa.model.task_run import TaskRun from pybossa.model.user import User import pybossa.model as model from functools import wraps from factories import reset_all_pk_sequences import random import os os.environ['PYBOSSA_SETTINGS'] = '../settings_test.py' os.environ['PYBOSSA_REDIS_CACHE_DISABLED'] = '1' flask_app = create_app(run_as_server=False) def with_context(f): @wraps(f) def decorated_function(*args, **kwargs): with flask_app.app_context(): return f(*args, **kwargs) return decorated_function def delete_materialized_views(): """Delete materialized views.""" sql = text('''SELECT relname FROM pg_class WHERE relname LIKE '%dashboard%';''')
def setUp(self): self.flask_app = create_app() self.app = self.flask_app.test_client() self.db = db with self.flask_app.app_context(): model.rebuild_db()
def make_shell(): return dict(app=create_app(), model=model, db_session=db.session)
from pybossa.core import create_app, sentinel from pybossa.model.app import App from pybossa.model.category import Category from pybossa.model.task import Task from pybossa.model.task_run import TaskRun from pybossa.model.user import User import pybossa.model as model from functools import wraps from factories import reset_all_pk_sequences import random import os os.environ['PYBOSSA_SETTINGS'] = '../settings_test.py' os.environ['PYBOSSA_REDIS_CACHE_DISABLED'] = '1' flask_app = create_app() def with_context(f): @wraps(f) def decorated_function(*args, **kwargs): with flask_app.app_context(): return f(*args, **kwargs) return decorated_function def rebuild_db(): """Rebuild the DB.""" db.drop_all() db.create_all()
# # PyBossa is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # PyBossa is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with PyBossa. If not, see <http://www.gnu.org/licenses/>. #!/usr/bin/env python import sys from rq import Queue, Connection, Worker from pybossa.core import create_app, sentinel app = create_app(run_as_server=False) # Provide queue names to listen to as arguments to this script, # similar to rqworker with app.app_context(): with Connection(sentinel.master): qs = map(Queue, sys.argv[1:]) or [Queue()] w = Worker(qs) w.work()
"""Initial database configuration. Note: This file does roughly the same as running ``python cli.py db_create``, but does't require reconfiguring the connection credentials for alembic """ from pybossa.core import create_app, db from pybossa.model.category import Category with create_app(run_as_server=False).app_context(): category = Category(name="Main", short_name="main", description="") db.create_all() if not db.session.query(Category).count(): db.session.add(category) db.session.commit()