def warm_cache(): # pragma: no cover
"""Background job to warm cache."""
from pybossa.core import create_app
app = create_app(run_as_server=False)
# Cache 3 pages
projects_cached = []
pages = range(1, 4)
import pybossa.cache.projects as cached_projects
import pybossa.cache.categories as cached_cat
import pybossa.cache.users as cached_users
import pybossa.cache.project_stats as stats
def warm_project(_id, short_name, featured=False):
if _id not in projects_cached:
cached_projects.get_project(short_name)
cached_projects.n_tasks(_id)
n_task_runs = cached_projects.n_task_runs(_id)
cached_projects.overall_progress(_id)
cached_projects.last_activity(_id)
cached_projects.n_completed_tasks(_id)
cached_projects.n_volunteers(_id)
if n_task_runs >= 1000 or featured:
# print ("Getting stats for %s as it has %s task runs" %
# (short_name, n_task_runs))
stats.get_stats(_id, app.config.get('GEO'))
projects_cached.append(_id)
# Cache top projects
projects = cached_projects.get_top()
for p in projects:
warm_project(p['id'], p['short_name'])
for page in pages:
projects = cached_projects.get_featured('featured', page,
app.config['APPS_PER_PAGE'])
for p in projects:
warm_project(p['id'], p['short_name'], featured=True)
# Categories
categories = cached_cat.get_used()
for c in categories:
for page in pages:
projects = cached_projects.get(c['short_name'], page,
app.config['APPS_PER_PAGE'])
for p in projects:
warm_project(p['id'], p['short_name'])
# Users
users = cached_users.get_leaderboard(app.config['LEADERBOARD'],
'anonymous')
for user in users:
# print "Getting stats for %s" % user['name']
cached_users.get_user_summary(user['name'])
cached_users.projects_contributed_cached(user['id'])
cached_users.published_projects_cached(user['id'])
cached_users.draft_projects_cached(user['id'])
cached_users.get_top()
return True
def warm_cache(): # pragma: no cover
"""Background job to warm cache."""
from pybossa.core import create_app
app = create_app(run_as_server=False)
projects_cached = []
import pybossa.cache.projects as cached_projects
import pybossa.cache.categories as cached_cat
import pybossa.cache.users as cached_users
import pybossa.cache.project_stats as stats
from pybossa.util import rank
from pybossa.core import user_repo
def warm_project(_id, short_name, featured=False):
if _id not in projects_cached:
#cached_projects.get_project(short_name)
#cached_projects.n_tasks(_id)
#n_task_runs = cached_projects.n_task_runs(_id)
#cached_projects.overall_progress(_id)
#cached_projects.last_activity(_id)
#cached_projects.n_completed_tasks(_id)
#cached_projects.n_volunteers(_id)
#cached_projects.browse_tasks(_id)
#if n_task_runs >= 1000 or featured:
# # print ("Getting stats for %s as it has %s task runs" %
# # (short_name, n_task_runs))
stats.update_stats(_id, app.config.get('GEO'))
projects_cached.append(_id)
# Cache top projects
projects = cached_projects.get_top()
for p in projects:
warm_project(p['id'], p['short_name'])
# Cache 3 pages
to_cache = 3 * app.config['APPS_PER_PAGE']
projects = rank(cached_projects.get_all_featured('featured'))[:to_cache]
for p in projects:
warm_project(p['id'], p['short_name'], featured=True)
# Categories
categories = cached_cat.get_used()
for c in categories:
projects = rank(cached_projects.get_all(c['short_name']))[:to_cache]
for p in projects:
warm_project(p['id'], p['short_name'])
# Users
users = cached_users.get_leaderboard(app.config['LEADERBOARD'])
for user in users:
# print "Getting stats for %s" % user['name']
print user_repo
u = user_repo.get_by_name(user['name'])
cached_users.get_user_summary(user['name'])
cached_users.projects_contributed_cached(u.id)
cached_users.published_projects_cached(u.id)
cached_users.draft_projects_cached(u.id)
return True
def warm_cache(): # pragma: no cover
"""Background job to warm cache."""
from pybossa.core import create_app
app = create_app(run_as_server=False)
projects_cached = []
import pybossa.cache.projects as cached_projects
import pybossa.cache.categories as cached_cat
import pybossa.cache.users as cached_users
import pybossa.cache.project_stats as stats
from pybossa.util import rank
from pybossa.core import user_repo
def warm_project(_id, short_name, featured=False):
if _id not in projects_cached:
#cached_projects.get_project(short_name)
#cached_projects.n_tasks(_id)
#n_task_runs = cached_projects.n_task_runs(_id)
#cached_projects.overall_progress(_id)
#cached_projects.last_activity(_id)
#cached_projects.n_completed_tasks(_id)
#cached_projects.n_volunteers(_id)
#cached_projects.browse_tasks(_id)
#if n_task_runs >= 1000 or featured:
# # print ("Getting stats for %s as it has %s task runs" %
# # (short_name, n_task_runs))
stats.update_stats(_id, app.config.get('GEO'))
projects_cached.append(_id)
# Cache top projects
projects = cached_projects.get_top()
for p in projects:
warm_project(p['id'], p['short_name'])
# Cache 3 pages
to_cache = 3 * app.config['APPS_PER_PAGE']
projects = rank(cached_projects.get_all_featured('featured'))[:to_cache]
for p in projects:
warm_project(p['id'], p['short_name'], featured=True)
# Categories
categories = cached_cat.get_used()
for c in categories:
projects = rank(cached_projects.get_all(c['short_name']))[:to_cache]
for p in projects:
warm_project(p['id'], p['short_name'])
# Users
users = cached_users.get_leaderboard(app.config['LEADERBOARD'])
for user in users:
# print "Getting stats for %s" % user['name']
print user_repo
u = user_repo.get_by_name(user['name'])
cached_users.get_user_summary(user['name'])
cached_users.projects_contributed_cached(u.id)
cached_users.published_projects_cached(u.id)
cached_users.draft_projects_cached(u.id)
return True
class TestIsReservedName(object):
from flask import Flask
from pybossa.core import setup_blueprints, create_app
app = create_app(run_as_server=False)
def test_returns_true_for_reserved_name_for_app_blueprint(self):
with self.app.app_context():
reserved = util.is_reserved_name('project', 'new')
assert reserved is True, reserved
reserved = util.is_reserved_name('project', 'category')
assert reserved is True, reserved
reserved = util.is_reserved_name('project', 'page')
assert reserved is True, reserved
reserved = util.is_reserved_name('project', 'draft')
assert reserved is True, reserved
reserved = util.is_reserved_name('project', 'published')
assert reserved is True, reserved
def test_returns_false_for_valid_name_for_app_blueprint(self):
with self.app.app_context():
reserved = util.is_reserved_name('project', 'test_project')
assert reserved is False, reserved
reserved = util.is_reserved_name('project', 'newProject')
assert reserved is False, reserved
def test_returns_true_for_reserved_name_for_account_blueprint(self):
with self.app.app_context():
reserved = util.is_reserved_name('account', 'register')
assert reserved is True, reserved
reserved = util.is_reserved_name('account', 'forgot-password')
assert reserved is True, reserved
reserved = util.is_reserved_name('account', 'profile')
assert reserved is True, reserved
reserved = util.is_reserved_name('account', 'signin')
assert reserved is True, reserved
reserved = util.is_reserved_name('account', 'reset-password')
assert reserved is True, reserved
def test_returns_false_for_valid_name_for_account_blueprint(self):
with self.app.app_context():
reserved = util.is_reserved_name('account', 'fulanito')
assert reserved is False, reserved
reserved = util.is_reserved_name('acount', 'profileFulanito')
assert reserved is False, reserved
def test_returns_false_for_empty_name_string(self):
with self.app.app_context():
reserved = util.is_reserved_name('account', '')
assert reserved is False, reserved
def warm_cache(): # pragma: no cover
"""Background job to warm cache."""
from pybossa.core import create_app
app = create_app(run_as_server=False)
# Cache 3 pages
apps_cached = []
pages = range(1, 4)
import pybossa.cache.apps as cached_apps
import pybossa.cache.categories as cached_cat
import pybossa.cache.users as cached_users
import pybossa.cache.project_stats as stats
def warm_app(id, short_name, featured=False):
if id not in apps_cached:
cached_apps.get_app(short_name)
cached_apps.n_tasks(id)
n_task_runs = cached_apps.n_task_runs(id)
cached_apps.overall_progress(id)
cached_apps.last_activity(id)
cached_apps.n_completed_tasks(id)
cached_apps.n_volunteers(id)
if n_task_runs >= 1000 or featured:
print ("Getting stats for %s as it has %s task runs" %
(short_name, n_task_runs))
stats.get_stats(id, app.config.get('GEO'))
apps_cached.append(id)
# Cache top projects
apps = cached_apps.get_top()
for a in apps:
warm_app(a['id'], a['short_name'])
for page in pages:
apps = cached_apps.get_featured('featured', page,
app.config['APPS_PER_PAGE'])
for a in apps:
warm_app(a['id'], a['short_name'], featured=True)
# Categories
categories = cached_cat.get_used()
for c in categories:
for page in pages:
apps = cached_apps.get(c['short_name'],
page,
app.config['APPS_PER_PAGE'])
for a in apps:
warm_app(a['id'], a['short_name'])
# Users
cached_users.get_leaderboard(app.config['LEADERBOARD'], 'anonymous')
cached_users.get_top()
return True
def main():
app = create_app(run_as_server=False)
with app.app_context():
def timer_function():
#Continue code here
image_source = sentinel.slave.lpop("GEOTAGX-SOURCERER-QUEUE")
if image_source:
split_image_source = image_source.split(DELIMITER)
base64Data = split_image_source[-1]
decodedJSONString = base64.b64decode(base64Data)
parsedJSONObject = json.loads(decodedJSONString)
if parsedJSONObject['source'] == 'geotagx-chrome-sourcerer':
#Handle Chrome Sourcerer
#TODO : Refactor into an OOP based implementation
SOURCE_URI = parsedJSONObject['source_uri']
IMAGE_URL = parsedJSONObject['image_url']
for category in parsedJSONObject['categories']:
category_objects = Category.query.filter(
Category.short_name == category)
for category_object in category_objects:
related_projects = Project.query.filter(
Project.category == category_object)
for related_project in related_projects:
# Start building Task Object
_task_object = Task()
_task_object.project_id = related_project.id
# Build Info Object from whatever data we have
_info_object = {}
_info_object['image_url'] = IMAGE_URL
_info_object['source_uri'] = SOURCE_URI
_info_object['id'] = parsedJSONObject['source'] + "_" + \
''.join(random.choice('0123456789ABCDEF') for i in range(16))
_task_object.info = _info_object
db.session.add(_task_object)
db.session.commit()
print _now(), _task_object
else:
print _now(), "GEOTAGX-SOURCERER-QUEUE Empty....."
while True:
timer_function()
time.sleep(TIME_DELAY)
def main():
app = create_app(run_as_server=False)
with app.app_context():
def timer_function():
#Continue code here
image_source = sentinel.slave.lpop("GEOTAGX-SOURCERER-QUEUE")
if image_source:
split_image_source = image_source.split(DELIMITER)
base64Data = split_image_source[-1]
decodedJSONString = base64.b64decode(base64Data)
parsedJSONObject = json.loads(decodedJSONString)
if parsedJSONObject['source'] == 'geotagx-chrome-sourcerer':
#Handle Chrome Sourcerer
#TODO : Refactor into an OOP based implementation
SOURCE_URI = parsedJSONObject['source_uri']
IMAGE_URL = parsedJSONObject['image_url']
for category in parsedJSONObject['categories']:
category_objects = Category.query.filter(Category.short_name == category)
for category_object in category_objects:
related_projects = Project.query.filter(Project.category == category_object)
for related_project in related_projects:
# Start building Task Object
_task_object = Task()
_task_object.project_id = related_project.id
# Build Info Object from whatever data we have
_info_object = {}
_info_object['image_url'] = IMAGE_URL
_info_object['source_uri'] = SOURCE_URI
_info_object['id'] = parsedJSONObject['source'] + "_" + \
''.join(random.choice('0123456789ABCDEF') for i in range(16))
_task_object.info = _info_object
db.session.add(_task_object)
db.session.commit()
print _now(), _task_object
else:
print _now(), "GEOTAGX-SOURCERER-QUEUE Empty....."
while True:
timer_function()
time.sleep(TIME_DELAY)
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PyBossa. If not, see <http://www.gnu.org/licenses/>.
#!/usr/bin/env python
import os
import sys
import optparse
import inspect
#import pybossa.model as model
from pybossa.core import create_app
app = create_app()
def warm_cache():
'''Warm cache'''
# Disable cache, so we can refresh the data in Redis
os.environ['PYBOSSA_REDIS_CACHE_DISABLED'] = '1'
# Cache 3 pages
apps_cached = []
pages = range(1, 4)
with app.app_context():
import pybossa.cache.projects as cached_apps
import pybossa.cache.categories as cached_cat
import pybossa.cache.users as cached_users
import pybossa.cache.project_stats as stats
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PyBossa. If not, see <http://www.gnu.org/licenses/>.
from pybossa.model import db, rebuild_db
from pybossa.core import create_app, sentinel
from pybossa.model.app import App
from pybossa.model.category import Category
from pybossa.model.task import Task
from pybossa.model.task_run import TaskRun
from pybossa.model.user import User
from functools import wraps
import random
flask_app = create_app()
def with_context(f):
@wraps(f)
def decorated_function(*args, **kwargs):
with flask_app.app_context():
return f(*args, **kwargs)
return decorated_function
class Test(object):
def setUp(self):
self.flask_app = flask_app
self.app = flask_app.test_client()
with self.flask_app.app_context():
rebuild_db()
import sys
import optparse
import inspect
#import pybossa.model as model
from pybossa.core import db, create_app
from pybossa.model.app import App
from pybossa.model.user import User
from pybossa.model.category import Category
from alembic.config import Config
from alembic import command
from html2text import html2text
from sqlalchemy.sql import text
app = create_app()
def setup_alembic_config():
if "DATABASE_URL" not in os.environ:
alembic_cfg = Config("alembic.ini")
else:
dynamic_filename = "alembic-heroku.ini"
with file("alembic.ini.template") as f:
with file(dynamic_filename, "w") as conf:
for line in f.readlines():
if line.startswith("sqlalchemy.url"):
conf.write("sqlalchemy.url = %s\n" %
os.environ['DATABASE_URL'])
else:
conf.write(line)
from pybossa.core import create_app, sentinel
from pybossa.model.project import Project
from pybossa.model.category import Category
from pybossa.model.task import Task
from pybossa.model.task_run import TaskRun
from pybossa.model.user import User
import pybossa.model as model
from functools import wraps
from factories import reset_all_pk_sequences
import random
import os
os.environ['PYBOSSA_SETTINGS'] = '../settings_test.py'
os.environ['PYBOSSA_REDIS_CACHE_DISABLED'] = '1'
flask_app = create_app(run_as_server=False)
def with_context(f):
@wraps(f)
def decorated_function(*args, **kwargs):
with flask_app.app_context():
return f(*args, **kwargs)
return decorated_function
def delete_materialized_views():
"""Delete materialized views."""
sql = text('''SELECT relname
FROM pg_class WHERE relname LIKE '%dashboard%';''')
def setUp(self):
self.flask_app = create_app()
self.app = self.flask_app.test_client()
self.db = db
with self.flask_app.app_context():
model.rebuild_db()
def setUp(self):
self.flask_app = create_app()
self.app = self.flask_app.test_client()
self.db = db
with self.flask_app.app_context():
model.rebuild_db()
def make_shell():
return dict(app=create_app(),
model=model,
db_session=db.session)
from pybossa.core import create_app, sentinel
from pybossa.model.app import App
from pybossa.model.category import Category
from pybossa.model.task import Task
from pybossa.model.task_run import TaskRun
from pybossa.model.user import User
import pybossa.model as model
from functools import wraps
from factories import reset_all_pk_sequences
import random
import os
os.environ['PYBOSSA_SETTINGS'] = '../settings_test.py'
os.environ['PYBOSSA_REDIS_CACHE_DISABLED'] = '1'
flask_app = create_app()
def with_context(f):
@wraps(f)
def decorated_function(*args, **kwargs):
with flask_app.app_context():
return f(*args, **kwargs)
return decorated_function
def rebuild_db():
"""Rebuild the DB."""
db.drop_all()
db.create_all()
def make_shell():
return dict(app=create_app(), model=model, db_session=db.session)
#
# PyBossa is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyBossa is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PyBossa. If not, see <http://www.gnu.org/licenses/>.
#!/usr/bin/env python
import sys
from rq import Queue, Connection, Worker
from pybossa.core import create_app, sentinel
app = create_app(run_as_server=False)
# Provide queue names to listen to as arguments to this script,
# similar to rqworker
with app.app_context():
with Connection(sentinel.master):
qs = map(Queue, sys.argv[1:]) or [Queue()]
w = Worker(qs)
w.work()
"""Initial database configuration.
Note:
This file does roughly the same as running ``python cli.py db_create``, but
does't require reconfiguring the connection credentials for alembic
"""
from pybossa.core import create_app, db
from pybossa.model.category import Category
with create_app(run_as_server=False).app_context():
category = Category(name="Main", short_name="main", description="")
db.create_all()
if not db.session.query(Category).count():
db.session.add(category)
db.session.commit()