def get_export_task_jobs(queue):
"""Export tasks to zip."""
from pybossa.core import project_repo
import pybossa.cache.projects as cached_projects
from pybossa.pro_features import ProFeatureHandler
feature_handler = ProFeatureHandler(current_app.config.get('PRO_FEATURES'))
timeout = current_app.config.get('TIMEOUT')
if feature_handler.only_for_pro('updated_exports'):
if queue == 'high':
projects = cached_projects.get_from_pro_user()
else:
projects = (p.dictize()
for p in project_repo.filter_by(published=True)
if p.owner.pro is False)
else:
projects = (p.dictize()
for p in project_repo.filter_by(published=True))
for project in projects:
project_id = project.get('id')
job = dict(name=project_export,
args=[project_id],
kwargs={},
timeout=timeout,
queue=queue)
yield job
def get_enhanced_volumes(category):
"""Return the categories volumes enhanced with project data."""
volumes = category.info.get('volumes', [])
projects = project_repo.filter_by(category_id=category.id)
for volume in volumes:
vol_projects = [
dict(id=p.id,
name=p.name,
short_name=p.short_name,
published=p.published,
overall_progress=overall_progress(p.id)) for p in projects
if p.info.get('volume_id') == volume['id']
]
completed_projects = [
p for p in vol_projects if p['overall_progress'] == 100
]
ongoing_projects = [
p for p in vol_projects
if p['published'] and p not in completed_projects
]
volume['projects'] = vol_projects
volume['n_completed_projects'] = len(completed_projects)
volume['n_ongoing_projects'] = len(ongoing_projects)
return volumes
def password_protect_hidden_projects():
import random
from pybossa.core import project_repo, user_repo, mail
from pybossa.jobs import enqueue_job, send_mail
def generate_random_password():
CHARS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'
password = ''
for i in range(8):
password += random.choice(CHARS)
return password
def generate_email_for(project_name, owner_name, password):
subject = "Changes in your hidden project %s" % project_name
content = (
"""
Dear %s,
We are writing you to let you know that, due to recent changes in Crowdcrafting,
hidden projects will soon no longer be supported. However, you can still
protect your project with a password, allowing only people with it to
access and contribute to it.
We have checked that your project %s is hidden. We don't want to expose it
to the public, so we have protected it with a password instead. The current
password for your project is:
%s
You will be able to change it on your project settings page.
You can find more information about passwords in the documentation
(http://docs.pybossa.com/user/tutorial/#protecting-the-project-with-a-password).
If you have any doubts, please contact us and we will be pleased to help you!
Best regards,
Crowdcrafting team.
""" % (owner_name, project_name, password))
return subject, content
with app.app_context():
for project in project_repo.filter_by(hidden=1):
password = generate_random_password()
subject, content = generate_email_for(project.name, project.owner.name, password)
message = dict(recipients=[project.owner.email_addr],
subject=subject,
body=content)
job = dict(name=send_mail,
args=[message],
kwargs={},
timeout=(600),
queue='medium')
enqueue_job(job)
project.set_password(password)
project_repo.save(project)
def password_protect_hidden_projects():
import random
from pybossa.core import project_repo, user_repo, mail
from pybossa.jobs import enqueue_job, send_mail
def generate_random_password():
CHARS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'
password = ''
for i in range(8):
password += random.choice(CHARS)
return password
def generate_email_for(project_name, owner_name, password):
subject = "Changes in your hidden project %s" % project_name
content = (
"""
Dear %s,
We are writing you to let you know that, due to recent changes in Crowdcrafting,
hidden projects will soon no longer be supported. However, you can still
protect your project with a password, allowing only people with it to
access and contribute to it.
We have checked that your project %s is hidden. We don't want to expose it
to the public, so we have protected it with a password instead. The current
password for your project is:
%s
You will be able to change it on your project settings page.
You can find more information about passwords in the documentation
(http://docs.pybossa.com/en/latest/user/tutorial.html#protecting-the-project-with-a-password).
If you have any doubts, please contact us and we will be pleased to help you!
Best regards,
Crowdcrafting team.
""" % (owner_name, project_name, password))
return subject, content
with app.app_context():
for project in project_repo.filter_by(hidden=1):
password = generate_random_password()
subject, content = generate_email_for(project.name, project.owner.name, password)
message = dict(recipients=[project.owner.email_addr],
subject=subject,
body=content)
job = dict(name=send_mail,
args=[message],
kwargs={},
timeout=(600),
queue='medium')
enqueue_job(job)
project.set_password(password)
project_repo.save(project)
def get_projects_with_unknown_volumes(category):
"""Return all projects not linked to a known volume."""
volume_ids = [vol['id'] for vol in category.info.get('volumes', [])]
projects = project_repo.filter_by(category_id=category.id)
return [
dict(id=p.id, name=p.name, short_name=p.short_name) for p in projects
if not p.info.get('volume_id')
or p.info.get('volume_id') not in volume_ids
]
def get_parent(parent_template_id, volume_id, category):
"""Return a valid parent project."""
projects = project_repo.filter_by(category_id=category.id)
try:
return [
p for p in projects
if p.info.get('template_id') == parent_template_id
and p.info.get('volume_id') == volume_id and validate_parent(p)
][0]
except IndexError:
return None
def index():
"""Return the Data page."""
projects = [p for p in project_repo.filter_by(published=True)
if not p.needs_password()]
title = "Data"
description = """Download open datasets of all crowdsourced data produced
via LibCrowds."""
display = {'tasks': current_app.config['DATA_DISPLAY_TASKS'],
'task_runs': current_app.config['DATA_DISPLAY_TASK_RUNS'],
'results': current_app.config['DATA_DISPLAY_RESULTS'],
'flickr': current_app.config['DATA_DISPLAY_FLICKR']}
return render_template('/index.html', projects=projects, display=display)
def get_export_task_jobs(queue):
"""Export tasks to zip."""
from pybossa.core import project_repo
import pybossa.cache.projects as cached_projects
from pybossa.pro_features import ProFeatureHandler
feature_handler = ProFeatureHandler(current_app.config.get('PRO_FEATURES'))
timeout = current_app.config.get('TIMEOUT')
if feature_handler.only_for_pro('updated_exports'):
if queue == 'high':
projects = cached_projects.get_from_pro_user()
else:
projects = (p.dictize() for p in project_repo.filter_by(published=True)
if p.owner.pro is False)
else:
projects = (p.dictize() for p in project_repo.filter_by(published=True))
for project in projects:
project_id = project.get('id')
job = dict(name=project_export,
args=[project_id], kwargs={},
timeout=timeout,
queue=queue)
yield job
def export_userdata(user_id, **kwargs):
from pybossa.core import user_repo, project_repo, task_repo, result_repo
from flask import current_app, url_for
json_exporter = JsonExporter()
user = user_repo.get(user_id)
user_data = user.dictize()
del user_data['passwd_hash']
projects = project_repo.filter_by(owner_id=user.id)
projects_data = [project.dictize() for project in projects]
taskruns = task_repo.filter_task_runs_by(user_id=user.id)
taskruns_data = [tr.dictize() for tr in taskruns]
pdf = json_exporter._make_zip(None, '', 'personal_data', user_data, user_id,
'personal_data.zip')
upf = json_exporter._make_zip(None, '', 'user_projects', projects_data, user_id,
'user_projects.zip')
ucf = json_exporter._make_zip(None, '', 'user_contributions', taskruns_data, user_id,
'user_contributions.zip')
upload_method = current_app.config.get('UPLOAD_METHOD')
if upload_method == 'local':
upload_method = 'uploads.uploaded_file'
personal_data_link = url_for(upload_method,
filename="user_%s/%s" % (user_id, pdf))
personal_projects_link = url_for(upload_method,
filename="user_%s/%s" % (user_id,
upf))
personal_contributions_link = url_for(upload_method,
filename="user_%s/%s" % (user_id,
ucf))
body = render_template('/account/email/exportdata.md',
user=user.dictize(),
personal_data_link=personal_data_link,
personal_projects_link=personal_projects_link,
personal_contributions_link=personal_contributions_link,
config=current_app.config)
html = render_template('/account/email/exportdata.html',
user=user.dictize(),
personal_data_link=personal_data_link,
personal_projects_link=personal_projects_link,
personal_contributions_link=personal_contributions_link,
config=current_app.config)
subject = 'Your personal data'
mail_dict = dict(recipients=[user.email_addr],
subject=subject,
body=body,
html=html)
send_mail(mail_dict)
def analyse_empty_results(category_id):
"""Analyse empty results for a category."""
category = project_repo.get_category(category_id)
if not category:
abort(404)
if request.method == 'POST':
presenter = category.info.get('presenter')
projects = project_repo.filter_by(category_id=category.id)
for project in projects:
analyse_empty(project.id, presenter)
flash('Analysis of empty results queued', 'success')
csrf = None
else:
csrf = generate_csrf()
response = dict(csrf=csrf)
return handle_content_type(response)
def update_project_stats():
"""Update project stats for draft projects."""
from pybossa.core import db
from pybossa.core import project_repo, task_repo, result_repo
from pybossa.model.task import Task
from pybossa.model.task_run import TaskRun
from pybossa.model.counter import Counter
import pybossa.cache.project_stats as stats
projects = project_repo.filter_by(published=False)
for project in projects:
print "Working on project: %s" % project.short_name
sql_query = """INSERT INTO project_stats
(project_id, n_tasks, n_task_runs, n_results, n_volunteers,
n_completed_tasks, overall_progress, average_time,
n_blogposts, last_activity, info)
VALUES (%s, 0, 0, 0, 0, 0, 0, 0, 0, 0, '{}');""" % (project.id)
db.engine.execute(sql_query)
def update_project_stats():
"""Update project stats for draft projects."""
from pybossa.core import db
from pybossa.core import project_repo, task_repo, result_repo
from pybossa.model.task import Task
from pybossa.model.task_run import TaskRun
from pybossa.model.counter import Counter
import pybossa.cache.project_stats as stats
projects = project_repo.filter_by(published=False)
for project in projects:
print "Working on project: %s" % project.short_name
sql_query = """INSERT INTO project_stats
(project_id, n_tasks, n_task_runs, n_results, n_volunteers,
n_completed_tasks, overall_progress, average_time,
n_blogposts, last_activity, info)
VALUES (%s, 0, 0, 0, 0, 0, 0, 0, 0, 0, '{}');""" % (project.id)
db.engine.execute(sql_query)
def get_project_jobs(queue):
"""Return a list of jobs based on user type."""
from pybossa.core import project_repo
from pybossa.cache import projects as cached_projects
timeout = current_app.config.get('TIMEOUT')
if queue == 'super':
projects = cached_projects.get_from_pro_user()
elif queue == 'high':
projects = (p.dictize() for p in project_repo.filter_by(published=True)
if p.owner.pro is False)
else:
projects = []
for project in projects:
project_id = project.get('id')
project_short_name = project.get('short_name')
job = dict(name=get_project_stats,
args=[project_id, project_short_name], kwargs={},
timeout=timeout,
queue=queue)
yield job
def get_project_jobs(queue):
"""Return a list of jobs based on user type."""
from pybossa.core import project_repo
from pybossa.cache import projects as cached_projects
timeout = current_app.config.get('TIMEOUT')
if queue == 'super':
projects = cached_projects.get_from_pro_user()
elif queue == 'high':
projects = (p.dictize() for p in project_repo.filter_by(published=True)
if p.owner.pro is False)
else:
projects = []
for project in projects:
project_id = project.get('id')
project_short_name = project.get('short_name')
job = dict(name=get_project_stats,
args=[project_id, project_short_name], kwargs={},
timeout=timeout,
queue=queue)
yield job
def project_filters(short_name):
"""Return all filters currently associated with the category's projects."""
category = project_repo.get_category_by(short_name=short_name)
if not category: # pragma: no cover
abort(404)
projects = project_repo.filter_by(category_id=category.id)
all_filters = [(k, v) for project in projects
for k, v in project.info.get('filters', {}).items()]
filters = {}
for _filter in all_filters:
key = _filter[0]
value = _filter[1]
filter_opts = filters.get(key, [])
filter_opts.append(value)
filter_opts = list(set(filter_opts))
filters[key] = filter_opts
response = dict(filters=filters)
return handle_content_type(response)
def create_results():
"""Create results when migrating."""
from pybossa.core import project_repo, task_repo, result_repo
from pybossa.model.result import Result
projects = project_repo.filter_by(published=True)
for project in projects:
print "Working on project: %s" % project.short_name
tasks = task_repo.filter_tasks_by(state='completed',
project_id=project.id)
print "Analyzing %s tasks" % len(tasks)
for task in tasks:
result = result_repo.get_by(project_id=project.id, task_id=task.id)
if result is None:
result = Result(project_id=project.id,
task_id=task.id,
task_run_ids=[tr.id for tr in task.task_runs],
last_version=True)
db.session.add(result)
db.session.commit()
print "Project %s completed!" % project.short_name
def create_results():
"""Create results when migrating."""
from pybossa.core import project_repo, task_repo, result_repo
from pybossa.model.result import Result
projects = project_repo.filter_by(published=True)
for project in projects:
print "Working on project: %s" % project.short_name
tasks = task_repo.filter_tasks_by(state='completed',
project_id=project.id)
print "Analyzing %s tasks" % len(tasks)
for task in tasks:
result = result_repo.get_by(project_id=project.id, task_id=task.id)
if result is None:
result = Result(project_id=project.id,
task_id=task.id,
task_run_ids=[tr.id for tr in task.task_runs],
last_version=True)
db.session.add(result)
db.session.commit()
print "Project %s completed!" % project.short_name
def progress(short_name):
"""Return progress for each volume and template."""
category = project_repo.get_category_by(short_name=short_name)
if not category: # pragma: no cover
abort(404)
tmpl_index = {t['id']: t for t in category.info.get('templates', [])}
vol_index = {v['id']: v for v in category.info.get('volumes', [])}
data = {v_id: {t_id: None for t_id in tmpl_index} for v_id in vol_index}
projects = project_repo.filter_by(category_id=category.id)
for project in projects:
ps = project_stats.get_stats(project.id, full=True)
try:
row = data[project.info['volume_id']]
row[project.info['template_id']] = ps.overall_progress
except KeyError:
continue
# Replace IDs with names and flatten
flat_data = []
for vol_id in data:
row = {'Volume': vol_index[vol_id]['name']}
for tmpl_id in data[vol_id]:
tmpl_name = tmpl_index[tmpl_id]['name']
if tmpl_name == 'Volume':
tmpl_name = '_Volume'
row[tmpl_name] = data[vol_id][tmpl_id]
flat_data.append(row)
if request.args.get('csv'):
df = pandas.DataFrame(flat_data)
df.set_index('Volume', inplace=True)
csv = df.to_csv(encoding='utf8')
response = dict(progress=csv)
return handle_content_type(response)
response = dict(progress=flat_data)
return handle_content_type(response)
def update_volume(short_name, volume_id):
"""Update a volume."""
category = project_repo.get_category_by(short_name=short_name)
if not category: # pragma: no cover
abort(404)
ensure_authorized_to('update', category)
volumes = category.info.get('volumes', [])
try:
volume = [v for v in volumes if v['id'] == volume_id][0]
except IndexError:
abort(404)
form = VolumeForm(**volume)
form.category_id.data = category.id
all_importers = importer.get_all_importer_names()
form.importer.choices = [(name, name) for name in all_importers]
upload_form = AvatarUploadForm()
import_form = GenericBulkTaskImportForm()(volume['importer'],
**volume.get('data', {}))
def update():
"""Helper function to update the current volume."""
try:
idx = [
i for i, _vol in enumerate(volumes) if _vol['id'] == volume_id
][0]
except IndexError: # pragma: no cover
abort(404)
volumes[idx] = volume
category.info['volumes'] = volumes
project_repo.update_category(category)
cat_projects = project_repo.filter_by(category_id=category.id)
has_projects = len(
[p for p in cat_projects if p.info.get('volume_id') == volume_id]) > 0
if request.method == 'POST':
# Process task import form
if (request.form.get('btn') == 'Import'
or request.body.get('btn') == 'Import'):
import_form = GenericBulkTaskImportForm()(volume['importer'],
request.body)
if import_form.validate():
if has_projects:
flash('Update failed as projects have already been built',
'error')
else:
volume['data'] = import_form.get_import_data()
import_data = import_form.get_import_data()
try:
importer.count_tasks_to_import(**import_data)
update()
flash('Volume updated', 'success')
except BulkImportException as err:
flash(err.message, 'error')
else:
flash('Please correct the errors', 'error')
# Process volume details form
elif request.form.get('btn') != 'Upload':
form = VolumeForm(request.body)
all_importers = importer.get_all_importer_names()
form.importer.choices = [(name, name) for name in all_importers]
if form.validate():
if has_projects:
flash('Update failed as projects have already been built',
'error')
else:
volume['name'] = form.name.data
volume['short_name'] = form.short_name.data
volume['importer'] = form.importer.data
update()
flash('Volume updated', 'success')
else:
flash('Please correct the errors', 'error')
# Process thumbnail upload form
else:
if upload_form.validate_on_submit():
_file = request.files['avatar']
coordinates = (upload_form.x1.data, upload_form.y1.data,
upload_form.x2.data, upload_form.y2.data)
suffix = time.time()
_file.filename = "volume_{0}_{1}.png".format(
volume['id'], suffix)
container = "category_{}".format(category.id)
uploader.upload_file(_file,
container=container,
coordinates=coordinates)
# Delete previous thumbnail from storage
if volume.get('thumbnail'):
uploader.delete_file(volume['thumbnail'], container)
volume['thumbnail'] = _file.filename
volume['container'] = container
upload_method = current_app.config.get('UPLOAD_METHOD')
thumbnail_url = get_avatar_url(upload_method, _file.filename,
container)
volume['thumbnail_url'] = thumbnail_url
update()
project_repo.save_category(category)
flash('Thumbnail updated', 'success')
url = url_for('.get_volumes', short_name=category.short_name)
return redirect_content_type(url)
else:
flash('You must provide a file', 'error')
response = dict(form=form,
all_importers=all_importers,
upload_form=upload_form,
import_form=import_form,
volume=volume,
has_projects=has_projects)
return handle_content_type(response)
