def get_export_task_jobs(queue): """Export tasks to zip.""" from pybossa.core import project_repo import pybossa.cache.projects as cached_projects from pybossa.pro_features import ProFeatureHandler feature_handler = ProFeatureHandler(current_app.config.get('PRO_FEATURES')) timeout = current_app.config.get('TIMEOUT') if feature_handler.only_for_pro('updated_exports'): if queue == 'high': projects = cached_projects.get_from_pro_user() else: projects = (p.dictize() for p in project_repo.filter_by(published=True) if p.owner.pro is False) else: projects = (p.dictize() for p in project_repo.filter_by(published=True)) for project in projects: project_id = project.get('id') job = dict(name=project_export, args=[project_id], kwargs={}, timeout=timeout, queue=queue) yield job
def get_enhanced_volumes(category): """Return the categories volumes enhanced with project data.""" volumes = category.info.get('volumes', []) projects = project_repo.filter_by(category_id=category.id) for volume in volumes: vol_projects = [ dict(id=p.id, name=p.name, short_name=p.short_name, published=p.published, overall_progress=overall_progress(p.id)) for p in projects if p.info.get('volume_id') == volume['id'] ] completed_projects = [ p for p in vol_projects if p['overall_progress'] == 100 ] ongoing_projects = [ p for p in vol_projects if p['published'] and p not in completed_projects ] volume['projects'] = vol_projects volume['n_completed_projects'] = len(completed_projects) volume['n_ongoing_projects'] = len(ongoing_projects) return volumes
def password_protect_hidden_projects(): import random from pybossa.core import project_repo, user_repo, mail from pybossa.jobs import enqueue_job, send_mail def generate_random_password(): CHARS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789' password = '' for i in range(8): password += random.choice(CHARS) return password def generate_email_for(project_name, owner_name, password): subject = "Changes in your hidden project %s" % project_name content = ( """ Dear %s, We are writing you to let you know that, due to recent changes in Crowdcrafting, hidden projects will soon no longer be supported. However, you can still protect your project with a password, allowing only people with it to access and contribute to it. We have checked that your project %s is hidden. We don't want to expose it to the public, so we have protected it with a password instead. The current password for your project is: %s You will be able to change it on your project settings page. You can find more information about passwords in the documentation (http://docs.pybossa.com/user/tutorial/#protecting-the-project-with-a-password). If you have any doubts, please contact us and we will be pleased to help you! Best regards, Crowdcrafting team. """ % (owner_name, project_name, password)) return subject, content with app.app_context(): for project in project_repo.filter_by(hidden=1): password = generate_random_password() subject, content = generate_email_for(project.name, project.owner.name, password) message = dict(recipients=[project.owner.email_addr], subject=subject, body=content) job = dict(name=send_mail, args=[message], kwargs={}, timeout=(600), queue='medium') enqueue_job(job) project.set_password(password) project_repo.save(project)
def password_protect_hidden_projects(): import random from pybossa.core import project_repo, user_repo, mail from pybossa.jobs import enqueue_job, send_mail def generate_random_password(): CHARS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789' password = '' for i in range(8): password += random.choice(CHARS) return password def generate_email_for(project_name, owner_name, password): subject = "Changes in your hidden project %s" % project_name content = ( """ Dear %s, We are writing you to let you know that, due to recent changes in Crowdcrafting, hidden projects will soon no longer be supported. However, you can still protect your project with a password, allowing only people with it to access and contribute to it. We have checked that your project %s is hidden. We don't want to expose it to the public, so we have protected it with a password instead. The current password for your project is: %s You will be able to change it on your project settings page. You can find more information about passwords in the documentation (http://docs.pybossa.com/en/latest/user/tutorial.html#protecting-the-project-with-a-password). If you have any doubts, please contact us and we will be pleased to help you! Best regards, Crowdcrafting team. """ % (owner_name, project_name, password)) return subject, content with app.app_context(): for project in project_repo.filter_by(hidden=1): password = generate_random_password() subject, content = generate_email_for(project.name, project.owner.name, password) message = dict(recipients=[project.owner.email_addr], subject=subject, body=content) job = dict(name=send_mail, args=[message], kwargs={}, timeout=(600), queue='medium') enqueue_job(job) project.set_password(password) project_repo.save(project)
def get_projects_with_unknown_volumes(category): """Return all projects not linked to a known volume.""" volume_ids = [vol['id'] for vol in category.info.get('volumes', [])] projects = project_repo.filter_by(category_id=category.id) return [ dict(id=p.id, name=p.name, short_name=p.short_name) for p in projects if not p.info.get('volume_id') or p.info.get('volume_id') not in volume_ids ]
def get_parent(parent_template_id, volume_id, category): """Return a valid parent project.""" projects = project_repo.filter_by(category_id=category.id) try: return [ p for p in projects if p.info.get('template_id') == parent_template_id and p.info.get('volume_id') == volume_id and validate_parent(p) ][0] except IndexError: return None
def index(): """Return the Data page.""" projects = [p for p in project_repo.filter_by(published=True) if not p.needs_password()] title = "Data" description = """Download open datasets of all crowdsourced data produced via LibCrowds.""" display = {'tasks': current_app.config['DATA_DISPLAY_TASKS'], 'task_runs': current_app.config['DATA_DISPLAY_TASK_RUNS'], 'results': current_app.config['DATA_DISPLAY_RESULTS'], 'flickr': current_app.config['DATA_DISPLAY_FLICKR']} return render_template('/index.html', projects=projects, display=display)
def export_userdata(user_id, **kwargs): from pybossa.core import user_repo, project_repo, task_repo, result_repo from flask import current_app, url_for json_exporter = JsonExporter() user = user_repo.get(user_id) user_data = user.dictize() del user_data['passwd_hash'] projects = project_repo.filter_by(owner_id=user.id) projects_data = [project.dictize() for project in projects] taskruns = task_repo.filter_task_runs_by(user_id=user.id) taskruns_data = [tr.dictize() for tr in taskruns] pdf = json_exporter._make_zip(None, '', 'personal_data', user_data, user_id, 'personal_data.zip') upf = json_exporter._make_zip(None, '', 'user_projects', projects_data, user_id, 'user_projects.zip') ucf = json_exporter._make_zip(None, '', 'user_contributions', taskruns_data, user_id, 'user_contributions.zip') upload_method = current_app.config.get('UPLOAD_METHOD') if upload_method == 'local': upload_method = 'uploads.uploaded_file' personal_data_link = url_for(upload_method, filename="user_%s/%s" % (user_id, pdf)) personal_projects_link = url_for(upload_method, filename="user_%s/%s" % (user_id, upf)) personal_contributions_link = url_for(upload_method, filename="user_%s/%s" % (user_id, ucf)) body = render_template('/account/email/exportdata.md', user=user.dictize(), personal_data_link=personal_data_link, personal_projects_link=personal_projects_link, personal_contributions_link=personal_contributions_link, config=current_app.config) html = render_template('/account/email/exportdata.html', user=user.dictize(), personal_data_link=personal_data_link, personal_projects_link=personal_projects_link, personal_contributions_link=personal_contributions_link, config=current_app.config) subject = 'Your personal data' mail_dict = dict(recipients=[user.email_addr], subject=subject, body=body, html=html) send_mail(mail_dict)
def analyse_empty_results(category_id): """Analyse empty results for a category.""" category = project_repo.get_category(category_id) if not category: abort(404) if request.method == 'POST': presenter = category.info.get('presenter') projects = project_repo.filter_by(category_id=category.id) for project in projects: analyse_empty(project.id, presenter) flash('Analysis of empty results queued', 'success') csrf = None else: csrf = generate_csrf() response = dict(csrf=csrf) return handle_content_type(response)
def update_project_stats(): """Update project stats for draft projects.""" from pybossa.core import db from pybossa.core import project_repo, task_repo, result_repo from pybossa.model.task import Task from pybossa.model.task_run import TaskRun from pybossa.model.counter import Counter import pybossa.cache.project_stats as stats projects = project_repo.filter_by(published=False) for project in projects: print "Working on project: %s" % project.short_name sql_query = """INSERT INTO project_stats (project_id, n_tasks, n_task_runs, n_results, n_volunteers, n_completed_tasks, overall_progress, average_time, n_blogposts, last_activity, info) VALUES (%s, 0, 0, 0, 0, 0, 0, 0, 0, 0, '{}');""" % (project.id) db.engine.execute(sql_query)
def get_project_jobs(queue): """Return a list of jobs based on user type.""" from pybossa.core import project_repo from pybossa.cache import projects as cached_projects timeout = current_app.config.get('TIMEOUT') if queue == 'super': projects = cached_projects.get_from_pro_user() elif queue == 'high': projects = (p.dictize() for p in project_repo.filter_by(published=True) if p.owner.pro is False) else: projects = [] for project in projects: project_id = project.get('id') project_short_name = project.get('short_name') job = dict(name=get_project_stats, args=[project_id, project_short_name], kwargs={}, timeout=timeout, queue=queue) yield job
def project_filters(short_name): """Return all filters currently associated with the category's projects.""" category = project_repo.get_category_by(short_name=short_name) if not category: # pragma: no cover abort(404) projects = project_repo.filter_by(category_id=category.id) all_filters = [(k, v) for project in projects for k, v in project.info.get('filters', {}).items()] filters = {} for _filter in all_filters: key = _filter[0] value = _filter[1] filter_opts = filters.get(key, []) filter_opts.append(value) filter_opts = list(set(filter_opts)) filters[key] = filter_opts response = dict(filters=filters) return handle_content_type(response)
def create_results(): """Create results when migrating.""" from pybossa.core import project_repo, task_repo, result_repo from pybossa.model.result import Result projects = project_repo.filter_by(published=True) for project in projects: print "Working on project: %s" % project.short_name tasks = task_repo.filter_tasks_by(state='completed', project_id=project.id) print "Analyzing %s tasks" % len(tasks) for task in tasks: result = result_repo.get_by(project_id=project.id, task_id=task.id) if result is None: result = Result(project_id=project.id, task_id=task.id, task_run_ids=[tr.id for tr in task.task_runs], last_version=True) db.session.add(result) db.session.commit() print "Project %s completed!" % project.short_name
def progress(short_name): """Return progress for each volume and template.""" category = project_repo.get_category_by(short_name=short_name) if not category: # pragma: no cover abort(404) tmpl_index = {t['id']: t for t in category.info.get('templates', [])} vol_index = {v['id']: v for v in category.info.get('volumes', [])} data = {v_id: {t_id: None for t_id in tmpl_index} for v_id in vol_index} projects = project_repo.filter_by(category_id=category.id) for project in projects: ps = project_stats.get_stats(project.id, full=True) try: row = data[project.info['volume_id']] row[project.info['template_id']] = ps.overall_progress except KeyError: continue # Replace IDs with names and flatten flat_data = [] for vol_id in data: row = {'Volume': vol_index[vol_id]['name']} for tmpl_id in data[vol_id]: tmpl_name = tmpl_index[tmpl_id]['name'] if tmpl_name == 'Volume': tmpl_name = '_Volume' row[tmpl_name] = data[vol_id][tmpl_id] flat_data.append(row) if request.args.get('csv'): df = pandas.DataFrame(flat_data) df.set_index('Volume', inplace=True) csv = df.to_csv(encoding='utf8') response = dict(progress=csv) return handle_content_type(response) response = dict(progress=flat_data) return handle_content_type(response)
def update_volume(short_name, volume_id): """Update a volume.""" category = project_repo.get_category_by(short_name=short_name) if not category: # pragma: no cover abort(404) ensure_authorized_to('update', category) volumes = category.info.get('volumes', []) try: volume = [v for v in volumes if v['id'] == volume_id][0] except IndexError: abort(404) form = VolumeForm(**volume) form.category_id.data = category.id all_importers = importer.get_all_importer_names() form.importer.choices = [(name, name) for name in all_importers] upload_form = AvatarUploadForm() import_form = GenericBulkTaskImportForm()(volume['importer'], **volume.get('data', {})) def update(): """Helper function to update the current volume.""" try: idx = [ i for i, _vol in enumerate(volumes) if _vol['id'] == volume_id ][0] except IndexError: # pragma: no cover abort(404) volumes[idx] = volume category.info['volumes'] = volumes project_repo.update_category(category) cat_projects = project_repo.filter_by(category_id=category.id) has_projects = len( [p for p in cat_projects if p.info.get('volume_id') == volume_id]) > 0 if request.method == 'POST': # Process task import form if (request.form.get('btn') == 'Import' or request.body.get('btn') == 'Import'): import_form = GenericBulkTaskImportForm()(volume['importer'], request.body) if import_form.validate(): if has_projects: flash('Update failed as projects have already been built', 'error') else: volume['data'] = import_form.get_import_data() import_data = import_form.get_import_data() try: importer.count_tasks_to_import(**import_data) update() flash('Volume updated', 'success') except BulkImportException as err: flash(err.message, 'error') else: flash('Please correct the errors', 'error') # Process volume details form elif request.form.get('btn') != 'Upload': form = VolumeForm(request.body) all_importers = importer.get_all_importer_names() form.importer.choices = [(name, name) for name in all_importers] if form.validate(): if has_projects: flash('Update failed as projects have already been built', 'error') else: volume['name'] = form.name.data volume['short_name'] = form.short_name.data volume['importer'] = form.importer.data update() flash('Volume updated', 'success') else: flash('Please correct the errors', 'error') # Process thumbnail upload form else: if upload_form.validate_on_submit(): _file = request.files['avatar'] coordinates = (upload_form.x1.data, upload_form.y1.data, upload_form.x2.data, upload_form.y2.data) suffix = time.time() _file.filename = "volume_{0}_{1}.png".format( volume['id'], suffix) container = "category_{}".format(category.id) uploader.upload_file(_file, container=container, coordinates=coordinates) # Delete previous thumbnail from storage if volume.get('thumbnail'): uploader.delete_file(volume['thumbnail'], container) volume['thumbnail'] = _file.filename volume['container'] = container upload_method = current_app.config.get('UPLOAD_METHOD') thumbnail_url = get_avatar_url(upload_method, _file.filename, container) volume['thumbnail_url'] = thumbnail_url update() project_repo.save_category(category) flash('Thumbnail updated', 'success') url = url_for('.get_volumes', short_name=category.short_name) return redirect_content_type(url) else: flash('You must provide a file', 'error') response = dict(form=form, all_importers=all_importers, upload_form=upload_form, import_form=import_form, volume=volume, has_projects=has_projects) return handle_content_type(response)