def validate_s3_bucket(task):
valid = valid_or_no_s3_bucket(task.info)
if not valid:
current_app.logger.error(
'Invalid S3 bucket. project id: {}, task info: {}'.format(
task.project_id, task.info))
return valid
def create_tasks(self, task_repo, project, **form_data):
"""Create tasks."""
from pybossa.model.task import Task
"""Create tasks from a remote source using an importer object and
avoiding the creation of repeated tasks"""
empty = True
n = 0
importer = self._create_importer_for(**form_data)
tasks = importer.tasks()
import_headers = importer.headers()
mismatch_headers = []
if import_headers:
msg = None
if not project:
msg = gettext('Could not load project info')
else:
task_presenter_headers = project.get_presenter_headers()
mismatch_headers = [
header for header in task_presenter_headers
if header not in import_headers
]
if mismatch_headers:
msg = 'Imported columns do not match task presenter code. '
additional_msg = 'Mismatched columns: {}'.format(
(', '.join(mismatch_headers))[:80])
current_app.logger.error(msg)
current_app.logger.error(', '.join(mismatch_headers))
msg += additional_msg
if msg:
# Failed validation
current_app.logger.error(msg)
return ImportReport(message=msg, metadata=None, total=0)
s3_bucket_failures = 0
for task_data in tasks:
task = Task(project_id=project.id)
[setattr(task, k, v) for k, v in task_data.iteritems()]
found = task_repo.find_duplicate(project_id=project.id,
info=task.info)
if found is None:
if valid_or_no_s3_bucket(task.info):
task_repo.save(task)
n += 1
empty = False
else:
s3_bucket_failures += 1
current_app.logger.error(
'Invalid S3 bucket. project id: {}, task info: {}'.
format(project.id, task.info))
additional_msg = ' {} task import failed due to invalid S3 bucket.'\
.format(s3_bucket_failures) if s3_bucket_failures else ''
if form_data.get('type') == 'localCSV':
s3_url = form_data.get('csv_filename')
importer.delete_local_csv_import_s3_file(s3_url)
if empty:
msg = gettext('It looks like there were no new records to import.')
msg += additional_msg
return ImportReport(message=msg, metadata=None, total=n)
metadata = importer.import_metadata()
msg = str(n) + " " + gettext('new tasks were imported successfully ')
if n == 1:
msg = str(n) + " " + gettext('new task was imported successfully ')
msg += additional_msg
report = ImportReport(message=msg, metadata=metadata, total=n)
return report
