def validate_s3_bucket(task): valid = valid_or_no_s3_bucket(task.info) if not valid: current_app.logger.error( 'Invalid S3 bucket. project id: {}, task info: {}'.format( task.project_id, task.info)) return valid
def create_tasks(self, task_repo, project, **form_data): """Create tasks.""" from pybossa.model.task import Task """Create tasks from a remote source using an importer object and avoiding the creation of repeated tasks""" empty = True n = 0 importer = self._create_importer_for(**form_data) tasks = importer.tasks() import_headers = importer.headers() mismatch_headers = [] if import_headers: msg = None if not project: msg = gettext('Could not load project info') else: task_presenter_headers = project.get_presenter_headers() mismatch_headers = [ header for header in task_presenter_headers if header not in import_headers ] if mismatch_headers: msg = 'Imported columns do not match task presenter code. ' additional_msg = 'Mismatched columns: {}'.format( (', '.join(mismatch_headers))[:80]) current_app.logger.error(msg) current_app.logger.error(', '.join(mismatch_headers)) msg += additional_msg if msg: # Failed validation current_app.logger.error(msg) return ImportReport(message=msg, metadata=None, total=0) s3_bucket_failures = 0 for task_data in tasks: task = Task(project_id=project.id) [setattr(task, k, v) for k, v in task_data.iteritems()] found = task_repo.find_duplicate(project_id=project.id, info=task.info) if found is None: if valid_or_no_s3_bucket(task.info): task_repo.save(task) n += 1 empty = False else: s3_bucket_failures += 1 current_app.logger.error( 'Invalid S3 bucket. project id: {}, task info: {}'. format(project.id, task.info)) additional_msg = ' {} task import failed due to invalid S3 bucket.'\ .format(s3_bucket_failures) if s3_bucket_failures else '' if form_data.get('type') == 'localCSV': s3_url = form_data.get('csv_filename') importer.delete_local_csv_import_s3_file(s3_url) if empty: msg = gettext('It looks like there were no new records to import.') msg += additional_msg return ImportReport(message=msg, metadata=None, total=n) metadata = importer.import_metadata() msg = str(n) + " " + gettext('new tasks were imported successfully ') if n == 1: msg = str(n) + " " + gettext('new task was imported successfully ') msg += additional_msg report = ImportReport(message=msg, metadata=metadata, total=n) return report