def import_members_from_csv(self, f): with csv_text_io_wrapper(f) as ftxt: reader = csv.reader(ftxt.read().splitlines()) emails = set() for num_row, row in enumerate(reader, 1): if len(row) != 1: raise UserValueError( _('Row {}: malformed CSV data').format(num_row)) email = row[0].strip().lower() if email and not validate_email(email): raise UserValueError( _('Row {row}: invalid email address: {email}').format( row=num_row, email=email)) if email in emails: raise UserValueError( _('Row {}: email address is not unique').format(num_row)) emails.add(email) users = set( User.query.filter(~User.is_deleted, User.all_emails.in_(emails))) users_emails = {user.email for user in users} unknown_emails = emails - users_emails new_members = users - self.role.members return new_members, users, unknown_emails
def import_registrations_from_csv(regform, fileobj, skip_moderation=True, notify_users=False): """Import event registrants from a CSV file into a form.""" with csv_text_io_wrapper(fileobj) as ftxt: reader = csv.reader(ftxt.read().splitlines()) query = db.session.query(Registration.email).with_parent(regform).filter( Registration.is_active) registered_emails = {email for (email, ) in query} used_emails = set() todo = [] for row_num, row in enumerate(reader, 1): try: first_name, last_name, affiliation, position, phone, email = [ value.strip() for value in row ] email = email.lower() except ValueError: raise UserValueError( _('Row {}: malformed CSV data - please check that the number of columns is correct' ).format(row_num)) if not email: raise UserValueError( _('Row {}: missing e-mail address').format(row_num)) if not validate_email(email): raise UserValueError( _('Row {}: invalid e-mail address').format(row_num)) if not first_name or not last_name: raise UserValueError( _('Row {}: missing first or last name').format(row_num)) if email in registered_emails: raise UserValueError( _('Row {}: a registration with this email already exists'). format(row_num)) if email in used_emails: raise UserValueError( _('Row {}: email address is not unique').format(row_num)) used_emails.add(email) todo.append({ 'email': email, 'first_name': first_name.title(), 'last_name': last_name.title(), 'affiliation': affiliation, 'phone': phone, 'position': position }) return [ create_registration(regform, data, notify_user=notify_users, skip_moderation=skip_moderation) for data in todo ]
def import_registrations_from_csv(regform, fileobj, skip_moderation=True, notify_users=False): """Import event registrants from a CSV file into a form.""" with csv_text_io_wrapper(fileobj) as ftxt: reader = csv.reader(ftxt.read().splitlines()) reg_data = (db.session.query(Registration.user_id, Registration.email) .with_parent(regform) .filter(Registration.is_active) .all()) registered_user_ids = {rd.user_id for rd in reg_data if rd.user_id is not None} registered_emails = {rd.email for rd in reg_data} used_emails = set() email_row_map = {} todo = [] for row_num, row in enumerate(reader, 1): try: first_name, last_name, affiliation, position, phone, email = [value.strip() for value in row] email = email.lower() except ValueError: raise UserValueError(_('Row {}: malformed CSV data - please check that the number of columns is correct') .format(row_num)) if not email: raise UserValueError(_('Row {}: missing e-mail address').format(row_num)) if not validate_email(email): raise UserValueError(_('Row {}: invalid e-mail address').format(row_num)) if not first_name or not last_name: raise UserValueError(_('Row {}: missing first or last name').format(row_num)) if email in registered_emails: raise UserValueError(_('Row {}: a registration with this email already exists').format(row_num)) user = get_user_by_email(email) if user and user.id in registered_user_ids: raise UserValueError(_('Row {}: a registration for this user already exists').format(row_num)) if email in used_emails: raise UserValueError(_('Row {}: email address is not unique').format(row_num)) if conflict_row_num := email_row_map.get(email): raise UserValueError(_('Row {}: email address belongs to the same user as in row {}') .format(row_num, conflict_row_num)) used_emails.add(email) if user: email_row_map.update((e, row_num) for e in user.all_emails) todo.append({ 'email': email, 'first_name': first_name.title(), 'last_name': last_name.title(), 'affiliation': affiliation, 'phone': phone, 'position': position })
def import_user_records_from_csv(fileobj, columns): """Parse and do basic validation of user data from a CSV file. :param fileobj: the CSV file to be read :param columns: A list of column names, 'first_name', 'last_name', & 'email' are compulsory. :return: A list dictionaries each representing one row, the keys of which are given by the column names. """ with csv_text_io_wrapper(fileobj) as ftxt: reader = csv.reader(ftxt.read().splitlines()) used_emails = set() email_row_map = {} user_records = [] for row_num, row in enumerate(reader, 1): values = [value.strip() for value in row] if len(columns) != len(values): raise UserValueError( _('Row {}: malformed CSV data - please check that the number of columns is correct' ).format(row_num)) record = dict(zip(columns, values)) if not record['email']: raise UserValueError( _('Row {}: missing e-mail address').format(row_num)) record['email'] = record['email'].lower() if not validate_email(record['email']): raise UserValueError( _('Row {}: invalid e-mail address').format(row_num)) if not record['first_name'] or not record['last_name']: raise UserValueError( _('Row {}: missing first or last name').format(row_num)) record['first_name'] = record['first_name'].title() record['last_name'] = record['last_name'].title() if record['email'] in used_emails: raise UserValueError( _('Row {}: email address is not unique').format(row_num)) if conflict_row_num := email_row_map.get(record['email']): raise UserValueError( _('Row {}: email address belongs to the same user as in row {}' ).format(row_num, conflict_row_num)) used_emails.add(record['email']) if user := get_user_by_email(record['email']): email_row_map.update((e, row_num) for e in user.all_emails)
def import_contributions_from_csv(event, f): """Import timetable contributions from a CSV file into an event.""" with csv_text_io_wrapper(f) as ftxt: reader = csv.reader(ftxt.read().splitlines()) contrib_data = [] for num_row, row in enumerate(reader, 1): try: start_dt, duration, title, first_name, last_name, affiliation, email = ( value.strip() for value in row) email = email.lower() except ValueError: raise UserValueError( _('Row {}: malformed CSV data - please check that the number of columns is correct' ).format(num_row)) try: parsed_start_dt = event.tzinfo.localize( dateutil.parser.parse(start_dt)) if start_dt else None except ValueError: raise UserValueError( _('Row {row}: can\'t parse date: "{date}"').format( row=num_row, date=start_dt)) try: parsed_duration = timedelta( minutes=int(duration)) if duration else None except ValueError: raise UserValueError( _("Row {row}: can't parse duration: {duration}").format( row=num_row, duration=duration)) if not title: raise UserValueError( _('Row {}: contribution title is required').format(num_row)) if email and not validate_email(email): raise UserValueError( _('Row {row}: invalid email address: {email}').format( row=num_row, email=email)) contrib_data.append({ 'start_dt': parsed_start_dt, 'duration': parsed_duration or timedelta(minutes=20), 'title': title, 'speaker': { 'first_name': first_name, 'last_name': last_name, 'affiliation': affiliation, 'email': email } }) # now that we're sure the data is OK, let's pre-allocate the friendly ids # for the contributions in question Contribution.allocate_friendly_ids(event, len(contrib_data)) contributions = [] all_changes = defaultdict(list) for contrib_fields in contrib_data: speaker_data = contrib_fields.pop('speaker') with track_time_changes() as changes: contribution = create_contribution(event, contrib_fields, extend_parent=True) contributions.append(contribution) for key, val in changes[event].items(): all_changes[key].append(val) email = speaker_data['email'] if not email: continue # set the information of the speaker person = get_event_person(event, speaker_data) link = ContributionPersonLink(person=person, is_speaker=True) link.populate_from_dict({ 'first_name': speaker_data['first_name'], 'last_name': speaker_data['last_name'], 'affiliation': speaker_data['affiliation'] }) contribution.person_links.append(link) return contributions, all_changes