def init_riders(): large_delete_all( Rider ) tdf = tdf.strip() lines = tdf.split( '\n' ) with transaction.atomic(): for count, line in enumerate(lines): if count % 20 != 0: # Only add every 20 riders. continue fields = line.split( '\t' ) full_name = fields[1].strip() names = full_name.split() first_name = names[0] last_name = ' '.join( names[1:] ) team = fields[3] gender = 0 years_old = int(fields[4].strip('*')) date_of_birth = datetime.date( 2012 - years_old, 3, 3 ) license = years_old safe_print( first_name, last_name, team, gender, date_of_birth, license ) r = Rider( first_name=unicode(first_name, 'iso-8859-1'), last_name=unicode(last_name, 'iso-8859-1'), team=unicode(team, 'iso-8859-1'), gender=gender, date_of_birth=date_of_birth, license=license ) r.save()
def init_categories(): large_delete_all(Category) large_delete_all(CategoryFormat) for (name, description, catStr) in rfs: safe_print(name, description) rf = CategoryFormat(name=name + ' - Reference', description=description) rf.save() sequence = 0 with transaction.atomic(): for cat in catStr.split('\n'): try: code, gender, description = cat.split('\t') except ValueError: continue safe_print(u' ', code, gender, description) c = Category(format=rf, code=code, gender=['Men', 'Women', 'Open'].index(gender), description=description, sequence=sequence) c.save() sequence += 1
def init_static(): for cls, values in [ (Discipline, disciplines), (RaceClass, race_classes), (NumberSet, number_sets), ]: large_delete_all(cls) with transaction.atomic(): for i, name in enumerate(values): safe_print(i, name) cls(name=name, sequence=i).save()
def init_license_holders(): global tdf large_delete_all(LicenseHolder) large_delete_all(Team) tdf = tdf.decode('iso-8859-1').strip() lines = tdf.split('\n') @transaction.atomic def process_records(lines): for count, line in enumerate(lines): fields = line.split('\t') full_name = fields[1].strip() names = full_name.split() first_name = names[0] last_name = ' '.join(names[1:]) team = fields[3].strip() gender = 0 years_old = int(fields[4].strip('*')) date_of_birth = datetime.date(2013 - years_old, 3, 3) nationality = fields[2].strip() nationality_code = uci_country_codes.get(nationality.upper(), 'CAN') uci_code = nationality_code + date_of_birth.strftime('%Y%m%d') safe_print(first_name, last_name, gender, date_of_birth, nationality, uci_code, team) fields = { 'first_name': first_name, 'last_name': last_name, 'gender': gender, 'nationality': nationality, 'date_of_birth': date_of_birth, 'uci_code': uci_code, 'license_code': u'{}'.format(count + 1), } if not LicenseHolder.objects.filter(**fields).exists(): LicenseHolder(**fields).save() if not Team.objects.filter(name=team).exists(): Team( name=team, team_code=team[:3], team_type=7, ).save() process_records(lines)
def init_prereg(competition_name='', worksheet_name='', clear_existing=False, competitionId=None, worksheet_contents=None, message_stream=sys.stdout): tt = TimeTracker() team_lookup = TeamLookup() tstart = datetime.datetime.now() today = datetime.date.today() if message_stream == sys.stdout or message_stream == sys.stderr: def ms_write(s, flush=False): message_stream.write(removeDiacritic(s)) else: def ms_write(s, flush=False): message_stream.write(unicode(s)) sys.stdout.write(removeDiacritic(s)) if flush: sys.stdout.flush() fix_bad_license_codes() if competitionId is not None: competition = Competition.objects.get(pk=competitionId) else: try: competition = Competition.objects.get(name=competition_name) except Competition.DoesNotExist: ms_write(u'**** Cannot find Competition: "{}"\n'.format( competition_name)) return except Competition.MultipleObjectsReturned: ms_write( u'**** Found multiple Competitions matching: "{}"\n'.format( competition_name)) return optional_events = { normalize(event.name): event for event in competition.get_events() if event.optional } pattern_optional_events = defaultdict(list) # Check for duplicate event names. event_name_count = defaultdict(int) for event in competition.get_events(): if event.optional: event_name_count[event.name] += 1 for event_name, count in event_name_count.iteritems(): if count > 1: ms_write( u'**** Error: Duplicate Optional Event Name: "{}". Perferences to Optional Events may not work properly.\n' .format(event_name)) role_code = {} for role_type, roles in Participant.COMPETITION_ROLE_CHOICES: role_code.update({ unicode(name).lower().replace(' ', '').replace('.', ''): code for code, name in roles }) # Construct a cache to find categories quicker. category_code_gender_suffix = re.compile(r'\(Open\)$|\(Men\)$|\(Women\)$') cat_gender = defaultdict(list) category_numbers_set = {None: set()} for category in Category.objects.filter( format=competition.category_format).order_by('gender', 'code'): cat_gender[category.code].append((category, category.gender)) cn = competition.get_category_numbers(category) category_numbers_set[category] = cn.get_numbers() if cn else set() def get_category(category_code_search, gender_search): category_code_search = category_code_gender_suffix.sub( '', category_code_search).strip() for category, gender in cat_gender.get(category_code_search, []): if gender_search == gender or gender == 2: return category return None times = defaultdict(float) has_legal_entity = competition.legal_entity ifm = standard_field_map(exclude=([] if has_legal_entity else ['waiver'])) if has_legal_entity: waiver_signed_date = competition.legal_entity.waiver_expiry_date + datetime.timedelta( days=1) # Process the records in large transactions for efficiency. def process_ur_records(ur_records): for i, ur in ur_records: tt.start('get_fields_from_row') v = ifm.finder(ur) date_of_birth = v('date_of_birth', None) try: date_of_birth = date_from_value(date_of_birth) except Exception as e: ms_write( 'Row {}: Ignoring birthdate (must be YYYY-MM-DD) "{}" ({}) {}' .format(i, date_of_birth, ur, e)) date_of_birth = None date_of_birth = date_of_birth if date_of_birth != invalid_date_of_birth else None uci_code = to_str(v('uci_code', None)) # If no date of birth, get it from the UCI code. if not date_of_birth and uci_code and not uci_code.isdigit(): try: date_of_birth = datetime.date(int(uci_code[3:7]), int(uci_code[7:9]), int(uci_code[9:11])) except: pass # If no date of birth, make one up based on the age. age = to_int(v('age', None)) if not date_of_birth and age: date_of_birth = date_of_birth_from_age(age, today) year_only_dob = True else: year_only_dob = False # As a last resort, pick the default DOB date_of_birth = date_of_birth or invalid_date_of_birth license_code = to_int_str(v('license_code', u'')).upper().strip() last_name = to_str(v('last_name', u'')) first_name = to_str(v('first_name', u'')) name = to_str(v('name', u'')) if not name: name = ' '.join(n for n in [first_name, last_name] if n) gender = to_str(v('gender', u'')) gender = gender_from_str(gender) if gender else None email = to_str(v('email', None)) phone = to_phone(v('phone', None)) city = to_str(v('city', None)) state_prov = to_str(v('state_prov', None)) zip_postal = to_str(v('zip_postal', None)) if zip_postal: zip_postal = validate_postal_code(zip_postal) preregistered = to_bool(v('preregistered', True)) paid = to_bool(v('paid', None)) bib = (to_int(v('bib', None)) or None) bib_auto = (bib is None and unicode(v('bib', u'')).lower() == u'auto') tag = to_int_str(v('tag', None)) note = to_str(v('note', None)) team_name = to_str(v('team', None)) club_name = to_str(v('club', None)) if not team_name: team_name = club_name category_code = to_str(v('category_code', None)) est_kmh = to_float(v('est_kmh', None)) est_mph = to_float(v('est_mph', None)) if est_mph is not None: est_kmh = 1.609344 * est_mph est_mph = None seed_option = to_str(v('seed_option', None)) if seed_option is not None: seed_option = seed_option.lower() if 'early' in seed_option: seed_option = 0 elif 'late' in seed_option: seed_option = 2 elif 'last' in seed_option: seed_option = 3 else: seed_option = 1 uci_id = to_uci_id(v('uci_id', None)) nation_code = to_str(v('nation_code', None)) emergency_contact_name = to_str(v('emergency_contact_name', None)) emergency_contact_phone = to_phone( v('emergency_contact_phone', None)) emergency_medical = to_str(v('emergency_medical', None)) participant_optional_events = [] for pattern, events in pattern_optional_events.iteritems(): included = to_bool(v(pattern, False)) participant_optional_events.extend( (event, included) for event in events) race_entered = to_str(v('race_entered', None)) role = role_code.get( v('role', '').lower().replace(' ', '').replace('.', ''), None) waiver = to_bool(v('waiver', None)) license_checked = to_bool(v('license_checked', None)) #------------------------------------------------------------------------------ # Get LicenseHolder. # license_holder = None #with transaction.atomic(): if True: tt.start('get_license_holder') if uci_id: license_holder = LicenseHolder.objects.filter( uci_id=uci_id).first() if not license_holder and license_code and license_code.upper( ) != u'TEMP': try: license_holder = LicenseHolder.objects.get( license_code=license_code) except LicenseHolder.DoesNotExist: ms_write( u'**** Row {}: cannot find LicenceHolder from LicenseCode: {}, Name="{}"\n' .format(i, license_code, name)) continue elif not license_holder: # No license code. Try to find the participant by last/first name, [date_of_birth] and [gender]. # Case insensitive comparison, accents ignored for names. q = Q(search_text__startswith=utils.get_search_text( [last_name, first_name])) if date_of_birth and date_of_birth != invalid_date_of_birth: if year_only_dob: q &= Q(date_of_birth__year=date_of_birth.year) else: q &= Q(date_of_birth=date_of_birth) if gender is not None: q &= Q(gender=gender) try: license_holder = LicenseHolder.objects.get(q) except LicenseHolder.DoesNotExist: # No name match. # Create a temporary license holder. try: license_holder = LicenseHolder( **{ attr: value for attr, value in { 'license_code': 'TEMP', 'last_name': last_name, 'first_name': first_name, 'gender': gender, 'date_of_birth': date_of_birth, 'uci_code': uci_code, 'nation_code': nation_code, 'uci_id': uci_id, 'email': email, 'phone': phone, 'city': city, 'state_prov': state_prov, 'zip_postal': zip_postal, 'emergency_contact_name': emergency_contact_name, 'emergency_contact_phone': emergency_contact_phone, 'emergency_medical': emergency_medical, 'existing_tag': tag if competition. use_existing_tags else None, }.iteritems() if value is not None }) truncate_char_fields(license_holder).save() except Exception as e: ms_write( u'**** Row {}: New License Holder Exception: {}, Name="{}"\n' .format( i, e, name, )) continue except LicenseHolder.MultipleObjectsReturned: ms_write( u'**** Row {}: found multiple LicenceHolders matching "Last, First" Name="{}"\n' .format( i, name, )) continue # Update the license_holder record with all new information. tt.start('update_license_holder') if set_attributes( license_holder, { 'date_of_birth': date_of_birth if not year_only_dob and date_of_birth != invalid_date_of_birth else None, 'uci_code': uci_code, 'email': email, 'phone': phone, 'city': city, 'state_prov': state_prov, 'zip_postal': zip_postal, 'nation_code': nation_code, 'uci_id': uci_id, 'emergency_contact_name': emergency_contact_name, 'emergency_contact_phone': emergency_contact_phone, 'emergency_medical': emergency_medical, 'existing_tag': tag if competition.use_existing_tags else None, }): try: truncate_char_fields(license_holder).save() truncate_char_fields(license_holder) except Exception as e: ms_write( u'**** Row {}: Update License Holder Exception: {}, Name="{}"\n' .format( i, e, name, )) continue #------------------------------------------------------------------------------ # Get Category. Open categories will match either Gender. # category = None if category_code: tt.start('get_category_from_code') category = get_category(category_code, license_holder.gender) if category is None: ms_write( u'**** Row {}: cannot match Category (ignoring): "{}" Name="{}"\n' .format( i, category_code, name, )) #------------------------------------------------------------------------------ # Get Team # tt.start('get_team') team = None if Team.is_independent_name(team_name): team = None else: if team_name not in team_lookup: msg = u'Row {:>6}: Added team: {}\n'.format( i, team_name, ) ms_write(msg) team = team_lookup[team_name] #------------------------------------------------------------------------------ tt.start('get_participant') participant_keys = { 'competition': competition, 'license_holder': license_holder, } if category is not None: participant_keys['category'] = category try: participant = Participant.objects.get(**participant_keys) except Participant.DoesNotExist: participant = Participant(**participant_keys) except Participant.MultipleObjectsReturned: ms_write( u'**** Row {}: found multiple Participants for this license_holder, Name="{}".\n' .format( i, name, )) continue # First get the defaults based on previous hints and categories. tt.start('init_participant_defaults') participant.init_default_values() # Now, override default values with specified ones. tt.start('override_participant_defaults') for attr, value in ( ('category', category), ('bib', bib), ('tag', tag), ('note', note), ('preregistered', preregistered), ('paid', paid), ('seed_option', seed_option), ('est_kmh', est_kmh), ('role', role), ('license_checked', license_checked), ): if value is not None: setattr(participant, attr, value) if team_name: participant.team = team # Ensure the existing bib number is compatible with the category. tt.start('update_bib_new_category') participant.update_bib_new_category() # Only auto-assign a bib only if there isn't one already. if bib_auto and not participant.bib: tt.start('get_bib_auto') bib = participant.get_bib_auto() # If we have an assigned bib, ensure it is respected. tt.start('validate_bib') if bib and bib in category_numbers_set[participant.category]: participant.bib = bib # Add the assigned bib to the number set if it exists. if competition.number_set: competition.number_set.assign_bib( participant.license_holder, participant.bib) # Do a final check for bib compatibility. if participant.bib and participant.bib not in category_numbers_set[ participant.category]: participant.bib = None participant.preregistered = True tt.start('participant_save') try: truncate_char_fields(participant).save() except IntegrityError as e: ms_write( u'**** Row {}: Error={}\nBib={} Category={} License={} Name="{}"\n' .format( i, e, bib, category_code, license_code, name, )) success, integrity_error_message, conflict_participant = participant.explain_integrity_error( ) if success: ms_write(u'{}\n'.format(integrity_error_message)) ms_write(u'{}\n'.format(conflict_participant)) continue tt.start('add_to_default_optional_events') participant.add_to_default_optional_events() if participant_optional_events: participant_optional_events = { event: (included and event.could_participate(participant)) for event, included in participant_optional_events } option_included = { event.option_id: included for event, included in participant_optional_events.iteritems() } ParticipantOption.sync_option_ids(participant, option_included) override_events_str = u' ' + u', '.join( u'"{}"={}'.format(event.name, included) for event, included in sorted( participant_optional_events.iteritems())) else: override_events_str = '' if waiver is not None: tt.start('waiver') if waiver: participant.sign_waiver_now(waiver_signed_date) else: participant.unsign_waiver_now() if not category and not participant_optional_events: tt.start('add_other_categories') participant.add_other_categories() tt.end() ms_write( u'Row {row:>6}: {license:>8} {dob:>10} {uci}, {lname}, {fname}, {city}, {state_prov} {ov}\n' .format( row=i, license=license_holder.license_code, dob=license_holder.date_of_birth.strftime('%Y-%m-%d'), uci=license_holder.uci_code, lname=license_holder.last_name, fname=license_holder.first_name, city=license_holder.city, state_prov=license_holder.state_prov, ov=override_events_str, )) sheet_name = None if worksheet_contents is not None: wb = open_workbook(file_contents=worksheet_contents) else: try: fname, sheet_name = worksheet_name.split('$') except: fname = worksheet_name wb = open_workbook(fname) ur_records = [] import_utils.datemode = wb.datemode ws = None for cur_sheet_name in wb.sheet_names(): if cur_sheet_name == sheet_name or sheet_name is None: ms_write(u'Reading sheet: {}\n'.format(cur_sheet_name)) ws = wb.sheet_by_name(cur_sheet_name) break if not ws: ms_write(u'Cannot find sheet "{}"\n'.format(sheet_name)) return num_rows = ws.nrows num_cols = ws.ncols for r in xrange(num_rows): row = ws.row(r) if r == 0: # Get the header fields from the first row. fields = [unicode(v.value).strip() for v in row] # Add all Optional Event patterns. for field in fields: try: pattern = normalize(field) except: continue for event_name, event in optional_events.iteritems(): if fnmatch(event_name, pattern): pattern_optional_events[pattern].append(event) for pattern in pattern_optional_events.iterkeys(): ifm.set_aliases(pattern, (pattern, )) ifm.set_headers(fields) ms_write(u'Header Row:\n') for col, f in enumerate(fields, 1): if f.lower().strip() in optional_events: ms_write(u' {}. {} (Optional Event)\n'.format( col, f)) else: name = ifm.get_name_from_alias(f) if name is not None: ms_write(u' {}. {} --> {}\n'.format( col, f, name)) else: ms_write(u' {}. ****{} (Ignored)\n'.format( col, f)) fields_lower = [f.lower() for f in fields] if clear_existing or 'bib' in ifm: ms_write(u'Recording previous license checkes...\n') if competition.report_label_license_check: tt.start('license_check_state_refresh') LicenseCheckState.refresh() ms_write(u'Clearing existing Participants...\n') tt.start('clearing_existing_participants') large_delete_all(Participant, Q(competition=competition)) if 'license_code' not in ifm and 'uci_id' not in ifm: ms_write( u'Header Row must contain one of (or both) License or UCI ID. Aborting.\n' ) return ms_write(u'\n') continue ur_records.append((r + 1, [v.value for v in row])) if len(ur_records) == 1000: process_ur_records(ur_records) ur_records = [] process_ur_records(ur_records) ms_write(u'\n') for section, total in sorted(times.iteritems(), key=lambda e: e[1], reverse=True): ms_write(u'{}={:.6f}\n'.format(section, total)) ms_write(u'\n') ms_write(u'Initialization in: {}\n'.format(datetime.datetime.now() - tstart)) ms_write(u'\n') ms_write(tt.__repr__())
def init_seasons_pass(seasonsPassId, worksheet_name='', worksheet_contents=None, message_stream=sys.stdout, clear_existing=False): tstart = datetime.datetime.now() if message_stream == sys.stdout or message_stream == sys.stderr: def ms_write(s): message_stream.write(removeDiacritic(s)) else: def ms_write(s): message_stream.write(unicode(s)) sys.stdout.write(removeDiacritic(s)) try: seasons_pass = SeasonsPass.objects.get(pk=seasonsPassId) except SeasonsPass.DoesNotExist: ms_write(u'**** Cannot find SeasonsPass\n') return if clear_existing: large_delete_all(SeasonsPassHolder, Q(seasons_pass=seasons_pass)) ifm = standard_field_map() # Process the records in large transactions for efficiency. @transaction.atomic def process_ur_records(ur_records): for i, ur in ur_records: v = ifm.finder(ur) date_of_birth = v('date_of_birth', None) try: date_of_birth = date_from_value(date_of_birth) except Exception as e: ms_write( 'Row {i:>6}: Ignoring birthdate (must be YYYY-MM-DD) "{}" ({}) {}' .format(i, date_of_birth, ur, e)) date_of_birth = None date_of_birth = date_of_birth if date_of_birth != invalid_date_of_birth else None license_code = to_int_str(v('license_code', u'')).upper().strip() last_name = to_str(v('last_name', u'')) first_name = to_str(v('first_name', u'')) uci_id = to_uci_id(v('uci_id', None)) license_holder = None if not license_holder and license_code: license_holder = LicenseHolder.objects.filter( license_code=license_code).first() else: if not license_holder and uci_id: license_holder = LicenseHolder.objects.filter( uci_id=uci_id).first() if not license_holder and last_name: q = Q(search_text__startswith=utils.get_search_text( [last_name, first_name])) if date_of_birth: q &= Q(date_of_birth=date_of_birth) license_holder = LicenseHolder.objects.filter(q).first() if not license_holder: ms_write( u'Row {i:>6}: Cannot find License Holder by License Code or LastName, FirstName [Date of Birth]\n' .format(i=i, )) continue seasons_pass.add(license_holder) ms_write( u'Row {i:>6}: {license_code:>8} {dob:>10} {last_name}, {first_name}, {city}, {state_prov}\n' .format( i=i, license_code=license_holder.license_code, dob=license_holder.date_of_birth.strftime('%Y-%m-%d'), last_name=license_holder.last_name, first_name=license_holder.first_name, city=license_holder.city, state_prov=license_holder.state_prov, )) sheet_name = None if worksheet_contents is not None: wb = open_workbook(file_contents=worksheet_contents) else: try: fname, sheet_name = worksheet_name.split('$') except: fname = worksheet_name wb = open_workbook(fname) ur_records = [] import_utils.datemode = wb.datemode ws = None for cur_sheet_name in wb.sheet_names(): ms_write(u'Reading sheet: {}\n'.format(cur_sheet_name)) ws = wb.sheet_by_name(cur_sheet_name) break if not ws: ms_write(u'Cannot find sheet.\n') return num_rows = ws.nrows num_cols = ws.ncols for r in xrange(num_rows): row = ws.row(r) if r == 0: # Get the header fields from the first row. fields = [unicode(f.value).strip() for f in row] ifm.set_headers(fields) ms_write(u'Header Row:\n') for col, f in enumerate(fields, 1): name = ifm.get_name_from_alias(f) if name is not None: ms_write(u' {}. {} --> {}\n'.format(col, f, name)) else: ms_write(u' {}. ****{} (Ignored)\n'.format(col, f)) continue ur_records.append((r + 1, [v.value for v in row])) if len(ur_records) == 300: process_ur_records(ur_records) ur_records = [] process_ur_records(ur_records) ms_write(u'\n') ms_write(u'Initialization in: {}\n'.format(datetime.datetime.now() - tstart))