def wrap(xtime, func, arg=None): """Catch programming logic errors.""" try: if xtime is None: if arg is not None: return func(arg) return func() if arg is not None: return func_timeout(xtime, func, args=[arg]) return func_timeout(xtime, func) except FunctionTimedOut: errmsg = traceback.format_exc(10) debug(errmsg, 1) nowtime = datetime.now().time() msg = f"{nowtime}: Abandon {func.__name__} due to timeout ({xtime} secs)" wrap_trace() messagebox.showerror("Error", msg) return False except ( AssertionError, AttributeError, LookupError, NameError, TypeError, ValueError, ) as err: errmsg = f"\nSCM Helper Version: {VERSION}.\n" errmsg += traceback.format_exc(10) debug(errmsg, 0) msg = f"Internal SCM Helper Error:\n{err}\nPlease log an issue on github.\n" wrap_trace() messagebox.showerror("Error", msg) return False
def api_read(self, url, page): """Read URL page.""" club = self._config[C_CLUB] user_agent = USER_AGENT.replace("###CLUB_NAME###", club) headers = { "User-Agent": user_agent, "Authorization-Token": self._key, "Page": str(page), } debug(f"URL:\n{url}", 9) debug(f"Headers:\n{headers}", 8) response = requests.get(url, headers=headers) if response.ok: return response.json() if response.status_code == 404: # Worked, but not found return False notify(f"\nErroring getting data from {url}, page:{page}\n") notify(response.reason) notify("\n") return None
def get_config(self, password): """Get API key.""" # pylint: disable=import-outside-toplevel if len(self._config) == 0: if self.get_config_file() is False: return False if self.ipad: from scm_helper.ipad import Crypto self.crypto = Crypto(self._config[C_CLUB], password) # Salt else: from scm_helper.crypto import Crypto self.crypto = Crypto(self._config[C_CLUB], password) # Salt home = str(Path.home()) keyfile = os.path.join(home, CONFIG_DIR, KEYFILE) self._key = self.crypto.read_key(keyfile) if self._key is None: return False debug_config = self.config(C_DEBUG_LEVEL) set_debug_level(debug_config) debug(f"Quarter offset: {self.q_offset}", 9) return True
def check_dbs(self, xtype): """Check DBS and Safeguarding.""" if self.print_exception(EXCEPTION_NODBS) is False: debug(f"DBS Exception ignored: {self.name}", 7) return dbs_date = self.set_date(A_DBS_RENEWAL_DATE) safe_date = self.set_date(A_SAFEGUARDING_RENEWAL_DATE) notice = get_config(self.scm, C_MEMBERS, C_DBS, C_EXPIRY) if dbs_date: days = (dbs_date - self.scm.today).days if days < 0: dbs_date_str = dbs_date.strftime(PRINT_DATE_FORMAT) issue(self, E_DBS_EXPIRED, f"{xtype}, expired {dbs_date_str}") elif days < notice: dbs_date_str = dbs_date.strftime(PRINT_DATE_FORMAT) issue(self, E_DBS_EXPIRED, f"{xtype}, expires {dbs_date_str}") else: issue(self, E_NO_DBS, f"{xtype}") if self.print_exception(EXCEPTION_NOSAFEGUARD) is False: debug(f"Safeguard Exception ignored: {self.name}", 7) return if safe_date: if (safe_date - self.scm.today).days < 0: safe_date_str = safe_date.strftime(PRINT_DATE_FORMAT) issue(self, E_SAFEGUARD_EXPIRED, f"{xtype}, expired {safe_date_str}") elif (safe_date - self.scm.today).days < notice: safe_date_str = safe_date.strftime(PRINT_DATE_FORMAT) issue(self, E_SAFEGUARD_EXPIRED, f"{xtype}, expires {safe_date_str}") else: issue(self, E_NO_SAFEGUARD, f"{xtype}")
def linkage(self, members): """Link members.""" if (A_MEMBERS in self.data) and (len(self.data[A_MEMBERS]) > 0): for swimmer in self.data[A_MEMBERS]: if swimmer[A_GUID] not in members.by_guid: msg = ( f"GUID {swimmer[A_GUID]} missing in list - email address only?" ) debug(msg, 7) continue guid = members.by_guid[swimmer[A_GUID]] if guid.is_active: self.members.append(guid) else: name = guid.name issue(self, E_INACTIVE, f"member {name}", 0, "Fixable") if self.newdata and (A_MEMBERS in self.newdata): fix = self.newdata else: fix = {} fix[A_MEMBERS] = self.data[A_MEMBERS].copy() remove = {A_GUID: guid.guid} fix[A_MEMBERS].remove(remove) self.fixit(fix, f"Delete {guid.name} (inactive)")
def print_summary(self, backup=False): """Print summary.""" debug("Print summary called", 6) output = "" for aclass in self.classes: output += aclass.print_summary() output += f" Not confirmed: {self.members.count_not_confirmed}\n" if backup and self.backup_classes: for aclass in self.backup_classes: output += aclass.print_summary() if self.option(O_FIX): # fixed them! return output if self.option(O_VERIFY): return output # fixable not available with backup data length = len(self.fixable) if length > 0: output += f"\n{length} fixable errors...\n" output += self.list_fixes() output += "\n" return output
def linkage2(self): """Link parents to swimmers.""" # Hack - work around API issue. # if a parent, make sure swimmers are linked back # pylint: disable=protected-access for swimmer in self._swimmers: if len(swimmer._parents) == 0: debug(f"Found swimmer - API error - recovered {swimmer.name}", 7) swimmer._parents.append(self)
def check_coach_permissions(coach, role): """Check a coaches permissions.""" # pylint: disable=too-many-branches debug(f"Permission check: {coach.name}, {role.name}", 7) if coach.is_coach is False: issue(coach, E_NOT_A_COACH, f"Role: {role.name} (fixable)") fix = {} fix[A_ISCOACH] = "1" coach.fixit(fix, "Add 'Is a coach'") coach.set_in_coach_role() if coach.is_swimmer is False: if len(coach.sessions) > 0: issue(coach, E_COACH_WITH_SESSIONS, f"Role: {role.name}") if coach.print_exception(EXCEPTION_PERMISSIONS) is False: return fix = {} fixed = False data = coach.data["SessionRestrictions"] if data: fix["SessionRestrictions"] = data.copy() else: fix["SessionRestrictions"] = [] msg = "Fix permissions:\n" for session in coach.coach_sessions: match = False for permission in coach.restricted: if session == permission: match = True break if match is False: issue(coach, E_PERMISSION_MISSING, session.full_name) fix["SessionRestrictions"].append({A_GUID: session.guid}) msg += f" Add {session.name}\n" fixed = True for permission in coach.restricted: match = False for session in coach.coach_sessions: if session == permission: match = True break if match is False: issue(coach, E_PERMISSION_EXTRA, permission.full_name) fix["SessionRestrictions"].remove({A_GUID: permission.guid}) fixed = True msg += f" Remove {session.name}\n" if fixed: coach.fixit(fix, msg)
def convert_time(xtime): """Convert a time to a number of seconds.""" try: hms = xtime.split(":") if len(hms) == 2: res = float(hms[0]) * 60 + float(hms[1]) else: res = float(hms[0]) return res except ValueError: debug(f"invalid time {xtime} ", 3) return 999999
def api_write(self, entity, create): """Write data back to SCM.""" club = self._config[C_CLUB] user_agent = USER_AGENT.replace("###CLUB_NAME###", club) headers = { "content-type": "application/json", "User-Agent": user_agent, "Authorization-Token": self._key, } if get_config(entity.scm, C_ALLOW_UPDATE) is False: notify("Update prohibited by config.\n") return None debug(f"URL:\n{entity.url}", 9) debug(f"Headers:\n{headers}", 8) data = entity.newdata if create: debug(f"Post request:\n{data}", 7) response = requests.post(entity.url, json=data, headers=headers) else: debug(f"Put request:\n{data}", 7) response = requests.put(entity.url, json=data, headers=headers) if response.ok: return response if response.status_code == 404: # Worked, but not found return False notify(f"\nErroring posting data {entity.name}\n") notify(response.reason) notify("\n") return None
def get_data(self, backup): """Get data.""" debug(f"(version: {VERSION})", 1) notify("Reading Data...\n") loop = self.classes if backup: loop = self.classes + self.backup_classes for aclass in loop: if aclass.get_data() is False: return False return True
def check_confirmed_diff(swimmer, parent): """Check for differences in swimmer and parent.""" # pylint: disable=R0911 # Need them all child_mon = 0 parent_mon = 0 if swimmer.confirmed_date: child_mon = int((swimmer.confirmed_date.month - 1) / 3) * 3 if parent.confirmed_date: parent_mon = int((parent.confirmed_date.month - 1) / 3) * 3 if swimmer.age > get_config(swimmer.scm, C_SWIMMERS, C_PARENT, C_MAX_AGE): return False if child_mon == parent_mon: return False prefix = "Different confirmed dates" postfix = "- checking other details for consistency" debug(f"{prefix} {swimmer.name}, {parent.name} {postfix}", 8) if swimmer.email != parent.email: debug(f"email: {swimmer.email}: {parent.email}", 8) return True if swimmer.homephone != parent.homephone: debug(f"phone: {swimmer.homephone}: {parent.homephone}", 8) return True if swimmer.mobilephone != parent.mobilephone: debug(f"mobile: {swimmer.mobilephone}: {parent.mobilephone}", 8) return True if swimmer.address != parent.address: debug(f"address: {swimmer.address}: {parent.address}", 8) return True # Dates are different, but core attributes same # So set the confirmed date - to inhibit a confirm notice to parent if parent.confirmed_date is None: parent.set_confirmed(swimmer.confirmed_date) return False if swimmer.confirmed_date: if swimmer.confirmed_date > parent.confirmed_date: parent.set_confirmed(swimmer.confirmed_date) return False
def check_member(browser, member): """Check a member.""" notify(f"Checking {member.name}...\n") try: name = browser.find_element_by_xpath( "//table[1]/tbody/tr[2]/td[2]").text knownas = browser.find_element_by_xpath( "//table[1]/tbody/tr[2]/td[4]").text gender = browser.find_element_by_xpath( "//table[1]/tbody/tr[3]/td[4]").text current = browser.find_element_by_xpath( "//table[2]/tbody/tr[2]/td[4]").text category = browser.find_element_by_xpath( "//table[2]/tbody/tr[3]/td[4]").text except selenium.common.exceptions.NoSuchElementException: if member.print_exception(EXCEPTION_SE_HIDDEN) is False: debug(f"SE Exception ignored: {member.name}", 7) return "" res = f"\n{member.name} ({member.asa_number}) does not exist in SE database.\n" return res res = "" if member.print_exception(EXCEPTION_SE_NAME) is True: if name.lower() != member.name.lower(): res += f" Name: SCM-> {member.name}, SE-> {name}\n" if knownas and (member.knownas_only != knownas): firstname = name.split(" ") if knownas != firstname[0]: # in SE they are the same if no knownas res += f" Knownas: SCM-> {member.knownas_only}, SE-> {knownas}\n" if member.gender and (gender != GENDER[member.gender]): res += f" Gender: SCM-> {member.name}, SE-> {gender}\n" mycat = f"SE Category {member.asa_category}" if category != mycat: res += f" Category: SCM-> {mycat}, SE-> {category}\n" if current != "Current": res += " Not current\n" if res: res = f"\n{member.name} ({member.asa_number}) mismatch:\n" + res return res
def get_data(self): """Get data.""" notify(f"{self._name}... ") data = self.scm.api_read(self._url, 1) if data is None: return False count = self.create_entities(data) # line below is subtly different, who's who data is already a list. self._raw_data = data notify(f"{count}\n") if count != 1: debug("Who's who assumption failure", 0) return True
def apply_fix(self): """Fix an entity.""" printer = pprint.PrettyPrinter(indent=4) data = printer.pformat(self.newdata) err = f"Fix '{self.name}' with:\n {self.fixmsg}\nConfirm" debug("fixit:", 7) debug(data, 8) resp = interact_yesno(err) if resp is False: return False self.newdata[A_GUID] = self.guid notify(f"Fixing: {self.name}...") res = self.scm.api_write(self, False) if res: notify("Success.\n") return res
def wrap_trace(): """Give as many error details as possible.""" tback = sys.exc_info()[2] while 1: if not tback.tb_next: break tback = tback.tb_next stack = [] xframe = tback.tb_frame while xframe: stack.append(xframe) xframe = xframe.f_back stack.reverse() for frame in stack: for key, value in frame.f_locals.items(): # Print likely to cause error itself, but should get enough out of it... try: debug(f" {key}: {value.name}", 1) # pylint: disable=bare-except except: # noqa: E722 continue
def check_duplicate(self, member): """See if member already exists before adding.""" firtname = member[A_FIRSTNAME] lastname = member[A_LASTNAME] name = f"{firtname} {lastname}" if name in self.by_name: if member[A_ACTIVE] == "1" and self.by_name[name].is_active: act1 = member[A_ACTIVE] act2 = self.by_name[name].is_active debug(f"{name}: {act1}-{act2}", 6) issue(self.by_name[name], E_DUPLICATE, name) else: active = self.by_name[name].is_active if member[A_ACTIVE] == "0" and active is False: issue(self.by_name[name], E_DUPLICATE, "BOTH inactive", 9) else: issue(self.by_name[name], E_DUPLICATE, "One is inactive", -1) return if name in self.knownas: if member[A_ACTIVE] == "1" and self.knownas[name].is_active: issue(self.knownas[name], E_DUPLICATE, name, 0, "(Known as)") else: issue(self.knownas[name], E_DUPLICATE, "One is inactive (Known as)", -1)
def get_notes(self): """Extract Facebook name from Notes.""" notes = self.notes if notes is None: return note = FACEBOOK_RE.findall(notes) if note: for facebook in note: facebook = facebook.strip() self.facebook.append(facebook) debug(f"Found Facebook name in notes '{facebook}'", 8) note = API_RE.findall(notes) if note is None: return for api in note: exclusion = API_TEXT_RE.search(api) expiry = DATE_RE.search(api) when = self.scm.today gotdate = False if expiry: date = expiry.group(0) when = get_date(date, "%d-%m-%Y") gotdate = True else: expiry = DATE2_RE.search(api) if expiry: date = expiry.group(0) when = get_date(date, "%d/%m/%Y") gotdate = True if when: excl = exclusion.group(0).strip() if (self.scm.today - when).days <= 0: self.ignore_errors.append(excl) debug(f"Found API token in notes {api}", 8) else: debug(f"Token expired {api}", 8) elif gotdate: issue(self, E_DATE, f"Notes: {api}")
def analyse(self): """Analyse the group.""" # pylint: disable=too-many-branches # pylint: disable=too-many-statements # pylint: disable=too-many-locals no_session = False check_dbs = False wanted_session = None allowed = None xtype = None ignore = None confirm = None if self.config: ignore = self.config_item(C_IGNORE_GROUP) no_session = self.config_item(C_NO_SESSIONS) check_dbs = self.config_item(C_CHECK_DBS) wanted_sessions = self.config_item(C_SESSIONS) allowed = self.config_item(C_NO_SESSION_ALLOWED) xtype = self.config_item(C_TYPE) confirm = self.config_item(C_CONFIRMATION) if ignore: debug(f"Ignoring group {self.name}", 7) return if len(self.members) == 0: issue(self, E_NO_SWIMMERS, "Group") return if no_session: for member in self.members: if len(member.sessions) > 0: name = member.sessions[0].name issue(member, E_SESSIONS, f"Group: {self.name}, Session: {name}") if confirm: try: date = datetime.datetime.strptime(confirm, SCM_CSV_DATE_FORMAT) confirm = date except ValueError: notify( f"*** Error in date format in config file for groups config: {confirm} ***\n" ) confirm = None for member in self.members: self.check_age(member) if check_dbs: member.check_dbs(self.name) if confirm: err = False if member.confirmed_date: gap = (confirm - member.confirmed_date).days if gap >= 0: err = True else: err = True if err: issue(member, E_CONFIRMATION_EXPIRED, f"Group: {self.name}") msg = f"Confirmation Expired for Group: {member.name}" member.scm.lists.add(msg, member) if member.newstarter: continue if wanted_session: for session in wanted_sessions: if check_in_session(member, session, allowed) is False: res1 = self.print_exception( EXCEPTION_NONSWIMMINGMASTER) res2 = self.print_exception(EXCEPTION_GROUPNOSESSION) if res1 or res2: issue(member, E_NOT_IN_SESSION, f"Group: {self.name}") break if xtype: if check_type(member, xtype): continue if xtype == CTYPE_SWIMMER: # if swimmers wanted, allow it to be a coach if check_type(member, CTYPE_COACH) is True: continue msg = f"Group: {self.name}, Type required: {xtype} (fixable)" issue(member, E_TYPE, msg) fix = {} attr = None if xtype == CTYPE_MASTER: attr = "Masters" if xtype == CTYPE_SWIMMER: attr = "IsASwimmer" if xtype == CTYPE_SYNCHRO: attr = "SynchronisedSwimming" if xtype == CTYPE_COACH: attr = "IsACoach" if xtype == CTYPE_POLO: attr = "WaterPolo" if attr: fix[attr] = "1" member.fixit(fix, f"Add type: {attr}")
def analyse_enter(self, event): """Window for analysis result.""" debug(f"Event: {event}", 7) self.analyse_window()
def run(self): """Run analyser.""" self.gui.set_buttons(DISABLED) if self.gui.issue_window: self.gui.issue_text.config(state=NORMAL) self.gui.issue_text.delete("1.0", END) self.gui.notify_text.delete("1.0", END) if self.scm.get_config_file() is False: messagebox.showerror("Error", "Error in config file.") self.gui.set_buttons(NORMAL) return if self.archive: home = str(Path.home()) backup = os.path.join(home, CONFIG_DIR, BACKUP_DIR) dir_opt = {} dir_opt["initialdir"] = backup dir_opt["mustexist"] = True dir_opt["parent"] = self.gui.master where = filedialog.askdirectory(**dir_opt) if wrap(None, self.scm.decrypt, where) is False: messagebox.showerror("Error", f"Cannot read from archive: {where}") self.gui.master.after(AFTER, self.gui.set_normal) self.gui.thread = False return else: if wrap(None, self.scm.get_data, False) is False: messagebox.showerror("Analysis", "Failed to read data") self.gui.master.after(AFTER, self.gui.set_normal) self.gui.thread = False return if wrap(10, self.scm.linkage) is False: self.gui.master.after(AFTER, self.gui.set_normal) self.gui.thread = False return if wrap(10, self.scm.analyse) is False: self.gui.master.after(AFTER, self.gui.set_normal) self.gui.thread = False return self.gui.gotdata = True if self.gui.issue_window is None: self.gui.create_issue_window() debug("Analyse returned - creating result window", 8) output = self.gui.issues.print_by_error(None) result = self.scm.print_summary() self.gui.notify_text.insert(END, result) self.gui.notify_text.see(END) self.gui.issue_text.insert(END, output) self.gui.master.update_idletasks() self.gui.issue_window.lift() self.gui.master.after(AFTER, self.gui.set_normal) self.gui.thread = False debug("Analyse Thread complete, result posted", 8) return
def process_row(self, row, count): """Process and merge a row into records.""" # pylint: disable=too-many-locals # pylint: disable=too-many-return-statements # pylint: disable=too-many-branches # pylint: disable=too-many-statements if "Swimmer" not in row: if count == 1: notify("Is the header line missing in the CSV?\n") return swimmer = row["Swimmer"] asa = row["SE Number"] xdate = row["Date"] pool = row["Pool Size"] dist = row["Swim Distance"] stroke = row["Stroke"] timestr = row["Time"] relay = row["Relay"] location = row["Location"] gender = row["Gender"] gala = row["Gala"] swimage = None if row["Age"]: swimage = int(row["Age"]) if "DQ" in timestr: return if "NT" in timestr: return if dist == "25m": return if relay == "Yes": return if pool not in ("50", "25"): return if gala: location = gala elif location: pass else: location = "Unknown" if dist not in DISTANCE: debug(f"Line {count}: Unknown distance {dist}", 1) return if stroke not in STROKES: debug(f"Line {count}: Unknown stroke {stroke}", 1) return verify = get_config(self.scm, C_RECORDS, C_VERIFY) age_eoy = get_config(self.scm, C_RECORDS, C_AGE_EOY) se_only = get_config(self.scm, C_RECORDS, C_SE_ONLY) all_ages = get_config(self.scm, C_RECORDS, C_ALL_AGES) member = None if asa not in self.scm.members.by_asa: debug(f"Line {count}: No SE Number {swimmer}", 2) # We can't check, so go with it... if se_only: return verify = False age_eoy = False else: member = self.scm.members.by_asa[asa] swimmer = member.knownas # for consistency of spelling if swimage and swimage >= 25: age_eoy = True # Masters are always EOY try: swimdate = datetime.datetime.strptime(xdate, SCM_CSV_DATE_FORMAT) except: swimdate = datetime.datetime.strptime(xdate, SCM_ALT_CSV_DATE_FORMAT) if member and age_eoy: yob = member.dob.year swimyear = swimdate.year swimage = swimyear - yob if verify and member and member.date_joined and (swimdate < member.date_joined): debug(f"Line {count}: Ignored, not a member at time of swim", 2) return if swimage is None: return if swimage < 18: start_age = int(swimage / 2) * 2 end_age = start_age + 1 elif swimage in (18, 19): if all_ages: start_age = 19 else: start_age = 18 end_age = 24 else: start_age = int(swimage / 5) * 5 # round it if start_age == 20: if all_ages: start_age = 19 else: start_age = 18 end_age = 24 else: end_age = start_age + 4 agegroup = f"{start_age}-{end_age}" if all_ages: if swimage <= 18: agegroup = str(swimage) ALL_AGES[agegroup] += 1 else: AGES[agegroup] += 1 event = f"{gender} {agegroup} {dist} {stroke} {pool}" swim = { S_EVENT: event, S_ASA: asa, S_NAME: swimmer, S_TIMESTR: timestr, S_FTIME: convert_time(timestr), S_LOCATION: location, S_DATE: xdate, } self.records.check_swim(swim) return
def add_group(self, group): """Add a group to the swimmer.""" debug(f"Added {self.name} to {group.name}", 9) self.groups.append(group)