def wrap(xtime, func, arg=None):
"""Catch programming logic errors."""
try:
if xtime is None:
if arg is not None:
return func(arg)
return func()
if arg is not None:
return func_timeout(xtime, func, args=[arg])
return func_timeout(xtime, func)
except FunctionTimedOut:
errmsg = traceback.format_exc(10)
debug(errmsg, 1)
nowtime = datetime.now().time()
msg = f"{nowtime}: Abandon {func.__name__} due to timeout ({xtime} secs)"
wrap_trace()
messagebox.showerror("Error", msg)
return False
except (
AssertionError,
AttributeError,
LookupError,
NameError,
TypeError,
ValueError,
) as err:
errmsg = f"\nSCM Helper Version: {VERSION}.\n"
errmsg += traceback.format_exc(10)
debug(errmsg, 0)
msg = f"Internal SCM Helper Error:\n{err}\nPlease log an issue on github.\n"
wrap_trace()
messagebox.showerror("Error", msg)
return False
def api_read(self, url, page):
"""Read URL page."""
club = self._config[C_CLUB]
user_agent = USER_AGENT.replace("###CLUB_NAME###", club)
headers = {
"User-Agent": user_agent,
"Authorization-Token": self._key,
"Page": str(page),
}
debug(f"URL:\n{url}", 9)
debug(f"Headers:\n{headers}", 8)
response = requests.get(url, headers=headers)
if response.ok:
return response.json()
if response.status_code == 404: # Worked, but not found
return False
notify(f"\nErroring getting data from {url}, page:{page}\n")
notify(response.reason)
notify("\n")
return None
def get_config(self, password):
"""Get API key."""
# pylint: disable=import-outside-toplevel
if len(self._config) == 0:
if self.get_config_file() is False:
return False
if self.ipad:
from scm_helper.ipad import Crypto
self.crypto = Crypto(self._config[C_CLUB], password) # Salt
else:
from scm_helper.crypto import Crypto
self.crypto = Crypto(self._config[C_CLUB], password) # Salt
home = str(Path.home())
keyfile = os.path.join(home, CONFIG_DIR, KEYFILE)
self._key = self.crypto.read_key(keyfile)
if self._key is None:
return False
debug_config = self.config(C_DEBUG_LEVEL)
set_debug_level(debug_config)
debug(f"Quarter offset: {self.q_offset}", 9)
return True
def check_dbs(self, xtype):
"""Check DBS and Safeguarding."""
if self.print_exception(EXCEPTION_NODBS) is False:
debug(f"DBS Exception ignored: {self.name}", 7)
return
dbs_date = self.set_date(A_DBS_RENEWAL_DATE)
safe_date = self.set_date(A_SAFEGUARDING_RENEWAL_DATE)
notice = get_config(self.scm, C_MEMBERS, C_DBS, C_EXPIRY)
if dbs_date:
days = (dbs_date - self.scm.today).days
if days < 0:
dbs_date_str = dbs_date.strftime(PRINT_DATE_FORMAT)
issue(self, E_DBS_EXPIRED, f"{xtype}, expired {dbs_date_str}")
elif days < notice:
dbs_date_str = dbs_date.strftime(PRINT_DATE_FORMAT)
issue(self, E_DBS_EXPIRED, f"{xtype}, expires {dbs_date_str}")
else:
issue(self, E_NO_DBS, f"{xtype}")
if self.print_exception(EXCEPTION_NOSAFEGUARD) is False:
debug(f"Safeguard Exception ignored: {self.name}", 7)
return
if safe_date:
if (safe_date - self.scm.today).days < 0:
safe_date_str = safe_date.strftime(PRINT_DATE_FORMAT)
issue(self, E_SAFEGUARD_EXPIRED, f"{xtype}, expired {safe_date_str}")
elif (safe_date - self.scm.today).days < notice:
safe_date_str = safe_date.strftime(PRINT_DATE_FORMAT)
issue(self, E_SAFEGUARD_EXPIRED, f"{xtype}, expires {safe_date_str}")
else:
issue(self, E_NO_SAFEGUARD, f"{xtype}")
def linkage(self, members):
"""Link members."""
if (A_MEMBERS in self.data) and (len(self.data[A_MEMBERS]) > 0):
for swimmer in self.data[A_MEMBERS]:
if swimmer[A_GUID] not in members.by_guid:
msg = (
f"GUID {swimmer[A_GUID]} missing in list - email address only?"
)
debug(msg, 7)
continue
guid = members.by_guid[swimmer[A_GUID]]
if guid.is_active:
self.members.append(guid)
else:
name = guid.name
issue(self, E_INACTIVE, f"member {name}", 0, "Fixable")
if self.newdata and (A_MEMBERS in self.newdata):
fix = self.newdata
else:
fix = {}
fix[A_MEMBERS] = self.data[A_MEMBERS].copy()
remove = {A_GUID: guid.guid}
fix[A_MEMBERS].remove(remove)
self.fixit(fix, f"Delete {guid.name} (inactive)")
def print_summary(self, backup=False):
"""Print summary."""
debug("Print summary called", 6)
output = ""
for aclass in self.classes:
output += aclass.print_summary()
output += f" Not confirmed: {self.members.count_not_confirmed}\n"
if backup and self.backup_classes:
for aclass in self.backup_classes:
output += aclass.print_summary()
if self.option(O_FIX): # fixed them!
return output
if self.option(O_VERIFY):
return output # fixable not available with backup data
length = len(self.fixable)
if length > 0:
output += f"\n{length} fixable errors...\n"
output += self.list_fixes()
output += "\n"
return output
def linkage2(self):
"""Link parents to swimmers."""
# Hack - work around API issue.
# if a parent, make sure swimmers are linked back
# pylint: disable=protected-access
for swimmer in self._swimmers:
if len(swimmer._parents) == 0:
debug(f"Found swimmer - API error - recovered {swimmer.name}", 7)
swimmer._parents.append(self)
def check_coach_permissions(coach, role):
"""Check a coaches permissions."""
# pylint: disable=too-many-branches
debug(f"Permission check: {coach.name}, {role.name}", 7)
if coach.is_coach is False:
issue(coach, E_NOT_A_COACH, f"Role: {role.name} (fixable)")
fix = {}
fix[A_ISCOACH] = "1"
coach.fixit(fix, "Add 'Is a coach'")
coach.set_in_coach_role()
if coach.is_swimmer is False:
if len(coach.sessions) > 0:
issue(coach, E_COACH_WITH_SESSIONS, f"Role: {role.name}")
if coach.print_exception(EXCEPTION_PERMISSIONS) is False:
return
fix = {}
fixed = False
data = coach.data["SessionRestrictions"]
if data:
fix["SessionRestrictions"] = data.copy()
else:
fix["SessionRestrictions"] = []
msg = "Fix permissions:\n"
for session in coach.coach_sessions:
match = False
for permission in coach.restricted:
if session == permission:
match = True
break
if match is False:
issue(coach, E_PERMISSION_MISSING, session.full_name)
fix["SessionRestrictions"].append({A_GUID: session.guid})
msg += f" Add {session.name}\n"
fixed = True
for permission in coach.restricted:
match = False
for session in coach.coach_sessions:
if session == permission:
match = True
break
if match is False:
issue(coach, E_PERMISSION_EXTRA, permission.full_name)
fix["SessionRestrictions"].remove({A_GUID: permission.guid})
fixed = True
msg += f" Remove {session.name}\n"
if fixed:
coach.fixit(fix, msg)
def convert_time(xtime):
"""Convert a time to a number of seconds."""
try:
hms = xtime.split(":")
if len(hms) == 2:
res = float(hms[0]) * 60 + float(hms[1])
else:
res = float(hms[0])
return res
except ValueError:
debug(f"invalid time {xtime} ", 3)
return 999999
def api_write(self, entity, create):
"""Write data back to SCM."""
club = self._config[C_CLUB]
user_agent = USER_AGENT.replace("###CLUB_NAME###", club)
headers = {
"content-type": "application/json",
"User-Agent": user_agent,
"Authorization-Token": self._key,
}
if get_config(entity.scm, C_ALLOW_UPDATE) is False:
notify("Update prohibited by config.\n")
return None
debug(f"URL:\n{entity.url}", 9)
debug(f"Headers:\n{headers}", 8)
data = entity.newdata
if create:
debug(f"Post request:\n{data}", 7)
response = requests.post(entity.url, json=data, headers=headers)
else:
debug(f"Put request:\n{data}", 7)
response = requests.put(entity.url, json=data, headers=headers)
if response.ok:
return response
if response.status_code == 404: # Worked, but not found
return False
notify(f"\nErroring posting data {entity.name}\n")
notify(response.reason)
notify("\n")
return None
def get_data(self, backup):
"""Get data."""
debug(f"(version: {VERSION})", 1)
notify("Reading Data...\n")
loop = self.classes
if backup:
loop = self.classes + self.backup_classes
for aclass in loop:
if aclass.get_data() is False:
return False
return True
def check_confirmed_diff(swimmer, parent):
"""Check for differences in swimmer and parent."""
# pylint: disable=R0911
# Need them all
child_mon = 0
parent_mon = 0
if swimmer.confirmed_date:
child_mon = int((swimmer.confirmed_date.month - 1) / 3) * 3
if parent.confirmed_date:
parent_mon = int((parent.confirmed_date.month - 1) / 3) * 3
if swimmer.age > get_config(swimmer.scm, C_SWIMMERS, C_PARENT, C_MAX_AGE):
return False
if child_mon == parent_mon:
return False
prefix = "Different confirmed dates"
postfix = "- checking other details for consistency"
debug(f"{prefix} {swimmer.name}, {parent.name} {postfix}", 8)
if swimmer.email != parent.email:
debug(f"email: {swimmer.email}: {parent.email}", 8)
return True
if swimmer.homephone != parent.homephone:
debug(f"phone: {swimmer.homephone}: {parent.homephone}", 8)
return True
if swimmer.mobilephone != parent.mobilephone:
debug(f"mobile: {swimmer.mobilephone}: {parent.mobilephone}", 8)
return True
if swimmer.address != parent.address:
debug(f"address: {swimmer.address}: {parent.address}", 8)
return True
# Dates are different, but core attributes same
# So set the confirmed date - to inhibit a confirm notice to parent
if parent.confirmed_date is None:
parent.set_confirmed(swimmer.confirmed_date)
return False
if swimmer.confirmed_date:
if swimmer.confirmed_date > parent.confirmed_date:
parent.set_confirmed(swimmer.confirmed_date)
return False
def check_member(browser, member):
"""Check a member."""
notify(f"Checking {member.name}...\n")
try:
name = browser.find_element_by_xpath(
"//table[1]/tbody/tr[2]/td[2]").text
knownas = browser.find_element_by_xpath(
"//table[1]/tbody/tr[2]/td[4]").text
gender = browser.find_element_by_xpath(
"//table[1]/tbody/tr[3]/td[4]").text
current = browser.find_element_by_xpath(
"//table[2]/tbody/tr[2]/td[4]").text
category = browser.find_element_by_xpath(
"//table[2]/tbody/tr[3]/td[4]").text
except selenium.common.exceptions.NoSuchElementException:
if member.print_exception(EXCEPTION_SE_HIDDEN) is False:
debug(f"SE Exception ignored: {member.name}", 7)
return ""
res = f"\n{member.name} ({member.asa_number}) does not exist in SE database.\n"
return res
res = ""
if member.print_exception(EXCEPTION_SE_NAME) is True:
if name.lower() != member.name.lower():
res += f" Name: SCM-> {member.name}, SE-> {name}\n"
if knownas and (member.knownas_only != knownas):
firstname = name.split(" ")
if knownas != firstname[0]: # in SE they are the same if no knownas
res += f" Knownas: SCM-> {member.knownas_only}, SE-> {knownas}\n"
if member.gender and (gender != GENDER[member.gender]):
res += f" Gender: SCM-> {member.name}, SE-> {gender}\n"
mycat = f"SE Category {member.asa_category}"
if category != mycat:
res += f" Category: SCM-> {mycat}, SE-> {category}\n"
if current != "Current":
res += " Not current\n"
if res:
res = f"\n{member.name} ({member.asa_number}) mismatch:\n" + res
return res
def get_data(self):
"""Get data."""
notify(f"{self._name}... ")
data = self.scm.api_read(self._url, 1)
if data is None:
return False
count = self.create_entities(data)
# line below is subtly different, who's who data is already a list.
self._raw_data = data
notify(f"{count}\n")
if count != 1:
debug("Who's who assumption failure", 0)
return True
def apply_fix(self):
"""Fix an entity."""
printer = pprint.PrettyPrinter(indent=4)
data = printer.pformat(self.newdata)
err = f"Fix '{self.name}' with:\n {self.fixmsg}\nConfirm"
debug("fixit:", 7)
debug(data, 8)
resp = interact_yesno(err)
if resp is False:
return False
self.newdata[A_GUID] = self.guid
notify(f"Fixing: {self.name}...")
res = self.scm.api_write(self, False)
if res:
notify("Success.\n")
return res
def wrap_trace():
"""Give as many error details as possible."""
tback = sys.exc_info()[2]
while 1:
if not tback.tb_next:
break
tback = tback.tb_next
stack = []
xframe = tback.tb_frame
while xframe:
stack.append(xframe)
xframe = xframe.f_back
stack.reverse()
for frame in stack:
for key, value in frame.f_locals.items():
# Print likely to cause error itself, but should get enough out of it...
try:
debug(f" {key}: {value.name}", 1)
# pylint: disable=bare-except
except: # noqa: E722
continue
def check_duplicate(self, member):
"""See if member already exists before adding."""
firtname = member[A_FIRSTNAME]
lastname = member[A_LASTNAME]
name = f"{firtname} {lastname}"
if name in self.by_name:
if member[A_ACTIVE] == "1" and self.by_name[name].is_active:
act1 = member[A_ACTIVE]
act2 = self.by_name[name].is_active
debug(f"{name}: {act1}-{act2}", 6)
issue(self.by_name[name], E_DUPLICATE, name)
else:
active = self.by_name[name].is_active
if member[A_ACTIVE] == "0" and active is False:
issue(self.by_name[name], E_DUPLICATE, "BOTH inactive", 9)
else:
issue(self.by_name[name], E_DUPLICATE, "One is inactive", -1)
return
if name in self.knownas:
if member[A_ACTIVE] == "1" and self.knownas[name].is_active:
issue(self.knownas[name], E_DUPLICATE, name, 0, "(Known as)")
else:
issue(self.knownas[name], E_DUPLICATE, "One is inactive (Known as)", -1)
def get_notes(self):
"""Extract Facebook name from Notes."""
notes = self.notes
if notes is None:
return
note = FACEBOOK_RE.findall(notes)
if note:
for facebook in note:
facebook = facebook.strip()
self.facebook.append(facebook)
debug(f"Found Facebook name in notes '{facebook}'", 8)
note = API_RE.findall(notes)
if note is None:
return
for api in note:
exclusion = API_TEXT_RE.search(api)
expiry = DATE_RE.search(api)
when = self.scm.today
gotdate = False
if expiry:
date = expiry.group(0)
when = get_date(date, "%d-%m-%Y")
gotdate = True
else:
expiry = DATE2_RE.search(api)
if expiry:
date = expiry.group(0)
when = get_date(date, "%d/%m/%Y")
gotdate = True
if when:
excl = exclusion.group(0).strip()
if (self.scm.today - when).days <= 0:
self.ignore_errors.append(excl)
debug(f"Found API token in notes {api}", 8)
else:
debug(f"Token expired {api}", 8)
elif gotdate:
issue(self, E_DATE, f"Notes: {api}")
def analyse(self):
"""Analyse the group."""
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
# pylint: disable=too-many-locals
no_session = False
check_dbs = False
wanted_session = None
allowed = None
xtype = None
ignore = None
confirm = None
if self.config:
ignore = self.config_item(C_IGNORE_GROUP)
no_session = self.config_item(C_NO_SESSIONS)
check_dbs = self.config_item(C_CHECK_DBS)
wanted_sessions = self.config_item(C_SESSIONS)
allowed = self.config_item(C_NO_SESSION_ALLOWED)
xtype = self.config_item(C_TYPE)
confirm = self.config_item(C_CONFIRMATION)
if ignore:
debug(f"Ignoring group {self.name}", 7)
return
if len(self.members) == 0:
issue(self, E_NO_SWIMMERS, "Group")
return
if no_session:
for member in self.members:
if len(member.sessions) > 0:
name = member.sessions[0].name
issue(member, E_SESSIONS,
f"Group: {self.name}, Session: {name}")
if confirm:
try:
date = datetime.datetime.strptime(confirm, SCM_CSV_DATE_FORMAT)
confirm = date
except ValueError:
notify(
f"*** Error in date format in config file for groups config: {confirm} ***\n"
)
confirm = None
for member in self.members:
self.check_age(member)
if check_dbs:
member.check_dbs(self.name)
if confirm:
err = False
if member.confirmed_date:
gap = (confirm - member.confirmed_date).days
if gap >= 0:
err = True
else:
err = True
if err:
issue(member, E_CONFIRMATION_EXPIRED,
f"Group: {self.name}")
msg = f"Confirmation Expired for Group: {member.name}"
member.scm.lists.add(msg, member)
if member.newstarter:
continue
if wanted_session:
for session in wanted_sessions:
if check_in_session(member, session, allowed) is False:
res1 = self.print_exception(
EXCEPTION_NONSWIMMINGMASTER)
res2 = self.print_exception(EXCEPTION_GROUPNOSESSION)
if res1 or res2:
issue(member, E_NOT_IN_SESSION,
f"Group: {self.name}")
break
if xtype:
if check_type(member, xtype):
continue
if xtype == CTYPE_SWIMMER: # if swimmers wanted, allow it to be a coach
if check_type(member, CTYPE_COACH) is True:
continue
msg = f"Group: {self.name}, Type required: {xtype} (fixable)"
issue(member, E_TYPE, msg)
fix = {}
attr = None
if xtype == CTYPE_MASTER:
attr = "Masters"
if xtype == CTYPE_SWIMMER:
attr = "IsASwimmer"
if xtype == CTYPE_SYNCHRO:
attr = "SynchronisedSwimming"
if xtype == CTYPE_COACH:
attr = "IsACoach"
if xtype == CTYPE_POLO:
attr = "WaterPolo"
if attr:
fix[attr] = "1"
member.fixit(fix, f"Add type: {attr}")
def analyse_enter(self, event):
"""Window for analysis result."""
debug(f"Event: {event}", 7)
self.analyse_window()
def run(self):
"""Run analyser."""
self.gui.set_buttons(DISABLED)
if self.gui.issue_window:
self.gui.issue_text.config(state=NORMAL)
self.gui.issue_text.delete("1.0", END)
self.gui.notify_text.delete("1.0", END)
if self.scm.get_config_file() is False:
messagebox.showerror("Error", "Error in config file.")
self.gui.set_buttons(NORMAL)
return
if self.archive:
home = str(Path.home())
backup = os.path.join(home, CONFIG_DIR, BACKUP_DIR)
dir_opt = {}
dir_opt["initialdir"] = backup
dir_opt["mustexist"] = True
dir_opt["parent"] = self.gui.master
where = filedialog.askdirectory(**dir_opt)
if wrap(None, self.scm.decrypt, where) is False:
messagebox.showerror("Error",
f"Cannot read from archive: {where}")
self.gui.master.after(AFTER, self.gui.set_normal)
self.gui.thread = False
return
else:
if wrap(None, self.scm.get_data, False) is False:
messagebox.showerror("Analysis", "Failed to read data")
self.gui.master.after(AFTER, self.gui.set_normal)
self.gui.thread = False
return
if wrap(10, self.scm.linkage) is False:
self.gui.master.after(AFTER, self.gui.set_normal)
self.gui.thread = False
return
if wrap(10, self.scm.analyse) is False:
self.gui.master.after(AFTER, self.gui.set_normal)
self.gui.thread = False
return
self.gui.gotdata = True
if self.gui.issue_window is None:
self.gui.create_issue_window()
debug("Analyse returned - creating result window", 8)
output = self.gui.issues.print_by_error(None)
result = self.scm.print_summary()
self.gui.notify_text.insert(END, result)
self.gui.notify_text.see(END)
self.gui.issue_text.insert(END, output)
self.gui.master.update_idletasks()
self.gui.issue_window.lift()
self.gui.master.after(AFTER, self.gui.set_normal)
self.gui.thread = False
debug("Analyse Thread complete, result posted", 8)
return
def process_row(self, row, count):
"""Process and merge a row into records."""
# pylint: disable=too-many-locals
# pylint: disable=too-many-return-statements
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
if "Swimmer" not in row:
if count == 1:
notify("Is the header line missing in the CSV?\n")
return
swimmer = row["Swimmer"]
asa = row["SE Number"]
xdate = row["Date"]
pool = row["Pool Size"]
dist = row["Swim Distance"]
stroke = row["Stroke"]
timestr = row["Time"]
relay = row["Relay"]
location = row["Location"]
gender = row["Gender"]
gala = row["Gala"]
swimage = None
if row["Age"]:
swimage = int(row["Age"])
if "DQ" in timestr:
return
if "NT" in timestr:
return
if dist == "25m":
return
if relay == "Yes":
return
if pool not in ("50", "25"):
return
if gala:
location = gala
elif location:
pass
else:
location = "Unknown"
if dist not in DISTANCE:
debug(f"Line {count}: Unknown distance {dist}", 1)
return
if stroke not in STROKES:
debug(f"Line {count}: Unknown stroke {stroke}", 1)
return
verify = get_config(self.scm, C_RECORDS, C_VERIFY)
age_eoy = get_config(self.scm, C_RECORDS, C_AGE_EOY)
se_only = get_config(self.scm, C_RECORDS, C_SE_ONLY)
all_ages = get_config(self.scm, C_RECORDS, C_ALL_AGES)
member = None
if asa not in self.scm.members.by_asa:
debug(f"Line {count}: No SE Number {swimmer}", 2)
# We can't check, so go with it...
if se_only:
return
verify = False
age_eoy = False
else:
member = self.scm.members.by_asa[asa]
swimmer = member.knownas # for consistency of spelling
if swimage and swimage >= 25:
age_eoy = True # Masters are always EOY
try:
swimdate = datetime.datetime.strptime(xdate, SCM_CSV_DATE_FORMAT)
except:
swimdate = datetime.datetime.strptime(xdate,
SCM_ALT_CSV_DATE_FORMAT)
if member and age_eoy:
yob = member.dob.year
swimyear = swimdate.year
swimage = swimyear - yob
if verify and member and member.date_joined and (swimdate <
member.date_joined):
debug(f"Line {count}: Ignored, not a member at time of swim", 2)
return
if swimage is None:
return
if swimage < 18:
start_age = int(swimage / 2) * 2
end_age = start_age + 1
elif swimage in (18, 19):
if all_ages:
start_age = 19
else:
start_age = 18
end_age = 24
else:
start_age = int(swimage / 5) * 5
# round it
if start_age == 20:
if all_ages:
start_age = 19
else:
start_age = 18
end_age = 24
else:
end_age = start_age + 4
agegroup = f"{start_age}-{end_age}"
if all_ages:
if swimage <= 18:
agegroup = str(swimage)
ALL_AGES[agegroup] += 1
else:
AGES[agegroup] += 1
event = f"{gender} {agegroup} {dist} {stroke} {pool}"
swim = {
S_EVENT: event,
S_ASA: asa,
S_NAME: swimmer,
S_TIMESTR: timestr,
S_FTIME: convert_time(timestr),
S_LOCATION: location,
S_DATE: xdate,
}
self.records.check_swim(swim)
return
def add_group(self, group):
"""Add a group to the swimmer."""
debug(f"Added {self.name} to {group.name}", 9)
self.groups.append(group)