def backup_data(self):
"""Backup."""
if self.ipad:
notify("Not implemented on iPad")
return False
if self.get_data(True) is False:
return False
backup = self.classes + self.backup_classes
for aclass in backup:
if self.crypto.encrypt_backup(aclass.name, aclass.json) is False:
return False
# Backup config file too.
home = str(Path.home())
today = date.today()
cfg = os.path.join(home, CONFIG_DIR)
directory = os.path.join(home, CONFIG_DIR, BACKUP_DIR, f"{today}")
src = os.path.join(cfg, CONFIG_FILE)
dst = os.path.join(directory, CONFIG_FILE)
copyfile(src, dst)
# Backup keyfile file too.
src = os.path.join(cfg, KEYFILE)
dst = os.path.join(directory, KEYFILE)
copyfile(src, dst)
return True
def debug(msg, level):
"""Debug error handler."""
if level > HANDLER.debug_level:
return
msg += "\n"
notify(msg)
def update(self):
"""Create a new list to add to SCM."""
cfg = get_config(self.scm, C_LISTS)
if cfg is None:
return
if get_config(self.scm, C_LISTS, C_EDIT) is not True:
notify("List update prohibited by config.\n")
return
lists = get_config(self.scm, C_LISTS, C_LIST)
self._suffix = get_config(self.scm, C_LISTS, C_SUFFIX)
if lists:
for xlist in lists:
newlist = NewList(self.scm, xlist, self._url)
self.newlists.append(newlist)
newlist.populate()
# Separate for loop, as add_to_list may have created some too
for xlist in self.newlists:
xlist.generate_data(self._suffix)
if xlist.upload() is None:
pass # not sure what to do, just carry on!
def __init__(self, scm):
"""Initialise."""
self.scm = scm
self.records = None
self.relay = None
try:
home = str(Path.home())
mydir = os.path.join(home, CONFIG_DIR, RECORDS_DIR)
if os.path.exists(mydir) is False:
os.mkdir(mydir)
filename = os.path.join(mydir, F_BASELINE)
if os.path.isfile(filename) is False:
with open(filename, FILE_WRITE, encoding="utf8") as file:
file.write(DEFAULT_RECORDS)
filename = os.path.splitext(filename)[0]
filename += HEADER
if os.path.isfile(filename) is False:
with open(filename, FILE_WRITE, encoding="utf8") as file:
file.write(DEFAULT_HEADER)
except EnvironmentError as error:
notify(f"Cannot create default records:\n{error}\n")
def fix_search(self):
"""fix_search_index."""
if A_KNOWNAS in self.data:
if self.data[A_KNOWNAS]:
self.newdata = {}
self.newdata[A_GUID] = self.guid
notify(f"Deleting index for {self.name}...")
self.newdata[A_FIRSTNAME] = "XXX"
res = self.scm.api_write(self, False)
if res is False:
notify("Hit a snag!\n")
return False
notify("Recreating...")
self.newdata[A_FIRSTNAME] = self.data[A_FIRSTNAME]
res = self.scm.api_write(self, False)
if res is False:
msg = "Hit a snag - Firstname"
msg2 = "deleted - oops sorry - restore manually!\n"
notify(f"{msg} '{self.data[A_FIRSTNAME]}' {msg2}")
return False
notify("Success.\n")
return True
def get_date(date, xformat):
"""Parse a date."""
try:
res = datetime.datetime.strptime(date, xformat)
return res
except ValueError as error:
notify(f"Error in date: {error}")
return None
def verify_schema(data):
"""Verify the config file."""
try:
SCHEMA.validate(data)
return True
except SchemaError as error:
notify(f"Error in config file:\n{error}\n")
return False
def analyse(self):
"""Analyse the data."""
notify("Analysing...\n")
for aclass in self.classes:
aclass.analyse()
notify("Done.\n")
def restore(self, xtype):
"""Restore a record."""
name = interact(f"Enter name of {xtype} to restore: ")
if name in self.by_name:
return self.by_name[name].restore(self._url)
if name in self.knownas:
return self.knownas[name].restore(self._url)
notify(f"{name} not found in backup of {xtype}.\n")
return False
def linkage(self):
"""Set up cross reference links between Entities."""
notify("Linking...\n")
for aclass in self.classes:
aclass.linkage()
if verify_schema_data(self) is False:
return False
return True
def se_check(self):
"""Check against an SE online."""
if self.scm.ipad:
notify("Not implemented on iPad")
return False
# pylint: disable=import-outside-toplevel
from scm_helper.browser import se_check
return se_check(self.scm, self.entities)
def list_fixes(self):
"""List any fixes."""
if len(self.fixable) == 0:
notify("Nothing to fix\n")
return False
res = ""
for fix in self.fixable:
res += f"{fix.name}: {fix.fixmsg}\n"
return res
def fix_secat(self):
"""fix_se categories."""
res = self.members.fix_secat()
if res is False:
return res
msg = "Fixed SE Catagories"
notify(f"\n{msg}\n")
return True
def read_newtimes(self, filename):
"""Read swimtimes."""
times = SwimTimes(self.records)
res = times.merge_times(filename, self.scm)
if self.records.newrecords:
for record in sorted(self.records.newrecords):
notify(self.records.newrecords[record])
self.records.write_records()
del times
return res
def apply_fixes(self):
"""Apply any fixes."""
if len(self.fixable) == 0:
notify("Nothing to fix\n")
return False
for fix in self.fixable:
if fix.apply_fix() is None:
self.fixable = []
return False
self.fixable = []
return True
def get_data(self, backup):
"""Get data."""
debug(f"(version: {VERSION})", 1)
notify("Reading Data...\n")
loop = self.classes
if backup:
loop = self.classes + self.backup_classes
for aclass in loop:
if aclass.get_data() is False:
return False
return True
def api_write(self, entity, create):
"""Write data back to SCM."""
club = self._config[C_CLUB]
user_agent = USER_AGENT.replace("###CLUB_NAME###", club)
headers = {
"content-type": "application/json",
"User-Agent": user_agent,
"Authorization-Token": self._key,
}
if get_config(entity.scm, C_ALLOW_UPDATE) is False:
notify("Update prohibited by config.\n")
return None
debug(f"URL:\n{entity.url}", 9)
debug(f"Headers:\n{headers}", 8)
data = entity.newdata
if create:
debug(f"Post request:\n{data}", 7)
response = requests.post(entity.url, json=data, headers=headers)
else:
debug(f"Put request:\n{data}", 7)
response = requests.put(entity.url, json=data, headers=headers)
if response.ok:
return response
if response.status_code == 404: # Worked, but not found
return False
notify(f"\nErroring posting data {entity.name}\n")
notify(response.reason)
notify("\n")
return None
def decrypt_backup(self, name, xdate):
"""Decrypt file."""
try:
home = str(Path.home())
backup = os.path.join(home, CONFIG_DIR, BACKUP_DIR)
filename = os.path.join(backup, xdate, f"{name}.enc")
data = self.decrypt_file(filename)
return json.loads(data.decode())
except OSError as error:
notify(f"Cannot open file: {error}\n")
return None
def check_member(browser, member):
"""Check a member."""
notify(f"Checking {member.name}...\n")
try:
name = browser.find_element_by_xpath(
"//table[1]/tbody/tr[2]/td[2]").text
knownas = browser.find_element_by_xpath(
"//table[1]/tbody/tr[2]/td[4]").text
gender = browser.find_element_by_xpath(
"//table[1]/tbody/tr[3]/td[4]").text
current = browser.find_element_by_xpath(
"//table[2]/tbody/tr[2]/td[4]").text
category = browser.find_element_by_xpath(
"//table[2]/tbody/tr[3]/td[4]").text
except selenium.common.exceptions.NoSuchElementException:
if member.print_exception(EXCEPTION_SE_HIDDEN) is False:
debug(f"SE Exception ignored: {member.name}", 7)
return ""
res = f"\n{member.name} ({member.asa_number}) does not exist in SE database.\n"
return res
res = ""
if member.print_exception(EXCEPTION_SE_NAME) is True:
if name.lower() != member.name.lower():
res += f" Name: SCM-> {member.name}, SE-> {name}\n"
if knownas and (member.knownas_only != knownas):
firstname = name.split(" ")
if knownas != firstname[0]: # in SE they are the same if no knownas
res += f" Knownas: SCM-> {member.knownas_only}, SE-> {knownas}\n"
if member.gender and (gender != GENDER[member.gender]):
res += f" Gender: SCM-> {member.name}, SE-> {gender}\n"
mycat = f"SE Category {member.asa_category}"
if category != mycat:
res += f" Category: SCM-> {mycat}, SE-> {category}\n"
if current != "Current":
res += " Not current\n"
if res:
res = f"\n{member.name} ({member.asa_number}) mismatch:\n" + res
return res
def se_check(scm, members):
"""Check members against the SE database."""
home = str(Path.home())
cookiefile = os.path.join(home, CONFIG_DIR, SE_COOKIES)
base_url = get_config(scm, C_SWIM_ENGLAND, C_BASE_URL)
if base_url is False:
notify("Swim England config missing\n")
return None
notify(f"Opening browser for {base_url}...\n")
browser = start_browser(scm)
if browser is None:
return None
read_cookies(browser, cookiefile, base_url, None)
check_url = get_config(scm, C_SWIM_ENGLAND, C_CHECK_URL)
test_id_url = get_config(scm, C_SWIM_ENGLAND, C_TEST_ID)
browser.get(f"{check_url}{test_id_url}")
try:
browser.find_element_by_xpath("//table[1]/tbody/tr[2]/td[2]")
except selenium.common.exceptions.NoSuchElementException:
msg = """Please solve the 'I am not a robot', and then press enter here.
If you cannot solve the capture try deleting the file 'se_cookies.json'
in the config directory.
"""
interact_yesno(msg)
write_cookies(browser, cookiefile, None)
res = ""
for member in members:
if member.is_active is False:
continue
if member.asa_number is None:
continue
url = f"{check_url}{member.asa_number}"
browser.get(url)
res += check_member(browser, member)
browser.close()
return res
def print_swimmers_covid(self):
"""Print swimmers and coaches with no COVID dec."""
covid = get_config(self.scm, C_SESSIONS, C_COVID)
if covid is None:
notify("Missing config for COVID option")
return ""
res = ""
for session in sorted(self.entities, key=lambda x: x.full_name):
if session.is_active:
c_res = session.print_swimmer_covid()
if c_res:
res += f"{session.full_name}\n{c_res}\n"
return res
def get_key(self, filename):
"""Encrypt file."""
apikey = interact("No SCM API keyfile, creating one. Enter API key: ")
try:
with open(filename, WRITE_BINARY, encoding="utf8") as file:
file.write(apikey)
file.close()
notify(f"Generate encrypted keyfille {filename}\n")
return apikey
except OSError as error:
notify(f"Cannot open/write key file: {error}\n")
return False
def read_key(self, filename):
"""Read API key."""
home = str(Path.home())
filename = os.path.join(home, CONFIG_DIR, filename)
if not os.path.exists(filename):
return self.get_key(filename)
try:
with open(filename, READ_BINARY, encoding="utf8") as file:
data = file.read()
file.close()
return data.strip()
except OSError as error:
notify(f"Cannot open key file: {error}\n")
return None
def decrypt(self, xdate):
"""Decrypt file."""
if self.ipad:
notify("Not implemented on iPad")
return False
restore = self.classes + self.backup_classes
for aclass in restore:
decrypted = self.crypto.decrypt_backup(aclass.name, xdate)
if decrypted is None:
return False
aclass.parse_data(decrypted)
notify("\n")
return True
def get_data(self):
"""Get data."""
notify(f"{self._name}... ")
data = self.scm.api_read(self._url, 1)
if data is None:
return False
count = self.create_entities(data)
# line below is subtly different, who's who data is already a list.
self._raw_data = data
notify(f"{count}\n")
if count != 1:
debug("Who's who assumption failure", 0)
return True
def write_cookies(browser, cookiefile, scm):
"""Write cookies."""
cookies = browser.get_cookies()
if cookies:
if scm:
data = pickle.dumps(cookies)
scm.crypto.encrypt_file(cookiefile, data)
return
try:
with open(cookiefile, FILE_WRITE, encoding="utf8") as file:
opt = json.dumps(cookies)
file.write(opt)
except EnvironmentError as error:
notify(f"Failed to write {cookiefile}\n{error}\n")
def start_browser(scm):
"""Start Browser."""
web_driver = get_config(scm, C_SELENIUM, C_WEB_DRIVER)
client = get_config(scm, C_SELENIUM, C_BROWSER)
if client is False:
notify("Selenium config missing\n")
return None
browser = getattr(selenium.webdriver, client)
try:
return browser(web_driver)
except selenium.common.exceptions.WebDriverException as error:
notify(
f"Failed to open {client} browser with: {web_driver}\n{error}\n")
return None
def readfile(self, filename, scm):
"""Read Facebook file."""
self._filename = filename
self.scm = scm
shortfile = ntpath.basename(filename)
notify(f"Reading {shortfile}...\n")
try:
with open(filename, FILE_READ, encoding="utf8") as file:
self.data = file.read().replace("\n", "")
file.close()
return True
except EnvironmentError:
notify(f"Cannot open facebook file: {filename}\n")
return False
def decrypt_file(self, filename):
"""Decrypt file."""
try:
fernet = Fernet(self.__key)
with open(filename, READ_BINARY) as file:
data = file.read()
file.close()
decrypted = fernet.decrypt(data)
return decrypted
except OSError as error:
notify(f"Cannot open file: {error}\n")
return None
except InvalidToken:
notify("Cannot decrypt file - wrong password?\n")
return None
def fix_search(self):
"""fix_search_index."""
home = str(Path.home())
cfg = os.path.join(home, CONFIG_DIR, "fixed_search.txt")
if os.path.isfile(cfg) is True:
notify("Not required - already fixed")
return False
res = self.members.fix_search()
if res is False:
return res
with open(cfg, mode="w", encoding="utf8") as file:
file.write(f"Fixed index: {self.today}")
msg = "Index recreated - give SCM time to process changes before testing."
notify(f"\n{msg}\n")
return True