def build_workout_switch_tables(self): if self._build_cur_workout.empty or len(self._build_tabs) == 0: return lift_name_tab = -1 for lift_name in self._build_tabs.keys(): lift_name_tab += 1 if lift_name == self._build_cur_lift: break if lift_name_tab == -1: return self.build_tabs.setCurrentIndex(lift_name_tab) if self._build_cur_lift in self._build_tabs: try: model = self._build_tabs[self._build_cur_lift].model() self.build_save_changes(model.dataFrame()) self._build_cur_workout_view = self._build_cur_workout[self._build_cur_workout.Lift == self._build_cur_lift] self._build_cur_workout_view = self._build_cur_workout_view[['Lift', 'Weight', 'Reps', 'RPE', 'e1RM']] model.setDataFrame(self._build_cur_workout_view) model.enableEditing(True) model.dataChanged.emit() model.layoutChanged.emit() except Exception as err: log_err(err, "Couldn't emit datachanged signals to table model for {}".format(self._build_cur_lift))
def loop_on_keys(keys): for key in keys: if key == 'times': target['times'] = [] times1 = cl1.pop(key) times2 = cl2.pop(key) if len(times1) != len(times2): log_err( f'"{key}" is different for class {CRN}: "{len(times1)}" vs. "{len(times2)}"' ) continue for idx in range(len(times1)): time1 = times1[idx].copy() time2 = times2[idx].copy() target['times'].append( merge_dicts(time1, time2, allowed=allowed)) continue val1 = cl1.pop(key, None) val2 = cl2.pop(key, None) if val1 != None and val2 != None: if val1 != val2 and key not in allowed: log_warn( f'"{key}" is different for class {CRN}: "{val1}" vs. "{val2}"' ) target[key] = val1 else: target[key] = val1 if val1 != None else val2
def save_bw_temp(self): try: bak_num = len(self.__bw_bak_files) bak_file = tempfile.mkstemp("{}.{}".format(bak_num, "xlsx"), "{}_Tmp_".format(self.bw_name))[1] self.weight_log.to_excel(bak_file) self.__bw_bak_files.append(bak_file) except OSError as e: log_err(e, "Could not create temp Weight Log file".format(self.__bw_bak_files[-1]))
def add_bodyweight_entry(self, date, high=None, low=None, bf=None, comments=None): try: date = pd.Timestamp(date) except ValueError as err: log_err(err, 'Error converting {} to Timestamp'.format(date)) for col, val in {'High': high, 'Low': low, 'BF_Per': bf, 'Comments': comments}.items(): self.weight_log.ix[date, col] = val self.calc_weight_data()
def save_wl_temp(self): try: bak_num = len(self.__wl_bak_files) bak_file = tempfile.mkstemp("{}.{}".format(bak_num, "xlsx"), "{}_Tmp_".format(self.wl_name))[1] self.workout_log.to_excel(bak_file) self.__wl_bak_files.append(bak_file) except OSError as e: log_err(e, "Could not create Workout Log backup file: {}".format(self.__wl_bak_files[-1]))
def save_lt_temp(self): try: bak_num = len(self.__lt_bak_files) bak_file = tempfile.mkstemp("{}.{}".format(bak_num, "xlsx"), "{}_Tmp_".format(self.lt_name))[1] self.lift_types.to_excel(bak_file) self.__lt_bak_files.append(bak_file) except OSError as e: log_err(e, "Could not create temp Lift Type file: {}".format(self.__lt_bak_files[-1]))
def _parse_offset(offset): """ split the "period" input into {multiplier} {type}, e.g. {3} {days} """ try: per_mult, per_freq = re.match('(\\d+)?([dwmy])', offset, re.IGNORECASE).groups() if per_mult: per_mult = int(per_mult) return per_mult, per_freq.upper() except (AttributeError, ValueError, re.error) as e: log_err(e, "Invalid period passed: {}".format(offset))
def get_slot(self, lift): """Returns the slot (i.e. 'upper' or 'lower' body exercise) of the given string""" try: if self.is_valid_lift(lift): return self.lift_types.Slot[self.lift_types.Lift == lift].values[0] else: raise ValueError except ValueError as e: log_err(e, """Error retrieving slot for lift '{}'""".format(lift))
def get_category(self, lift): """Returns the category (e.g. 'Squat', 'Bench', etc) of a given string""" try: if self.is_valid_lift(lift): return self.lift_types.Category[self.lift_types.Lift == lift].values[0] else: raise ValueError except ValueError as e: log_err(e, """Error retrieving category for lift '{}'""".format(lift))
def save(self, data=None): if data is None: data = self.data else: self.data.copy(data) try: data.to_sql('log', self.__sql_engine, if_exists='replace') except Exception as e: log_err(e, 'Error saving dataframe')
def __import_weight_log(filename, sheet_name=None): # TODO: refactor all accessory code into this one call (e.g. calc_) try: if not sheet_name: sheet_name = 0 weight_log = pd.read_excel(filename, sheetname=sheet_name, header=0, index_col="Date", parse_dates=[0]) log_message('Loaded {} records from weight log'.format(len(weight_log))) return weight_log except Exception as e: log_err(e, "Could not import {} database file {}".format(sheet_name, filename))
def __import_lift_types(filename, sheet_name=None): # TODO: refactor all accessory code into this one call (e.g. calc_) try: if not sheet_name: sheet_name = 0 lift_types = pd.read_excel(filename, sheetname=sheet_name, header=0) log_message('Loaded {} lifts from lift types'.format(len(sheet_name))) return lift_types except Exception as e: log_err(e, "Could not import {} from database file {}".format(sheet_name, filename))
def plot_ts(self, data, **kwargs): # TODO: Add handling for DataFrames if isinstance(data, pd.Series): try: dates = [int(datetime.datetime(x.year, x.month, x.day).timestamp()) for x in data.index] self.plot(x=dates, y=data.values, **kwargs) except Exception as err: log_err(err, "{}\nCould not plot time series".format(data)) else: self.plot(x=data[0], y=data[1], **kwargs)
def __load_data(self): self.data = None try: self.data = pd.read_sql_table('log', self.__sql_engine, index_col='Date', parse_dates=['Date']) except Exception as e: log_err(e, "Could not read database file") # See if the workout log contains data, if not prepare it to take some try: log_message('Loaded {} records from db'.format(len(self.data))) except TypeError: self.__build_dataframe()
def __init__(self, filename): if os.path.isfile(filename): self.__backup_file(filename) else: log_message('Database file not found: {}\n'.format(filename) + 'Building a new one from scratch') self.__build_dataframe() try: self.__sql_engine = create_engine('sqlite:///' + filename) except Exception as e: log_err(e, 'Could not create database engine with: {}'.format(filename)) self.__load_data()
def save_to_disk(self, filename=None): """Saves all data to disk""" if not filename: filename = self.filename try: writer = pd.ExcelWriter(filename) self.workout_log.to_excel(writer, sheet_name=self.wl_name) self.weight_log.to_excel(writer, sheet_name=self.bw_name) self.lift_types.to_excel(writer, sheet_name=self.lt_name) writer.save() except OSError as e: log_err(e, "Error saving database file!")
def parse(content): ''' Parse takes the content from the request and then populates the database with the data :param content: (html) The html containing the courses :param db: (TinyDB) the current database ''' soup = BeautifulSoup(content, 'html5lib') tables = soup.find_all('table', {'class': 'TblCourses'}) classes = {} for t in tables: rows = t.find_all('tr', {'class': 'CourseRow'}) for tr in rows: cols = tr.find_all(lambda tag: tag.name == 'td') if cols: # The first <td> is a field that is not relevant to us # it is either empty or contains a "flag" icon cols.pop(0) for i, c in enumerate(cols): a = c.find('a') cols[i] = (a.get_text() if a else cols[i].get_text()).strip() try: data = dict(zip(HEADERS, cols)) if data['CRN'] in classes: continue data['status'] = data['status'].lower() try: data = seatInfoSchema.load(data) except ValidationError as e: log_err('Marshmallow validation failed', details={ 'messages': e.messages, 'class': data }) continue classes[data['CRN']] = data except KeyError: continue return classes
def merge_workout_data(self, data, date=None): if not date: if isinstance(data.index[0], pd.Timestamp): date = data.index elif 'Date' in data: date = data.Date else: log_err(description='No date found in data') return data['Date'] = date self.workout_log.drop(date, inplace=True) self.workout_log.reset_index(inplace=True) self.workout_log = self.workout_log.merge(data, how='outer').set_index('Date') self.calc_workout_data()
def _drop_set_selected(self, index): if not index.isValid(): return try: drop_set_str = index.data().split('(')[0].strip() self.drop_set.setText(drop_set_str) parser = re.match(r'(\d+)x(\d+)@(\d+\.\d+)', drop_set_str) self._drop_weight, self._drop_reps, self._drop_rpe = parser.groups() self._drop_weight = int(self._drop_weight) self._drop_reps = int(self._drop_reps) self._drop_rpe = round(float(self._drop_rpe) * 2.0) / 2.0 self.values_changed.emit() except (ValueError, TypeError) as err: log_err(err, "Couldn't parse drop_set from RPE table: {}".format(drop_set_str))
def _parse_RPE(line): try: RPE, comment = rpe_re.match(line).groups() if RPE: try: RPE = round(float(RPE) * 2.0) / 2.0 if not (6.5 <= RPE <= 10.0): RPE = None except ValueError as e: log_err(e, "Invalid RPE passed {}".format(RPE)) except re.error as e: pass # No biggie, the line didn't match our criteria if comment: comment = comment.strip(' ()') return RPE, comment
def run_advanced_scraper(**kwargs): try: scraper = AdvancedScraper( ssb_url=SSB_URL, db_dir=DB_DIR, cache_dir=CACHE_DIR, hooks=FHDAScraperHooks, login=login, trace=True, **kwargs, ) scraper.run() except KeyboardInterrupt: log_err('Aborted advanced schedule scraper', start='\n')
def save_classes(self, db, depts, classes): db_depts = [] db_courses = [] db_classes = [] depts = {k.replace(' ', ''): v for k, v in depts.items()} for dept, t in classes.items(): db_depts.append({ 'id': dept, 'name': depts[dept], }) for course, section in t.items(): course_classes = [] course_titles = set() for cl in section.values(): try: data = classDataSchema.load(cl) classTimes = [ classTimeSchema.load(time) for time in cl['times'] ] except MarshValidationError as e: print(e, cl) continue data['times'] = classTimes db_classes.append(data) course_titles.add(data['title']) course_classes.append(data['CRN']) if len(course_titles) > 1: log_err( f'Multiple course titles for "{dept} {course}" {str(course_titles)}' ) db_courses.append({ 'dept': dept, 'course': course, 'title': course_titles.pop(), 'classes': course_classes }) db.drop_tables() db.table('departments').insert_multiple(db_depts) db.table('courses').insert_multiple(db_courses) db.table('classes').insert_multiple(db_classes)
def run_public_schedule_scraper(**kwargs): if not kwargs.get('db_dir'): kwargs['db_dir'] = DB_DIR try: scraper = ScheduleScraper( ssb_url=SSB_URL, cache_dir=CACHE_DIR, hooks=FHDAScraperHooks, # login=login, trace=True, **kwargs, ) scraper.run() except KeyboardInterrupt: log_err('Aborted public schedule scraper', start='\n')
def restore_from_backup(self, backup_filename=None): """Deletes the current database file and restores from a previously saved version""" if not backup_filename: backup_filename = self.filename + '.bak' if os.path.isfile(backup_filename): try: shutil.move(backup_filename, self.filename) except OSError as e: log_err(e, "Cannot restore from backup file ({})".format(backup_filename)) else: log_err("Could not find backup file ({})".format(backup_filename)) self.__load_data(backup_filename) # TODO: Get rid of these calls self.calc_workout_data() self.calc_weight_data()
def parse_sets(sets, tgt_e1RM=None): res = [] try: weight, reps, mult, rpes = re.match('(\\d+\\.?\\d+)?x(\\d+)x?(\\d+)?@?(.+)?', sets.strip(), re.IGNORECASE).groups() if weight: weight = round(float(weight)) reps = int(reps) if mult: mult = int(mult) else: mult = 1 if rpes: rpes = rpes.split(',') if len(rpes) == 1: rpe = round(float(rpes[0].strip()) * 2.0) / 2.0 rpe_vals = None else: rpe_vals = [] for rpe in rpes: rpe = float(rpe.strip()) rpe = round(float(rpe) * 2.0) / 2.0 if not 6.5 <= rpe <= 10: rpe = None rpe_vals.append(rpe) else: rpe = None rpe_vals = None # TODO: Find a way to integrate NSCA tables into this calc if rpe_vals and tgt_e1RM: # given set of x5@7,8,9 or 345x5@7,8,9 -> calculate the weights to use for rpe in rpe_vals: if (1 <= reps <= 10) and (6.5 <= rpe <= 10.0): percent_mul = rpe_table.ix[reps].ix[rpe] tgt_weight = tgt_e1RM * percent_mul res.append([tgt_weight, reps, rpe]) elif weight: res = [[weight, reps, None] for i in range(mult-1)] res.append([weight, reps, rpe]) return res except (AttributeError, re.error) as e: log_err(e, "Invalid set: {}".format(set)) return res
def merge_dbs(final: TinyDB, first: TinyDB, second: TinyDB, allowed): classes1 = {doc['CRN']: doc for doc in first.table('classes').all()} classes2 = {doc['CRN']: doc for doc in second.table('classes').all()} classes = [] for CRN in classes1.keys(): cl1 = classes1[CRN].copy() cl2 = classes2.get(CRN) if cl2: classes.append(merge_dicts(cl1, cl2.copy(), allowed=allowed)) else: log_err(f'Class {CRN} was only found in one DB!') classes.append(cl1) final.drop_tables() final.table('classes').insert_multiple(classes)
def from_fitnotes(from_file, to_file): try: fitnotes = pd.DataFrame.from_csv(from_file) new_log = pd.DataFrame(columns=Backend.excel_format[Backend.wl_name]) new_log = new_log.set_index('Date') new_log.Lift = fitnotes.Exercise new_log.Weight = fitnotes["Weight (lbs)"] new_log.Reps = fitnotes.Reps idx = 0 for date, line in fitnotes.Comment.iteritems(): if line is not np.NaN: RPE, comment = _parse_RPE(line) new_log.RPE.iloc[idx] = RPE new_log.Comments.iloc[idx] = comment idx += 1 # TODO: parse comments to RPE's print(new_log.tail(10)) new_log.to_excel(to_file) log_message("Imported {} lines from {} to {}".format(len(new_log), from_file, to_file)) except Exception as e: log_err(e, "Error import from FitNotes")
def log_add_entry(self): try: date = pd.Timestamp(self.log_date.date().toPyDate()) lift = self.log_lift.currentText() if not self.db.is_valid_lift(lift): raise ValueError('{} is not a valid lift'.format(lift)) raw_sets = self.log_sets.text().split(',') parsed_sets = [] for set in raw_sets: parsed_sets.extend(analysis.parse_sets(set.strip())) comments = self.log_comments.text() for set in parsed_sets: weight, reps, rpe = set self.db.add_set(date, lift, weight, reps, rpe, comments) self.db.calc_workout_data() self.log_date_changed(date - pd.Timedelta(days=1)) self.log_date_changed(date) self.wl_data_changed.emit() self.log_lift.setCurrentText('') self.log_sets.setText('') except (ValueError, TypeError) as err: log_err(err, "Couldn't parse log entry")
def an_add_analysis(self): data = self.db.workout_log group = self.an_type.currentText() val = self.an_value.currentText() try: if group == 'Slot': data = data[data.Slot == val] elif group == 'Category': data = data[data.Category == val] elif group == 'Lift': data = data[data.Lift == val] else: return except (ValueError, TypeError) as err: log_err(err, "Invalid analysis type selected: {}, {}".format(group, val)) analysis_funcs = {'E1RMs': analysis.get_max_e1RM, 'Volume': analysis.get_volume, 'Norm V': analysis.calc_normalized_tonnage, 'Fatigue': analysis.get_fatigue} selected_func = analysis_funcs[self.an_analysis.currentText()] analyzed_data = selected_func(data) title = '{}: {} -> {}'.format(self.an_analysis.currentText(), group, val) if self.an_mov_avg.isChecked(): period = self.an_period.value() if 'e1rm' in selected_func.__name__: analyzed_data = analyzed_data.resample('1D', label='right', how=np.max) elif 'vol' in selected_func.__name__ or 'fatigue' in selected_func.__name__: analyzed_data = analyzed_data.resample('1D', label='right', how=np.sum) analyzed_data = analyzed_data.fillna(0) analyzed_data = pd.rolling_mean(analyzed_data, period, min_periods=1) if 'fatigue' in selected_func.__name__: analyzed_data *= 7.0 title += ' ({} day avg)'.format(period) self.an_data[title] = analyzed_data self.analysis_data_changed.emit()
def make_set(self, lift, weight, reps, date, rpe=None, comments=None): res = pd.Series(index=self.excel_format[self.wl_name]) try: if self.is_valid_lift(lift): res.Lift = lift else: raise ValueError('Lift "{}" is not a valid lift'.format(lift)) if date: res.Date = pd.Timestamp(date) res.Slot = self.get_slot(lift) res.Category = self.get_category(lift) res.Weight = int(weight) res.Reps = int(reps) if rpe: res.RPE = round(float(rpe) * 2.0) / 2.0 else: res.RPE = None res.e1RM = analysis.calc_e1RM(res.Lift, res.Weight, res.Reps, res.RPE) res.Volume = res.Weight * res.Reps res.Comments = comments except (ValueError, TypeError) as err: log_err(err, 'Invalid data provided for set:\n{}'.format((lift, weight, reps, rpe, comments))) return res
def __check_for_blanks(self): wl_blanks = self.workout_log['Date', 'Lift', 'Weight', 'Reps'].isnull() if wl_blanks.any(): err = ValueError("Missing values for core data in workout log") log_err(err, str(wl_blanks)) raise err bw_blanks = self.weight_log['Date'].isnull() if bw_blanks.any(): err = ValueError("Missing values for core data in bodyweight log") log_err(err, str(bw_blanks)) raise err lift_blanks = self.lift_types.isnull() if lift_blanks.any(): err = ValueError("Missing values for core data lifts table") log_err(err, str(lift_blanks)) raise err
def log_cal_selection_changed(self): try: date = pd.Timestamp(self.log_cal.selectedDate().toPyDate()) self.log_date_changed(date) except (ValueError, TypeError) as err: log_err(err, "Couldn't convert log_cal's value to a date... skipping")
def parse(content, db): ''' Parse takes the content from the request and then populates the database with the data :param content: (html) The html containing the courses :param db: (TinyDB) the current database ''' soup = BeautifulSoup(content, 'html5lib') tables = soup.find_all('table', {'class': 'TblCourses'}) for t in tables: # TODO: verify whether replacing spaces yields correct dept names in all scenarios dept = t['dept'].replace(' ', '') dept_desc = t['dept-desc'] db.table('departments').insert({ 'id': dept, 'name': dept_desc, }) rows = t.find_all('tr', {'class': 'CourseRow'}) s = defaultdict(lambda: defaultdict(list)) for tr in rows: cols = tr.find_all(lambda tag: tag.name == 'td') if cols: # The first <td> is a field that is not relevant to us # it is either empty or contains a "flag" icon cols.pop(0) for i, c in enumerate(cols): a = c.find('a') cols[i] = (a.get_text() if a else cols[i].get_text()).strip() try: parsed_course = parse_course_str(cols[0]) key = parsed_course['course'] section = parsed_course['section'] data = dict(zip(HEADERS, cols)) if parsed_course['dept'] != dept: raise ValidationError( 'Departments do not match', f"'{parsed_course['dept']}' != '{dept}'") data['dept'] = dept data['course'] = key data['section'] = section data['status'] = data['status'].lower() data['units'] = data['units'].lstrip() try: data = interimClassDataSchema.load(data) except MarshValidationError as e: print(e.messages, data) continue crn = data['CRN'] if s[key][crn]: comb = set(s[key][crn][0].items()) ^ set(data.items()) if not comb: continue s[key][crn].append(data) except KeyError: continue except ValidationError as e: log_err('Unable to parse course - data validation failed', details={ 'message': e.message, 'details': e.details, 'course': cols, }) print('\n') continue for course, section in s.items(): db.table('courses').insert({ 'dept': dept, 'course': course, 'classes': list(section.keys()) }) for cl in section.values(): data = classDataSchema.load(cl[0]) classTime = [classTimeSchema.load(c) for c in cl] check_integrity(cl, data, classTime) data['times'] = classTime db.table('classes').insert(data)
def __backup_file(filename): """Backup the old database file in case anything gets corrupted during runtime""" try: shutil.copy(filename, filename + '.bak') except OSError as e: log_err(e, 'Error backing up DB File')
for ssb_campus in ['MC', 'WVC']: scraper = ScheduleScraper( ssb_url=SSB_URL, db_dir=DB_DIR, cache_dir=CACHE_DIR, ssb_campus=ssb_campus, hooks=WVMScraperHooks, # max_terms=4, # use_cache=False, # start_term='201231', trace=True, ) scraper.run() except KeyboardInterrupt: log_err('Aborted', start='\n') db_files = list_dbs(DB_DIR, prefix=f'sched_') termdbs = [] for filepath in db_files: matches = re.search(r'sched_(\w{2,3})_([0-9]{6})_database.json$', filepath) if matches and matches.groups(): campus, term = matches.groups() termdbs.append({'campus': campus, 'code': term}) with open(join(DB_DIR, 'metadata.json'), 'w') as outfile: json.dump({'terms': termdbs}, outfile)