def report_sync_results(self): subprocess.run(["clear"]) self.print_header_message() if not self.sync_results: return for task_result in self.sync_results: print_message(task_result[0], wrap=False, fg=task_result[1])
def print_description(self): if isinstance(self.setting.description, list): for line in self.setting.description: print_message(line) else: print_message(self.setting.description) print()
def synchronize_files(self): if self.all_files_are_in_sync: message = "All files for selected data sets are in sync!" print_message(message, fg="bright_green", bold=True) pause(message="Press any key to continue...") return Result.Ok() for file_type, file_type_dict in self.sync_files.items(): for data_set, (out_of_sync, missing_files, outdated_files) in file_type_dict.items(): if not out_of_sync: continue all_sync_files = [] missing_count = 0 outdated_count = 0 if missing_files: all_sync_files.extend(missing_files) missing_count = len(missing_files) if outdated_files: all_sync_files.extend(outdated_files) outdated_count = len(outdated_files) table_viewer = self.create_table_viewer( all_sync_files, data_set, file_type, missing_count, outdated_count) apply_changes = table_viewer.launch() if apply_changes: self.apply_pending_changes(file_type, data_set, missing_files, outdated_files) return Result.Ok()
def launch(self): subprocess.run(["clear"]) print_heading(self.menu_heading, fg="bright_yellow") if not node_is_installed(): print_message(INSTALL_ERROR, fg="bright_red", bold=True) pause(message="Press any key to continue...") return if not NODEJS_INBOX.exists(): NODEJS_INBOX.mkdir(parents=True, exist_ok=True) if node_modules_folder_exists(): message = UPDATE_MESSAGE prompt = UPDATE_PROMPT temp_folder = None command = "npm update --timeout=9999999" else: message = INSTALL_MESSAGE prompt = INSTALL_PROMPT temp_folder = TemporaryDirectory(dir=NIGHTMAREJS_FOLDER) command = f"npm install --timeout=9999999 --cache={temp_folder.name}" print_message(message, fg="bright_yellow") if not yes_no_prompt(prompt, wrap=False): return Result.Ok(self.exit_menu) subprocess.run(["clear"]) print_heading(self.menu_heading, fg="bright_yellow") result = run_command(command, cwd=str(NIGHTMAREJS_FOLDER)) if result.failure: return result if temp_folder: temp_folder.cleanup() pause(message="\nPress any key to continue...") return Result.Ok(self.exit_menu)
def launch(self): subprocess.run(["clear"]) print_heading("This feature is not currently available", fg="bright_yellow") print_message( "Sorry for any inconvenience, I promise to finish this soon!") pause(message="\nPress any key to return to the previous menu...") return Result.Ok(True)
def exit_app_error(message): if message: if isinstance(message, list): for m in message: print_message(m, fg="bright_red", bold=True) else: print_message(message, fg="bright_red", bold=True) return 1
def report_sync_results(self): subprocess.run(["clear"]) src_folder = "S3 bucket" if self.sync_direction == SyncDirection.DOWN_TO_LOCAL else "local folder" dest_folder = "local folder" if self.sync_direction == SyncDirection.DOWN_TO_LOCAL else "S3 bucket" print_heading(f"Syncing data from {src_folder} to {dest_folder}", fg="bright_yellow") for (result, text_color) in self.sync_results: print_message(result, fg=text_color)
def prompt_user_apply_patch_list(self): subprocess.run(["clear"]) prompt = "Would you like to apply the patch to fix the invalid data?" self.update_menu_heading("Apply Patch?", heading_color="bright_cyan") print_message( f"A patch list for {self.game_id} was successfully created!\n", fg="bright_cyan") return yes_no_prompt(prompt)
def prompt_user_view_patched_data(self): subprocess.run(["clear"]) prompt = "Would you like to see a report detailing the changes that were made by applying the patch list?" self.update_menu_heading("View Patch Results?", heading_color="bright_cyan") print_message( "The patch was successfully applied to the PitchFX data!\n", fg="bright_cyan") return yes_no_prompt(prompt)
def display_pitchfx_errors(self): games_plural = "games contain" if self.total_games_any_pfx_error > 1 else "game contains" ab_plural = "at bats" if self.total_at_bats_any_pitchfx_error > 1 else "at bat" message = ( f"{self.total_games_any_pfx_error} {games_plural} invalid PitchFX data for a total " f"of {self.total_at_bats_any_pitchfx_error} {ab_plural}, you can view details " "of each at bat and attempt to fix these errors using the Investigate Failures menu." ) print_message(message, fg="bright_cyan") print()
def display_games_failed_to_combine(self): error_message = f"Error prevented scraped data being combined for {len(self.failed_game_ids)} games:" error_details = [{ "bbref_game_id": game_id, "error": error } for game_id, error in self.combine_data_fail_results] print_message(error_message, wrap=False, fg="bright_red", bold=True) print_message(tabulate(error_details, headers="keys"), wrap=False, fg="bright_red")
def display_no_match_found(self, matches): subprocess.run(["clear"]) for match_dict in matches: match_dict["invalid_pfx"].pop("at_bat_id") match_dict["invalid_pfx"].pop("pitcher_id") match_dict["invalid_pfx"].pop("batter_id") subprocess.run(["clear"]) error = "No matching at bats were found for the invalid PitchFX data below:\n" print_message(error, fg="bright_red", bold=True) unmatched_rows = [match_dict["invalid_pfx"] for match_dict in matches] print_message(tabulate(unmatched_rows, headers="keys"), wrap=False) print() pause(message="Press any key to continue...")
def display_menu_text(self): print_heading(f"vigorish v{__version__}", fg="bright_yellow") if self.audit_report: self.display_audit_report() elif self.initial_setup_complete: print_message( "All prerequisites are installed and database is initialized.") else: self.display_initial_task_status() print_message("\nMain Menu:\n", fg="bright_green", bold=True, underline=True)
def display_audit_report(self): table_rows = [{ "season": f"MLB {year} ({report['total_games']} games)", "scraped": len(report["scraped"]), "combined": len(report["successful"]), "failed": len(report["failed"]) + len(report["pfx_error"]) + len(report["invalid_pfx"]), } for year, report in self.audit_report.items()] print_message(tabulate(table_rows, headers="keys"), wrap=False)
def display_results(self): subprocess.run(["clear"]) if self.combined_success_and_no_pfx_errors: plural = "games total" if self.total_games > 1 else "game" success_message = f"\nAll game data ({self.total_games} {plural}) combined, no errors" print_message(success_message, wrap=False, fg="bright_cyan", bold=True) if self.failed_game_ids: self.display_games_failed_to_combine() if self.all_pfx_errors: self.display_pitchfx_errors() pause(message="Press any key to continue...")
def launch(self): subprocess.run(["clear"]) setting_heading = f"Setting: {self.setting_name_title} (Type: {self.data_type.name})" print_heading(setting_heading, fg="bright_magenta") self.print_description() print_message(self.current_settings, wrap=False, fg="bright_yellow", bold=True) if yes_no_prompt("\nChange current setting?"): change_setting_menu = ChangeConfigSettting(self.app, self.setting_name) return change_setting_menu.launch() return Result.Ok(self.exit_menu)
def patch_invalid_pfx_single_game(self): result = self.patch_invalid_pfx.execute(self.game_id) if result.failure: header = f"Invalid PitchFX Data for {self.game_id}\n" subprocess.run(["clear"]) print_message(header, wrap=False, bold=True, underline=True) print_message(result.error, fg="bright_yellow") pause(message="Press any key to continue...") return Result.Ok(True) if not self.prompt_user_create_patch_list(): return Result.Ok(True) result = self.patch_invalid_pfx.match_missing_pfx_data() if result.failure: return result for result, matches in self.patch_invalid_pfx.match_results.items(): if result == "success": for num, match_dict in enumerate(matches, start=1): match_dict["patch"] = self.prompt_user_create_patch( num, len(matches), match_dict) if result == "no_matches": self.display_no_match_found(matches) if result == "many_matches": self.display_many_matches_found(matches) if "success" not in self.patch_invalid_pfx.match_results: header = f"Invalid PitchFX Data for {self.game_id}\n" message = ( "Unable to identify missing data that matches the invalid PitchFX data for this " "game. You should inspect the combined data JSON file for this game and " "investigate the invalid data manually.\n") subprocess.run(["clear"]) print_message(header, wrap=False, bold=True, underline=True) print_message(message, fg="bright_yellow") pause(message="Press any key to continue...") return Result.Ok(True) result = self.patch_invalid_pfx.create_patch_list() if result.failure: return result if not self.patch_invalid_pfx.patch_list or not self.prompt_user_apply_patch_list( ): return Result.Ok(True) result = self.patch_invalid_pfx.apply_patch_list() if result.failure: return result self.patch_results = result.value print() if self.patch_results["fixed_all_errors"]: patch_result = f"PitchFX data for {self.game_id} is now completely reconciled (no errors of any type)!\n" print_success(patch_result) if self.patch_results["invalid_pfx"]: patch_result = f"{self.game_id} still contains invalid PitchFX data after applying the patch list.\n" print_error(patch_result) if self.patch_results["pfx_errors"]: patch_result = f"{self.game_id} still contains PitchFX data errors associated with valid at bats.\n" print_error(patch_result) pause(message="Press any key to continue...") subprocess.run(["clear"]) if self.prompt_user_view_patched_data(): self.display_patched_data_tables( **self.patch_results["patch_diff_report"]) return Result.Ok()
def check_app_status(self): if not self.db_setup_complete: return color = get_random_cli_color() if not self.initialized: f = Figlet(font=get_random_figlet_font(), width=120) print_message(f.renderText("vigorish"), wrap=False, fg=f"bright_{color}") spinner = Halo(spinner=get_random_dots_spinner(), color=color) spinner.text = "Updating metrics..." if self.initialized else "Loading..." spinner.start() if self.initialized: del self.app.audit_report self.audit_report = self.app.audit_report spinner.stop()
def confirm_job_details(self, data_sets, start_date, end_date, job_name): subprocess.run(["clear"]) heading = "Confirm job details" data_set_space = "\n\t " confirm_job_name = "" if job_name: confirm_job_name = f"Job Name....: {job_name}\n" job_details = ( f"{confirm_job_name}" f"Start date..: {start_date.strftime(DATE_ONLY_2)}\n" f"End Date....: {end_date.strftime(DATE_ONLY_2)}\n" f"Data Sets...: {data_set_space.join([DATA_SET_TO_NAME_MAP[ds] for ds in data_sets])}" ) print_heading(heading, fg="bright_yellow") print_message(job_details, wrap=False, fg="bright_yellow") return yes_no_prompt(prompt="\nAre the details above correct?")
def combine_scraped_data_for_game(self): subprocess.run(["clear"]) spinner = Halo(spinner=get_random_dots_spinner(), color=get_random_cli_color()) spinner.text = f"Combining scraped data for {self.game_id}..." spinner.start() result = self.combine_data.execute(self.game_id) if (not result["gather_scraped_data_success"] or not result["combined_data_success"] or not result["update_pitch_apps_success"]): spinner.fail(f"Failed to combine data for {self.game_id}!") pause(message="Press any key to continue...") return Result.Fail(result["error"]) pfx_errors = result["results"]["pfx_errors"] fail_results = [ pfx_errors.pop("pitchfx_error", {}), pfx_errors.pop("invalid_pitchfx", {}), ] if all(len(f) <= 0 for f in fail_results): spinner.succeed( f"All scraped data for {self.game_id} was successfully combined!" ) pause(message="Press any key to continue...") return Result.Ok() spinner.stop() subprocess.run(["clear"]) total_pitch_apps = sum(len(f.keys()) for f in fail_results if f) pitch_apps_plural = "pitch appearances" if total_pitch_apps > 1 else "pitch appearance" total_at_bats = sum( len(at_bat_ids) for f in fail_results for at_bat_ids in f.values() if f) at_bats_plural = "at bats" if total_at_bats > 1 else "at bat" error_header = f"PitchFX data could not be reconciled for game: {self.game_id}\n" error_message = ( f"{total_pitch_apps} {pitch_apps_plural} with data errors ({total_at_bats} total {at_bats_plural})\n" ) print_message(error_header, wrap=False, fg="bright_red", bold=True, underline=True) print_message(error_message, fg="bright_red") if not self.prompt_user_investigate_failures(): pause(message="Press any key to continue...") return Result.Ok() subprocess.run(["clear"]) return self.patch_invalid_pfx_single_game()
def launch(self): subprocess.run(["clear"]) print_message("*** Job Details ***", fg="bright_yellow", bold=True) job_details = report_dict(self.job_details, title="", title_prefix="", title_suffix="") print_message(f"{job_details}\n", wrap=False, fg="bright_yellow") if self.db_job.errors: print_message("*** Errors ***", fg="bright_red", bold=True) print_message(self.db_job.error_messages, fg="bright_red") if self.job_status == JobStatus.INCOMPLETE: result = self.incomplete_job_options_prompt() if result.failure: return Result.Ok(self.exit_menu) user_choice = result.value if user_choice == "RUN": job_runner = JobRunner(app=self.app, db_job=self.db_job) result = job_runner.execute() if result.failure: return result return Result.Ok(True) if user_choice == "CANCEL": self.db_job.status = JobStatus.COMPLETE self.db_session.commit() subprocess.run(["clear"]) print_message("Job was successfully cancelled.", fg="bright_red", bold=True) pause(message="Press any key to continue...") return Result.Ok(True) if self.job_status == JobStatus.ERROR: result = self.failed_job_options_prompt() if result.failure: return Result.Ok(self.exit_menu) user_choice = result.value if user_choice == "RETRY": job_runner = JobRunner(app=self.app, db_job=self.db_job) result = job_runner.execute() if result.failure: return result return Result.Ok(True) else: pause(message="\nPress any key to return to the previous menu...") return Result.Ok(self.exit_menu)
def apply_pending_changes(self, file_type, data_set, missing_files, outdated_files): subprocess.run(["clear"]) self.spinner = Halo(spinner=get_random_dots_spinner(), color=get_random_cli_color()) self.spinner.start() self.s3_sync.events.sync_files_progress += self.update_sync_progress self.s3_sync.sync_files(self.sync_direction, missing_files, outdated_files, file_type, data_set, self.year) self.s3_sync.events.sync_files_progress -= self.update_sync_progress self.spinner.stop() src_folder = "S3 bucket" if self.sync_direction == SyncDirection.DOWN_TO_LOCAL else "local folder" dest_folder = "local folder" if self.sync_direction == SyncDirection.DOWN_TO_LOCAL else "S3 bucket" message = ( f"All changes have been applied, MLB {self.year} {data_set} {file_type} files in {src_folder} " f"have been synced to {dest_folder}!") print_message(message, fg="bright_green", bold=True) pause(message="Press any key to continue...")
def report_task_results(self): subprocess.run(["clear"]) job_name_id = f"Job Name: {self.job_name} (ID: {self.job_id.upper()})" if self.job_name == self.job_id: job_name_id = f"Job ID: {self.job_id.upper()}" start_date_str = self.start_date.strftime(DATE_ONLY_2) end_date_str = self.end_date.strftime(DATE_ONLY_2) date_range = f"Scraping: {start_date_str}-{end_date_str}" if self.scrape_date: scrape_date_str = self.scrape_date.strftime(DATE_ONLY_2) date_range += f" (Now: {scrape_date_str})" job_heading = f"{job_name_id} {date_range}" print_heading(job_heading, fg="bright_yellow") if not self.task_results: return for (message, text_color) in self.task_results: print_message(message, fg=text_color) return Result.Ok()
def display_many_matches_found(self, matches): subprocess.run(["clear"]) for match_dict in matches: match_dict["invalid_pfx"].pop("at_bat_id") match_dict["invalid_pfx"].pop("pitcher_id") match_dict["invalid_pfx"].pop("batter_id") for match in match_dict["missing_pfx"]: match.pop("at_bat_id") match.pop("pitcher_id") match.pop("batter_id") subprocess.run(["clear"]) error = "Multiple at bats were found for the invalid PitchFX data below (only one match is expected):\n" print_message(error, fg="bright_yellow", bold=True) all_rows = [ match_dict["invalid_pfx"], *list(match_dict["missing_pfx"]) ] print_message(tabulate(all_rows, headers="keys"), wrap=False) print() pause(message="Press any key to continue...")
def launch(self): subprocess.run(["clear"]) print_message(f"Variable Name: {self.setting_name}\n", fg="bright_magenta", bold=True) print_message(f"Current Value: {self.current_setting}\n", fg="bright_yellow", bold=True) if not yes_no_prompt(prompt="\nChange current setting?"): return Result.Ok(self.exit_menu) user_confirmed, new_value = False, None while not user_confirmed: subprocess.run(["clear"]) prompt = f"Enter a new value for {self.setting_name}:\n" new_value = Input( prompt, word_color=colors.foreground["default"]).launch() result = self.confirm_new_value(new_value) if result.failure: return Result.Ok(self.exit_menu) user_confirmed = result.value result = self.dotenv.change_value(self.setting_name, new_value) if not self.restart_required: return result print_message(RESTART_WARNING, fg="bright_magenta", bold=True) pause(message="Press any key to continue...") exit(0)
def launch_no_prompts(self, game_date): self.current_game_date = game_date self.scrape_year = game_date.year self.pbar_manager = enlighten.get_manager() self.init_progress_bars(game_date=self.all_dates_in_season[0]) subprocess.run(["clear"]) game_ids = self.date_game_id_map.get(self.current_game_date, None) if not game_ids: game_date_str = self.current_game_date.strftime(DATE_MONTH_NAME) message = f"All games on {game_date_str} have already been combined." print_message(message, fg="bright_cyan", bold=True) self.close_progress_bars() return ([], []) self.combine_selected_games(self.current_game_date, game_ids) self.date_progress_bar.update() self.close_progress_bars() invalid_pfx_game_ids, pfx_error_game_ids = [], [] if self.total_games_invalid_pitchfx: invalid_pfx_game_ids = list(self.invalid_pfx.keys()) if self.total_games_pitchfx_error: pfx_error_game_ids = list(self.pfx_errors.keys()) return (invalid_pfx_game_ids, pfx_error_game_ids)
def combine_games_for_date(self): result = audit_report_season_prompt(self.app.audit_report) if result.failure: return result self.scrape_year = result.value result = self.game_date_prompt() if result.failure: return result self.current_game_date = result.value self.pbar_manager = enlighten.get_manager() self.init_progress_bars(game_date=self.current_game_date) subprocess.run(["clear"]) game_ids = self.date_game_id_map.get(self.current_game_date, None) if not game_ids: game_date_str = self.current_game_date.strftime(DATE_MONTH_NAME) message = f"All games on {game_date_str} have already been combined." print_message(message, fg="bright_cyan", bold=True) self.close_progress_bars() return Result.Ok() result = self.combine_selected_games(self.current_game_date, game_ids) self.date_progress_bar.update() self.close_progress_bars() return result
def combine_scraped_data_for_game(self, combine_game_id): subprocess.run(["clear"]) spinner = Halo(color=get_random_cli_color(), spinner=get_random_dots_spinner()) spinner.text = f"Combining scraped data for {combine_game_id}..." spinner.start() result = self.combine_data.execute(combine_game_id) if not (result["gather_scraped_data_success"] and result["combined_data_success"] and result["update_pitch_apps_success"]): spinner.fail(f"Failed to combine data for {combine_game_id}!") print_message(result["error"], wrap=False, fg="bright_red", bold=True) return Result.Fail(result["error"]) spinner.stop() pfx_errors = result["results"]["pfx_errors"] if pfx_errors.get("pitchfx_error", []): self.pfx_errors[combine_game_id] = pfx_errors["pitchfx_error"] if pfx_errors.get("invalid_pitchfx", []): self.invalid_pfx[combine_game_id] = pfx_errors["invalid_pitchfx"] if self.total_pitch_apps_any_pitchfx_error > 0: pitch_apps_plural = ("pitch appearances" if self.total_pitch_apps_any_pitchfx_error > 1 else "pitch appearance") at_bats_plural = "at bats" if self.total_at_bats_any_pitchfx_error > 1 else "at bat" message = ( f"PitchFX data could not be reconciled for game: {combine_game_id}\n" f"{self.total_pitch_apps_any_pitchfx_error} {pitch_apps_plural} with data errors " f"({self.total_at_bats_any_pitchfx_error} total {at_bats_plural})\n" ) print_message(message, fg="bright_yellow", bold=True) else: message = f"All scraped data for {combine_game_id} was successfully combined!" print_message(message, fg="bright_cyan", bold=True) pause(message="Press any key to continue...") return Result.Ok()
def user_cancelled(db_session, active_job, spinner, signal_received, frame): spinner.stop() print_message("\nJob cancelled by user!", fg="yellow", bold=True) pause(message="Press any key to continue...") exit(0)
def display_initial_task_status(self): if node_is_installed(): print_message("Node.js Installed.............: YES", fg="bright_green", bold=True) else: print_message("Node.js Installed.............: NO", fg="bright_red", bold=True) if node_modules_folder_exists(): print_message("Electron/Nightmare Installed..: YES", fg="bright_green", bold=True) else: print_message("Electron/Nightmare Installed..: NO", fg="bright_red", bold=True) if self.db_setup_complete: print_message("SQLite DB Initialized.........: YES", fg="bright_green", bold=True) else: print_message("SQLite DB Initialized.........: NO", fg="bright_red", bold=True)