def report_sync_results(self):
subprocess.run(["clear"])
self.print_header_message()
if not self.sync_results:
return
for task_result in self.sync_results:
print_message(task_result[0], wrap=False, fg=task_result[1])
def print_description(self):
if isinstance(self.setting.description, list):
for line in self.setting.description:
print_message(line)
else:
print_message(self.setting.description)
print()
def synchronize_files(self):
if self.all_files_are_in_sync:
message = "All files for selected data sets are in sync!"
print_message(message, fg="bright_green", bold=True)
pause(message="Press any key to continue...")
return Result.Ok()
for file_type, file_type_dict in self.sync_files.items():
for data_set, (out_of_sync, missing_files,
outdated_files) in file_type_dict.items():
if not out_of_sync:
continue
all_sync_files = []
missing_count = 0
outdated_count = 0
if missing_files:
all_sync_files.extend(missing_files)
missing_count = len(missing_files)
if outdated_files:
all_sync_files.extend(outdated_files)
outdated_count = len(outdated_files)
table_viewer = self.create_table_viewer(
all_sync_files, data_set, file_type, missing_count,
outdated_count)
apply_changes = table_viewer.launch()
if apply_changes:
self.apply_pending_changes(file_type, data_set,
missing_files, outdated_files)
return Result.Ok()
def launch(self):
subprocess.run(["clear"])
print_heading(self.menu_heading, fg="bright_yellow")
if not node_is_installed():
print_message(INSTALL_ERROR, fg="bright_red", bold=True)
pause(message="Press any key to continue...")
return
if not NODEJS_INBOX.exists():
NODEJS_INBOX.mkdir(parents=True, exist_ok=True)
if node_modules_folder_exists():
message = UPDATE_MESSAGE
prompt = UPDATE_PROMPT
temp_folder = None
command = "npm update --timeout=9999999"
else:
message = INSTALL_MESSAGE
prompt = INSTALL_PROMPT
temp_folder = TemporaryDirectory(dir=NIGHTMAREJS_FOLDER)
command = f"npm install --timeout=9999999 --cache={temp_folder.name}"
print_message(message, fg="bright_yellow")
if not yes_no_prompt(prompt, wrap=False):
return Result.Ok(self.exit_menu)
subprocess.run(["clear"])
print_heading(self.menu_heading, fg="bright_yellow")
result = run_command(command, cwd=str(NIGHTMAREJS_FOLDER))
if result.failure:
return result
if temp_folder:
temp_folder.cleanup()
pause(message="\nPress any key to continue...")
return Result.Ok(self.exit_menu)
def launch(self):
subprocess.run(["clear"])
print_heading("This feature is not currently available",
fg="bright_yellow")
print_message(
"Sorry for any inconvenience, I promise to finish this soon!")
pause(message="\nPress any key to return to the previous menu...")
return Result.Ok(True)
def exit_app_error(message):
if message:
if isinstance(message, list):
for m in message:
print_message(m, fg="bright_red", bold=True)
else:
print_message(message, fg="bright_red", bold=True)
return 1
def report_sync_results(self):
subprocess.run(["clear"])
src_folder = "S3 bucket" if self.sync_direction == SyncDirection.DOWN_TO_LOCAL else "local folder"
dest_folder = "local folder" if self.sync_direction == SyncDirection.DOWN_TO_LOCAL else "S3 bucket"
print_heading(f"Syncing data from {src_folder} to {dest_folder}",
fg="bright_yellow")
for (result, text_color) in self.sync_results:
print_message(result, fg=text_color)
def prompt_user_apply_patch_list(self):
subprocess.run(["clear"])
prompt = "Would you like to apply the patch to fix the invalid data?"
self.update_menu_heading("Apply Patch?", heading_color="bright_cyan")
print_message(
f"A patch list for {self.game_id} was successfully created!\n",
fg="bright_cyan")
return yes_no_prompt(prompt)
def prompt_user_view_patched_data(self):
subprocess.run(["clear"])
prompt = "Would you like to see a report detailing the changes that were made by applying the patch list?"
self.update_menu_heading("View Patch Results?",
heading_color="bright_cyan")
print_message(
"The patch was successfully applied to the PitchFX data!\n",
fg="bright_cyan")
return yes_no_prompt(prompt)
def display_pitchfx_errors(self):
games_plural = "games contain" if self.total_games_any_pfx_error > 1 else "game contains"
ab_plural = "at bats" if self.total_at_bats_any_pitchfx_error > 1 else "at bat"
message = (
f"{self.total_games_any_pfx_error} {games_plural} invalid PitchFX data for a total "
f"of {self.total_at_bats_any_pitchfx_error} {ab_plural}, you can view details "
"of each at bat and attempt to fix these errors using the Investigate Failures menu."
)
print_message(message, fg="bright_cyan")
print()
def display_games_failed_to_combine(self):
error_message = f"Error prevented scraped data being combined for {len(self.failed_game_ids)} games:"
error_details = [{
"bbref_game_id": game_id,
"error": error
} for game_id, error in self.combine_data_fail_results]
print_message(error_message, wrap=False, fg="bright_red", bold=True)
print_message(tabulate(error_details, headers="keys"),
wrap=False,
fg="bright_red")
def display_no_match_found(self, matches):
subprocess.run(["clear"])
for match_dict in matches:
match_dict["invalid_pfx"].pop("at_bat_id")
match_dict["invalid_pfx"].pop("pitcher_id")
match_dict["invalid_pfx"].pop("batter_id")
subprocess.run(["clear"])
error = "No matching at bats were found for the invalid PitchFX data below:\n"
print_message(error, fg="bright_red", bold=True)
unmatched_rows = [match_dict["invalid_pfx"] for match_dict in matches]
print_message(tabulate(unmatched_rows, headers="keys"), wrap=False)
print()
pause(message="Press any key to continue...")
def display_menu_text(self):
print_heading(f"vigorish v{__version__}", fg="bright_yellow")
if self.audit_report:
self.display_audit_report()
elif self.initial_setup_complete:
print_message(
"All prerequisites are installed and database is initialized.")
else:
self.display_initial_task_status()
print_message("\nMain Menu:\n",
fg="bright_green",
bold=True,
underline=True)
def display_audit_report(self):
table_rows = [{
"season":
f"MLB {year} ({report['total_games']} games)",
"scraped":
len(report["scraped"]),
"combined":
len(report["successful"]),
"failed":
len(report["failed"]) + len(report["pfx_error"]) +
len(report["invalid_pfx"]),
} for year, report in self.audit_report.items()]
print_message(tabulate(table_rows, headers="keys"), wrap=False)
def display_results(self):
subprocess.run(["clear"])
if self.combined_success_and_no_pfx_errors:
plural = "games total" if self.total_games > 1 else "game"
success_message = f"\nAll game data ({self.total_games} {plural}) combined, no errors"
print_message(success_message,
wrap=False,
fg="bright_cyan",
bold=True)
if self.failed_game_ids:
self.display_games_failed_to_combine()
if self.all_pfx_errors:
self.display_pitchfx_errors()
pause(message="Press any key to continue...")
def launch(self):
subprocess.run(["clear"])
setting_heading = f"Setting: {self.setting_name_title} (Type: {self.data_type.name})"
print_heading(setting_heading, fg="bright_magenta")
self.print_description()
print_message(self.current_settings,
wrap=False,
fg="bright_yellow",
bold=True)
if yes_no_prompt("\nChange current setting?"):
change_setting_menu = ChangeConfigSettting(self.app,
self.setting_name)
return change_setting_menu.launch()
return Result.Ok(self.exit_menu)
def patch_invalid_pfx_single_game(self):
result = self.patch_invalid_pfx.execute(self.game_id)
if result.failure:
header = f"Invalid PitchFX Data for {self.game_id}\n"
subprocess.run(["clear"])
print_message(header, wrap=False, bold=True, underline=True)
print_message(result.error, fg="bright_yellow")
pause(message="Press any key to continue...")
return Result.Ok(True)
if not self.prompt_user_create_patch_list():
return Result.Ok(True)
result = self.patch_invalid_pfx.match_missing_pfx_data()
if result.failure:
return result
for result, matches in self.patch_invalid_pfx.match_results.items():
if result == "success":
for num, match_dict in enumerate(matches, start=1):
match_dict["patch"] = self.prompt_user_create_patch(
num, len(matches), match_dict)
if result == "no_matches":
self.display_no_match_found(matches)
if result == "many_matches":
self.display_many_matches_found(matches)
if "success" not in self.patch_invalid_pfx.match_results:
header = f"Invalid PitchFX Data for {self.game_id}\n"
message = (
"Unable to identify missing data that matches the invalid PitchFX data for this "
"game. You should inspect the combined data JSON file for this game and "
"investigate the invalid data manually.\n")
subprocess.run(["clear"])
print_message(header, wrap=False, bold=True, underline=True)
print_message(message, fg="bright_yellow")
pause(message="Press any key to continue...")
return Result.Ok(True)
result = self.patch_invalid_pfx.create_patch_list()
if result.failure:
return result
if not self.patch_invalid_pfx.patch_list or not self.prompt_user_apply_patch_list(
):
return Result.Ok(True)
result = self.patch_invalid_pfx.apply_patch_list()
if result.failure:
return result
self.patch_results = result.value
print()
if self.patch_results["fixed_all_errors"]:
patch_result = f"PitchFX data for {self.game_id} is now completely reconciled (no errors of any type)!\n"
print_success(patch_result)
if self.patch_results["invalid_pfx"]:
patch_result = f"{self.game_id} still contains invalid PitchFX data after applying the patch list.\n"
print_error(patch_result)
if self.patch_results["pfx_errors"]:
patch_result = f"{self.game_id} still contains PitchFX data errors associated with valid at bats.\n"
print_error(patch_result)
pause(message="Press any key to continue...")
subprocess.run(["clear"])
if self.prompt_user_view_patched_data():
self.display_patched_data_tables(
**self.patch_results["patch_diff_report"])
return Result.Ok()
def check_app_status(self):
if not self.db_setup_complete:
return
color = get_random_cli_color()
if not self.initialized:
f = Figlet(font=get_random_figlet_font(), width=120)
print_message(f.renderText("vigorish"),
wrap=False,
fg=f"bright_{color}")
spinner = Halo(spinner=get_random_dots_spinner(), color=color)
spinner.text = "Updating metrics..." if self.initialized else "Loading..."
spinner.start()
if self.initialized:
del self.app.audit_report
self.audit_report = self.app.audit_report
spinner.stop()
def confirm_job_details(self, data_sets, start_date, end_date, job_name):
subprocess.run(["clear"])
heading = "Confirm job details"
data_set_space = "\n\t "
confirm_job_name = ""
if job_name:
confirm_job_name = f"Job Name....: {job_name}\n"
job_details = (
f"{confirm_job_name}"
f"Start date..: {start_date.strftime(DATE_ONLY_2)}\n"
f"End Date....: {end_date.strftime(DATE_ONLY_2)}\n"
f"Data Sets...: {data_set_space.join([DATA_SET_TO_NAME_MAP[ds] for ds in data_sets])}"
)
print_heading(heading, fg="bright_yellow")
print_message(job_details, wrap=False, fg="bright_yellow")
return yes_no_prompt(prompt="\nAre the details above correct?")
def combine_scraped_data_for_game(self):
subprocess.run(["clear"])
spinner = Halo(spinner=get_random_dots_spinner(),
color=get_random_cli_color())
spinner.text = f"Combining scraped data for {self.game_id}..."
spinner.start()
result = self.combine_data.execute(self.game_id)
if (not result["gather_scraped_data_success"]
or not result["combined_data_success"]
or not result["update_pitch_apps_success"]):
spinner.fail(f"Failed to combine data for {self.game_id}!")
pause(message="Press any key to continue...")
return Result.Fail(result["error"])
pfx_errors = result["results"]["pfx_errors"]
fail_results = [
pfx_errors.pop("pitchfx_error", {}),
pfx_errors.pop("invalid_pitchfx", {}),
]
if all(len(f) <= 0 for f in fail_results):
spinner.succeed(
f"All scraped data for {self.game_id} was successfully combined!"
)
pause(message="Press any key to continue...")
return Result.Ok()
spinner.stop()
subprocess.run(["clear"])
total_pitch_apps = sum(len(f.keys()) for f in fail_results if f)
pitch_apps_plural = "pitch appearances" if total_pitch_apps > 1 else "pitch appearance"
total_at_bats = sum(
len(at_bat_ids) for f in fail_results for at_bat_ids in f.values()
if f)
at_bats_plural = "at bats" if total_at_bats > 1 else "at bat"
error_header = f"PitchFX data could not be reconciled for game: {self.game_id}\n"
error_message = (
f"{total_pitch_apps} {pitch_apps_plural} with data errors ({total_at_bats} total {at_bats_plural})\n"
)
print_message(error_header,
wrap=False,
fg="bright_red",
bold=True,
underline=True)
print_message(error_message, fg="bright_red")
if not self.prompt_user_investigate_failures():
pause(message="Press any key to continue...")
return Result.Ok()
subprocess.run(["clear"])
return self.patch_invalid_pfx_single_game()
def launch(self):
subprocess.run(["clear"])
print_message("*** Job Details ***", fg="bright_yellow", bold=True)
job_details = report_dict(self.job_details,
title="",
title_prefix="",
title_suffix="")
print_message(f"{job_details}\n", wrap=False, fg="bright_yellow")
if self.db_job.errors:
print_message("*** Errors ***", fg="bright_red", bold=True)
print_message(self.db_job.error_messages, fg="bright_red")
if self.job_status == JobStatus.INCOMPLETE:
result = self.incomplete_job_options_prompt()
if result.failure:
return Result.Ok(self.exit_menu)
user_choice = result.value
if user_choice == "RUN":
job_runner = JobRunner(app=self.app, db_job=self.db_job)
result = job_runner.execute()
if result.failure:
return result
return Result.Ok(True)
if user_choice == "CANCEL":
self.db_job.status = JobStatus.COMPLETE
self.db_session.commit()
subprocess.run(["clear"])
print_message("Job was successfully cancelled.",
fg="bright_red",
bold=True)
pause(message="Press any key to continue...")
return Result.Ok(True)
if self.job_status == JobStatus.ERROR:
result = self.failed_job_options_prompt()
if result.failure:
return Result.Ok(self.exit_menu)
user_choice = result.value
if user_choice == "RETRY":
job_runner = JobRunner(app=self.app, db_job=self.db_job)
result = job_runner.execute()
if result.failure:
return result
return Result.Ok(True)
else:
pause(message="\nPress any key to return to the previous menu...")
return Result.Ok(self.exit_menu)
def apply_pending_changes(self, file_type, data_set, missing_files,
outdated_files):
subprocess.run(["clear"])
self.spinner = Halo(spinner=get_random_dots_spinner(),
color=get_random_cli_color())
self.spinner.start()
self.s3_sync.events.sync_files_progress += self.update_sync_progress
self.s3_sync.sync_files(self.sync_direction, missing_files,
outdated_files, file_type, data_set, self.year)
self.s3_sync.events.sync_files_progress -= self.update_sync_progress
self.spinner.stop()
src_folder = "S3 bucket" if self.sync_direction == SyncDirection.DOWN_TO_LOCAL else "local folder"
dest_folder = "local folder" if self.sync_direction == SyncDirection.DOWN_TO_LOCAL else "S3 bucket"
message = (
f"All changes have been applied, MLB {self.year} {data_set} {file_type} files in {src_folder} "
f"have been synced to {dest_folder}!")
print_message(message, fg="bright_green", bold=True)
pause(message="Press any key to continue...")
def report_task_results(self):
subprocess.run(["clear"])
job_name_id = f"Job Name: {self.job_name} (ID: {self.job_id.upper()})"
if self.job_name == self.job_id:
job_name_id = f"Job ID: {self.job_id.upper()}"
start_date_str = self.start_date.strftime(DATE_ONLY_2)
end_date_str = self.end_date.strftime(DATE_ONLY_2)
date_range = f"Scraping: {start_date_str}-{end_date_str}"
if self.scrape_date:
scrape_date_str = self.scrape_date.strftime(DATE_ONLY_2)
date_range += f" (Now: {scrape_date_str})"
job_heading = f"{job_name_id} {date_range}"
print_heading(job_heading, fg="bright_yellow")
if not self.task_results:
return
for (message, text_color) in self.task_results:
print_message(message, fg=text_color)
return Result.Ok()
def display_many_matches_found(self, matches):
subprocess.run(["clear"])
for match_dict in matches:
match_dict["invalid_pfx"].pop("at_bat_id")
match_dict["invalid_pfx"].pop("pitcher_id")
match_dict["invalid_pfx"].pop("batter_id")
for match in match_dict["missing_pfx"]:
match.pop("at_bat_id")
match.pop("pitcher_id")
match.pop("batter_id")
subprocess.run(["clear"])
error = "Multiple at bats were found for the invalid PitchFX data below (only one match is expected):\n"
print_message(error, fg="bright_yellow", bold=True)
all_rows = [
match_dict["invalid_pfx"], *list(match_dict["missing_pfx"])
]
print_message(tabulate(all_rows, headers="keys"), wrap=False)
print()
pause(message="Press any key to continue...")
def launch(self):
subprocess.run(["clear"])
print_message(f"Variable Name: {self.setting_name}\n",
fg="bright_magenta",
bold=True)
print_message(f"Current Value: {self.current_setting}\n",
fg="bright_yellow",
bold=True)
if not yes_no_prompt(prompt="\nChange current setting?"):
return Result.Ok(self.exit_menu)
user_confirmed, new_value = False, None
while not user_confirmed:
subprocess.run(["clear"])
prompt = f"Enter a new value for {self.setting_name}:\n"
new_value = Input(
prompt, word_color=colors.foreground["default"]).launch()
result = self.confirm_new_value(new_value)
if result.failure:
return Result.Ok(self.exit_menu)
user_confirmed = result.value
result = self.dotenv.change_value(self.setting_name, new_value)
if not self.restart_required:
return result
print_message(RESTART_WARNING, fg="bright_magenta", bold=True)
pause(message="Press any key to continue...")
exit(0)
def launch_no_prompts(self, game_date):
self.current_game_date = game_date
self.scrape_year = game_date.year
self.pbar_manager = enlighten.get_manager()
self.init_progress_bars(game_date=self.all_dates_in_season[0])
subprocess.run(["clear"])
game_ids = self.date_game_id_map.get(self.current_game_date, None)
if not game_ids:
game_date_str = self.current_game_date.strftime(DATE_MONTH_NAME)
message = f"All games on {game_date_str} have already been combined."
print_message(message, fg="bright_cyan", bold=True)
self.close_progress_bars()
return ([], [])
self.combine_selected_games(self.current_game_date, game_ids)
self.date_progress_bar.update()
self.close_progress_bars()
invalid_pfx_game_ids, pfx_error_game_ids = [], []
if self.total_games_invalid_pitchfx:
invalid_pfx_game_ids = list(self.invalid_pfx.keys())
if self.total_games_pitchfx_error:
pfx_error_game_ids = list(self.pfx_errors.keys())
return (invalid_pfx_game_ids, pfx_error_game_ids)
def combine_games_for_date(self):
result = audit_report_season_prompt(self.app.audit_report)
if result.failure:
return result
self.scrape_year = result.value
result = self.game_date_prompt()
if result.failure:
return result
self.current_game_date = result.value
self.pbar_manager = enlighten.get_manager()
self.init_progress_bars(game_date=self.current_game_date)
subprocess.run(["clear"])
game_ids = self.date_game_id_map.get(self.current_game_date, None)
if not game_ids:
game_date_str = self.current_game_date.strftime(DATE_MONTH_NAME)
message = f"All games on {game_date_str} have already been combined."
print_message(message, fg="bright_cyan", bold=True)
self.close_progress_bars()
return Result.Ok()
result = self.combine_selected_games(self.current_game_date, game_ids)
self.date_progress_bar.update()
self.close_progress_bars()
return result
def combine_scraped_data_for_game(self, combine_game_id):
subprocess.run(["clear"])
spinner = Halo(color=get_random_cli_color(),
spinner=get_random_dots_spinner())
spinner.text = f"Combining scraped data for {combine_game_id}..."
spinner.start()
result = self.combine_data.execute(combine_game_id)
if not (result["gather_scraped_data_success"]
and result["combined_data_success"]
and result["update_pitch_apps_success"]):
spinner.fail(f"Failed to combine data for {combine_game_id}!")
print_message(result["error"],
wrap=False,
fg="bright_red",
bold=True)
return Result.Fail(result["error"])
spinner.stop()
pfx_errors = result["results"]["pfx_errors"]
if pfx_errors.get("pitchfx_error", []):
self.pfx_errors[combine_game_id] = pfx_errors["pitchfx_error"]
if pfx_errors.get("invalid_pitchfx", []):
self.invalid_pfx[combine_game_id] = pfx_errors["invalid_pitchfx"]
if self.total_pitch_apps_any_pitchfx_error > 0:
pitch_apps_plural = ("pitch appearances"
if self.total_pitch_apps_any_pitchfx_error > 1
else "pitch appearance")
at_bats_plural = "at bats" if self.total_at_bats_any_pitchfx_error > 1 else "at bat"
message = (
f"PitchFX data could not be reconciled for game: {combine_game_id}\n"
f"{self.total_pitch_apps_any_pitchfx_error} {pitch_apps_plural} with data errors "
f"({self.total_at_bats_any_pitchfx_error} total {at_bats_plural})\n"
)
print_message(message, fg="bright_yellow", bold=True)
else:
message = f"All scraped data for {combine_game_id} was successfully combined!"
print_message(message, fg="bright_cyan", bold=True)
pause(message="Press any key to continue...")
return Result.Ok()
def user_cancelled(db_session, active_job, spinner, signal_received, frame):
spinner.stop()
print_message("\nJob cancelled by user!", fg="yellow", bold=True)
pause(message="Press any key to continue...")
exit(0)
def display_initial_task_status(self):
if node_is_installed():
print_message("Node.js Installed.............: YES",
fg="bright_green",
bold=True)
else:
print_message("Node.js Installed.............: NO",
fg="bright_red",
bold=True)
if node_modules_folder_exists():
print_message("Electron/Nightmare Installed..: YES",
fg="bright_green",
bold=True)
else:
print_message("Electron/Nightmare Installed..: NO",
fg="bright_red",
bold=True)
if self.db_setup_complete:
print_message("SQLite DB Initialized.........: YES",
fg="bright_green",
bold=True)
else:
print_message("SQLite DB Initialized.........: NO",
fg="bright_red",
bold=True)
