def time_pitch_thrown(self):
match = PFX_TIMESTAMP_REGEX.match(self.park_sv_id)
if not match:
return None
group_dict = match.groupdict()
if int(group_dict["second"]) >= 60:
group_dict["second"] = 0
game_dict = validate_bbref_game_id(self.bbref_game_id).value
timestamp = datetime(
game_dict["game_date"].year,
int(group_dict["month"]),
int(group_dict["day"]),
int(group_dict["hour"]),
int(group_dict["minute"]),
int(group_dict["second"]),
)
thrown_replace_tz = timestamp.replace(tzinfo=TIME_ZONE_NEW_YORK)
thrown_as_tz = timestamp.replace(
tzinfo=timezone.utc).astimezone(TIME_ZONE_NEW_YORK)
if not self.game_start_time:
return thrown_as_tz
seconds_since_thrown_replace_tz = (
thrown_replace_tz - self.game_start_time).total_seconds()
seconds_since_thrown_as_tz = (thrown_as_tz -
self.game_start_time).total_seconds()
if seconds_since_thrown_as_tz < 0 and seconds_since_thrown_replace_tz < 0:
return None
if seconds_since_thrown_as_tz > 0 and seconds_since_thrown_replace_tz < 0:
return thrown_as_tz
if seconds_since_thrown_as_tz < 0 and seconds_since_thrown_replace_tz > 0:
return thrown_replace_tz
return thrown_as_tz if seconds_since_thrown_as_tz < seconds_since_thrown_replace_tz else thrown_replace_tz
def get_game_id_date_map(self, bbref_game_id):
game_dict = validate_bbref_game_id(bbref_game_id).value
return {
"game_id": bbref_game_id,
"game_date": game_dict["game_date"],
"count": 1
}
def bb_game_id(self):
result = validate_bbref_game_id(self.bbref_game_id)
game_date = result.value["game_date"]
game_number = result.value["game_number"]
away_team_id = get_brooks_team_id(
self.away_team_data.team_id_br).lower()
home_team_id = get_brooks_team_id(
self.home_team_data.team_id_br).lower()
return (
f"gid_{game_date.year}_{game_date.month:02d}_{game_date.day:02d}_"
f"{away_team_id}mlb_{home_team_id}mlb_{game_number}")
def delete_bbref_boxscore_local_file(self, bbref_game_id):
result = validate_bbref_game_id(bbref_game_id)
if result.failure:
return result
game_dict = result.value
return self.file_helper.perform_local_file_task(
task=LocalFileTask.DELETE_FILE,
data_set=DataSet.BBREF_BOXSCORES,
file_type=VigFile.PARSED_JSON,
game_date=game_dict["game_date"],
bbref_game_id=bbref_game_id,
)
def get_combined_game_data_s3(self, bbref_game_id): # pragma: no cover
result = validate_bbref_game_id(bbref_game_id)
if result.failure:
return result
game_dict = result.value
return self.file_helper.perform_s3_task(
task=S3FileTask.DOWNLOAD,
data_set=DataSet.ALL,
file_type=VigFile.COMBINED_GAME_DATA,
game_date=game_dict["game_date"],
bbref_game_id=bbref_game_id,
)
def get_brooks_pitchfx_patch_list_local_file(self, bbref_game_id):
result = validate_bbref_game_id(bbref_game_id)
if result.failure:
return result
game_dict = result.value
return self.file_helper.perform_local_file_task(
task=LocalFileTask.READ_FILE,
data_set=DataSet.BROOKS_PITCHFX,
file_type=VigFile.PATCH_LIST,
game_date=game_dict["game_date"],
bbref_game_id=bbref_game_id,
)
def get_bbref_boxscore_patch_list_local_file(self, bbref_game_id):
result = validate_bbref_game_id(bbref_game_id)
if result.failure:
return result
game_dict = result.value
return self.file_helper.perform_local_file_task(
task=LocalFileTask.READ_FILE,
data_set=DataSet.BBREF_BOXSCORES,
file_type=VigFile.PATCH_LIST,
game_date=game_dict["game_date"],
bbref_game_id=bbref_game_id,
)
def get_bbref_boxscore_patch_list_s3(self, bbref_game_id): # pragma: no cover
result = validate_bbref_game_id(bbref_game_id)
if result.failure:
return result
game_dict = result.value
return self.file_helper.perform_s3_task(
task=S3FileTask.DOWNLOAD,
data_set=DataSet.BBREF_BOXSCORES,
file_type=VigFile.PATCH_LIST,
game_date=game_dict["game_date"],
bbref_game_id=bbref_game_id,
)
def get_combined_game_data_local_file(self, bbref_game_id):
result = validate_bbref_game_id(bbref_game_id)
if result.failure:
return result
game_dict = result.value
return self.file_helper.perform_local_file_task(
task=LocalFileTask.READ_FILE,
data_set=DataSet.ALL,
file_type=VigFile.COMBINED_GAME_DATA,
game_date=game_dict["game_date"],
bbref_game_id=bbref_game_id,
)
def delete_bbref_boxscore_s3(self, bbref_game_id): # pragma: no cover
result = validate_bbref_game_id(bbref_game_id)
if result.failure:
return result
game_dict = result.value
return self.file_helper.perform_s3_task(
task=S3FileTask.DELETE,
data_set=DataSet.BBREF_BOXSCORES,
file_type=VigFile.PARSED_JSON,
game_date=game_dict["game_date"],
bbref_game_id=bbref_game_id,
)
def save_html_bbref_boxscore_local_file(self, bbref_game_id, html):
result = validate_bbref_game_id(bbref_game_id)
if result.failure:
return result
game_dict = result.value
return self.file_helper.perform_local_file_task(
task=LocalFileTask.WRITE_FILE,
data_set=DataSet.BBREF_BOXSCORES,
file_type=VigFile.SCRAPED_HTML,
game_date=game_dict["game_date"],
scraped_data=html,
bbref_game_id=bbref_game_id,
)
def save_combined_game_data_s3(self, combined_data): # pragma: no cover
result = validate_bbref_game_id(combined_data["bbref_game_id"])
if result.failure:
return result
game_dict = result.value
return self.file_helper.perform_s3_task(
task=S3FileTask.UPLOAD,
data_set=DataSet.ALL,
file_type=VigFile.COMBINED_GAME_DATA,
game_date=game_dict["game_date"],
scraped_data=combined_data,
bbref_game_id=combined_data["bbref_game_id"],
)
def delete_html_bbref_boxscore_s3(self, bbref_game_id): # pragma: no cover
"""Delete scraped HTML for bbref boxscore page from S3."""
result = validate_bbref_game_id(bbref_game_id)
if result.failure:
return result
game_dict = result.value
return self.file_helper.perform_s3_task(
task=S3FileTask.DELETE,
data_set=DataSet.BBREF_BOXSCORES,
file_type=VigFile.SCRAPED_HTML,
game_date=game_dict["game_date"],
bbref_game_id=bbref_game_id,
)
def download_html_bbref_boxscore(self, bbref_game_id): # pragma: no cover
"""Download raw HTML for bbref daily scoreboard page."""
result = validate_bbref_game_id(bbref_game_id)
if result.failure:
return result
game_dict = result.value
return self.file_helper.perform_s3_task(
task=S3FileTask.DOWNLOAD,
file_type=VigFile.SCRAPED_HTML,
data_set=DataSet.BBREF_BOXSCORES,
game_date=game_dict["game_date"],
bbref_game_id=bbref_game_id,
)
def save_brooks_pitchfx_patch_list_s3(self, patch_list): # pragma: no cover
result = validate_bbref_game_id(patch_list.url_id)
if result.failure:
return result
game_dict = result.value
return self.file_helper.perform_s3_task(
task=S3FileTask.UPLOAD,
data_set=DataSet.BROOKS_PITCHFX,
file_type=VigFile.PATCH_LIST,
game_date=game_dict["game_date"],
scraped_data=patch_list,
bbref_game_id=patch_list.url_id,
)
def save_combined_game_data_local_file(self, combined_data):
result = validate_bbref_game_id(combined_data["bbref_game_id"])
if result.failure:
return result
game_dict = result.value
return self.file_helper.perform_local_file_task(
task=LocalFileTask.WRITE_FILE,
data_set=DataSet.ALL,
file_type=VigFile.COMBINED_GAME_DATA,
game_date=game_dict["game_date"],
scraped_data=combined_data,
bbref_game_id=combined_data["bbref_game_id"],
)
def save_brooks_pitchfx_patch_list_local_file(self, patch_list):
result = validate_bbref_game_id(patch_list.url_id)
if result.failure:
return result
game_dict = result.value
return self.file_helper.perform_local_file_task(
task=LocalFileTask.WRITE_FILE,
data_set=DataSet.BROOKS_PITCHFX,
file_type=VigFile.PATCH_LIST,
game_date=game_dict["game_date"],
scraped_data=patch_list,
bbref_game_id=patch_list.url_id,
)
def delete_html_bbref_boxscore_local_file(self, bbref_game_id):
"""Delete scraped HTML for bbref boxscore page from local folder."""
result = validate_bbref_game_id(bbref_game_id)
if result.failure:
return result
game_dict = result.value
return self.file_helper.perform_local_file_task(
task=LocalFileTask.DELETE_FILE,
data_set=DataSet.BBREF_BOXSCORES,
file_type=VigFile.SCRAPED_HTML,
game_date=game_dict["game_date"],
bbref_game_id=bbref_game_id,
)
def decode_bbref_boxscore_patch_list_s3(self, bbref_game_id): # pragma: no cover
result = self.get_bbref_boxscore_patch_list_s3(bbref_game_id)
if result.failure:
return result
result = validate_bbref_game_id(bbref_game_id)
if result.failure:
return result
game_dict = result.value
return self.file_helper.perform_local_file_task(
task=LocalFileTask.DECODE_JSON,
data_set=DataSet.BBREF_BOXSCORES,
file_type=VigFile.PATCH_LIST,
game_date=game_dict["game_date"],
bbref_game_id=bbref_game_id,
delete_file=True,
)
def decode_brooks_pitchfx_patch_list_local_file(self, bbref_game_id):
result = self.get_brooks_pitchfx_patch_list_local_file(bbref_game_id)
if result.failure:
return result
result = validate_bbref_game_id(bbref_game_id)
if result.failure:
return result
game_dict = result.value
return self.file_helper.perform_local_file_task(
task=LocalFileTask.DECODE_JSON,
data_set=DataSet.BROOKS_PITCHFX,
file_type=VigFile.PATCH_LIST,
game_date=game_dict["game_date"],
bbref_game_id=bbref_game_id,
delete_file=False,
)
def upload_html_bbref_boxscore(self, bbref_game_id,
html): # pragma: no cover
result = self.save_html_local(DataSet.BBREF_BOXSCORES, bbref_game_id,
html)
if result.failure:
return result
result = validate_bbref_game_id(bbref_game_id)
if result.failure:
return result
game_dict = result.value
return self.file_helper.perform_s3_task(
task=S3FileTask.UPLOAD,
file_type=VigFile.SCRAPED_HTML,
data_set=DataSet.BBREF_BOXSCORES,
game_date=game_dict["game_date"],
bbref_game_id=bbref_game_id,
)
def get_time_pitch_thrown(self, pfx):
match = PFX_TIMESTAMP_REGEX.match(pfx["park_sv_id"])
if not match:
return None
group_dict = match.groupdict()
game_dict = validate_bbref_game_id(pfx["bbref_game_id"]).value
try:
timestamp = datetime(
game_dict["game_date"].year,
int(group_dict["month"]),
int(group_dict["day"]),
int(group_dict["hour"]),
int(group_dict["minute"]),
int(group_dict["second"]),
)
except ValueError:
return None
return timestamp.replace(
tzinfo=timezone.utc).astimezone(TIME_ZONE_NEW_YORK)
def verify_boxscore_date(boxscore_urls, game_date, url):
rel_url = boxscore_urls[0]
if "allstar" in rel_url:
return Result.Ok()
box_url = urljoin(url, rel_url)
game_id = get_bbref_game_id_from_url(box_url)
if not game_id:
return Result.Fail(f"Failed to parse game id from url: {box_url}")
result = validate_bbref_game_id(game_id)
if result.failure:
return result
game_date_id = result.value["game_date"]
if game_date_id != game_date.date():
scrape_date_str = game_date.strftime(DATE_ONLY_2)
error = (
f"BBref daily dashboard URL for {scrape_date_str} redirected to game results for the "
f"previous day. Please try again when boxscores for {scrape_date_str} are available."
)
return Result.Fail(error)
return Result.Ok()
def enter_game_id(self):
subprocess.run(["clear"])
print_heading("Scraped Data Viewer - Enter a BBRef Game ID",
fg="bright_yellow")
check_game_id = Input("Enter a BBRef Game ID: ").launch()
try:
result = validate_bbref_game_id(check_game_id)
except ValueError:
error = f"\n'{check_game_id}' is NOT a valid BBRef Game ID, please try again."
print_error(error)
pause(message="Press any key to continue...")
return Result.Fail("")
self.game_id = result.value["game_id"]
game_date = result.value["game_date"]
all_valid_game_ids = self.app.audit_report[
game_date.year]["successful"]
if self.game_id not in all_valid_game_ids:
error = f"\nRequirements to show data for {check_game_id} have not been met"
print_error(error)
pause(message="Press any key to continue...")
return Result.Fail("")
view_game_data = ViewGameData(self.app, self.game_id)
view_game_data.launch()
return Result.Fail("")
def get_date_status_from_bbref_game_id(db_session, bbref_game_id):
game_dict = validate_bbref_game_id(bbref_game_id).value
return get_date_status(db_session, game_dict["game_date"])
def get_game_date_from_bbref_game_id(self, bbref_game_id):
game_date = validate_bbref_game_id(bbref_game_id).value["game_date"]
return datetime(game_date.year, game_date.month, game_date.day)
def game_date(self):
game_dict = validate_bbref_game_id(self.bbref_game_id).value
return game_dict["game_date"]
def get_game_id_weights(self, game_ids):
return {
g: validate_bbref_game_id(g).value["game_date"]
for g in game_ids
}
def game_date(self):
result = validate_bbref_game_id(self.bbref_game_id)
return result.value["game_date"] if result.success else None
