def get_general(self) -> General:
general = General()
self.parser.to_object(
general,
"General",
"series_dir",
"int:threads",
"float:speed_up_default",
"int:max_days_back",
"log_level",
)
if not general.series_dir:
TealPrint.warning(f"Missing 'series_dir' in [General] in your configuration. Please add it.", exit=True)
# Convert string to LogLevel
if isinstance(general.log_level, str):
try:
general.log_level = TealLevel[general.log_level]
except KeyError:
TealPrint.warning(
f"Failed to set log_level from config, invalid level: {general.log_level}. Setting log_level to info"
)
general.log_level = TealLevel.info
return general
def __init__(self):
TealPrint.debug(f"Sqlite DB location: {SqliteGateway.__FILE_PATH}")
self.__connection = sqlite3.connect(SqliteGateway.__FILE_PATH)
self.__cursor = self.__connection.cursor()
# Create DB (if not exists)
self._create_db()
def remove_old() -> None:
"""Remove all old backups"""
TealPrint.info("Removing old backups", color=attr("bold"))
backup_path = Path(config.general.backup_location)
for backup in backup_path.glob("*"):
if backup.is_file() and date_helper.is_backup_old(backup):
message = f"🔥 {backup}"
TealPrint.info(message, indent=1)
remove(backup)
def main():
check_for_programs()
config.set_cli_args(get_args())
config_gateway.check_config_exists()
init_logs()
if config.daemon:
TealPrint.verbose(f"Starting {config.app_name} as a daemon")
_daemon()
else:
TealPrint.verbose(f"Running {config.app_name} once")
_run_once()
def _run_once():
# (Re)load config
config_gateway.read()
config.general = config_gateway.get_general()
channels = config_gateway.get_channels()
app_repo = AppAdapter()
try:
download_new_episodes = DownloadNewEpisodes(app_repo)
download_new_episodes.execute(channels)
finally:
TealPrint.clear_indent()
app_repo.close()
def _is_old(self, video: Video) -> bool:
TealPrint.verbose(f"🚦 Is the video old?", push_indent=True)
old_date = datetime.now().astimezone() - timedelta(
days=config.general.max_days_back)
video_date = datetime.strptime(video.date, "%Y-%m-%dT%H:%M:%S%z")
if video_date >= old_date:
TealPrint.verbose(f"🟢 Video is new", color=LogColors.passed)
TealPrint.pop_indent()
return False
else:
TealPrint.verbose(f"🔴 Video is old", color=LogColors.filtered)
TealPrint.pop_indent()
return True
def check_config_exists(self) -> None:
if not self.path.exists():
TealPrint.info(f"Could not find any configuration file in {self.path}")
user_input = input("Do you want to copy the example config and edit it (y/n)?")
if user_input.lower() == "y":
self.parser.copy_example_if_conf_not_exists(config.app_name)
editor = ""
if "EDITOR" in os.environ:
editor = os.environ["EDITOR"]
if editor == "" and platform.system() == "Windows":
editor = "notepad.exe"
elif editor == "":
editor = "vim"
run([editor, self.path])
else:
exit(0)
def _find_diff_files(self, path: Path, indent: int):
# File/Dir has changed
try:
TealPrint.debug(f"{path}", indent=indent)
if path.is_symlink() or (not path.is_dir()
and self.is_modified_within_diff(path)):
self.tar.add(path)
# Check children
else:
if not path.is_symlink():
for child in path.glob("*"):
self._find_diff_files(child, indent + 1)
for child in path.glob(".*"):
self._find_diff_files(child, indent + 1)
except FileNotFoundError:
# Skip if we didn't find a file
pass
def add_downloaded(self, channel_name: str, video_id: str):
"""Adds a downloaded episode to the DB
Args:
channel_name (str): Channel name (not channel_id)
video_id (str): YouTube's video id for the video that was downloaded
"""
episode_number = self.get_next_episode_number(channel_name)
TealPrint.debug(
f"💾 Save to DB {video_id} from {channel_name} with episode number {episode_number}.",
color=LogColors.added,
)
if not config.pretend:
sql = "INSERT INTO video (id, episode_number, channel_name) VALUES(?, ?, ?)"
self.__cursor.execute(sql,
(video_id, episode_number, channel_name))
self.__connection.commit()
def render(video: Video, in_file: Path, out_file: Path,
speed: float) -> bool:
completed_process = True
tmp_out = Path(gettempdir(), f"{video.id}_render_out.mp4")
if not config.pretend:
# Create parent directories
out_file.parent.mkdir(parents=True, exist_ok=True)
audio_speed = speed
video_speed = 1.0 / audio_speed
completed_process = (subprocess.run(
[
"ffmpeg",
"-y",
"-i",
in_file,
"-metadata",
f'title="{video.title}"',
"-threads",
str(config.general.threads),
"-filter_complex",
f"[0:v]setpts=({video_speed})*PTS[v];[0:a]atempo={audio_speed}[a]",
"-map",
"[v]",
"-map",
"[a]",
tmp_out,
],
stdout=FfmpegGateway._get_verbose_out(),
).returncode == 0)
if completed_process:
# Copy the temprory file to series/Minecraft
if not config.pretend:
copyfile(tmp_out, out_file)
TealPrint.debug("🗑 Deleting temporary files")
in_file.unlink(missing_ok=True)
tmp_out.unlink(missing_ok=True)
return completed_process
def get_channels(self) -> List[Channel]:
channels: List[Channel] = []
for section in self.parser.sections():
if ConfigGateway.is_channel_section(section):
channel = Channel()
channel.name = section
self.parser.to_object(
channel,
section,
"id",
"name",
"dir->collection_dir",
"float:speed",
"str_list:includes",
"str_list:excludes",
)
if not channel.id:
TealPrint.warning(
f"Missing 'id' for channel [{section}] in your configuration. Please add it.", exit=True
)
channels.append(channel)
return channels
def run(self) -> None:
# Only run if a MySQL username and password has been supplied
if not config.mysql.username and not config.mysql.password:
TealPrint.info(
"Skipping MySQL backup, no username and password supplied",
color=fg("yellow"),
)
return
TealPrint.info("Backing up MySQL", color=attr("bold"))
with tarfile.open(self.filepath, "w:gz") as tar:
# Multiple database
if len(config.mysql.databases) > 0:
for database in config.mysql.databases:
dump = self._get_database(database)
self._add_to_tar_file(tar, database, dump)
else:
dump = self._get_database()
self._add_to_tar_file(tar, "all-databases", dump)
TealPrint.info("✔ MySQL backup complete!")
def run(self) -> None:
"""Add files to tar"""
TealPrint.info(f"Backing up {self.name}", color=attr("bold"))
# Full backup
if self.part == BackupParts.full:
TealPrint.info(f"Doing a full backup", indent=1)
for path_glob in self.paths:
TealPrint.verbose(f"{path_glob}", indent=2)
for path in glob(path_glob):
TealPrint.debug(f"{path}", indent=3)
self.tar.add(path)
# Diff backup
else:
TealPrint.info("Doing a diff backup", indent=1)
for path_glob in self.paths:
TealPrint.verbose(f"{path_glob}", indent=2)
for path in glob(path_glob):
self._find_diff_files(Path(path), 3)
def close(self):
TealPrint.debug("Closing Sqlite DB connection")
self.__connection.commit()
self.__connection.close()
def execute(self, channels: List[Channel]) -> None:
for channel in channels:
TealPrint.info(channel.name,
color=LogColors.header,
push_indent=True)
videos = self.repo.get_latest_videos(channel)
if len(videos) == 0:
TealPrint.info(
f"🦘 Skipping {channel.name}, no new matching videos to download",
color=LogColors.skipped)
for video in videos:
TealPrint.verbose(f"🎞 {video.title}",
color=LogColors.header,
push_indent=True)
# Skip downloaded videos
if self.repo.has_downloaded(video):
TealPrint.verbose(
f"🟠Skipping {video.title}, already downloaded",
color=LogColors.skipped)
TealPrint.pop_indent()
continue
# Filter out
if self._filter_video(channel, video):
TealPrint.verbose(f"🔴 Video was filtered out",
color=LogColors.filtered)
TealPrint.pop_indent()
continue
TealPrint.verbose(f"🟢 Video passed all filters",
color=LogColors.passed)
TealPrint.verbose(f"🔽 Downloading...")
download_path = self.repo.download(video)
if download_path is None:
TealPrint.warning(f"âš Couldn't download {video.title}")
TealPrint.pop_indent()
continue
TealPrint.verbose(
f"🎞 Starting rendering, this may take a while...")
out_path = self._get_out_filepath(channel, video)
rendered = self.repo.render(video, download_path, out_path,
channel.speed)
if not rendered:
TealPrint.warning(f"âš Couldn't render {video.title}")
TealPrint.pop_indent()
continue
self.repo.set_as_downloaded(channel, video)
TealPrint.info(
f"✔ Video {video.title} downloaded successfully ➡ {out_path}"
)
TealPrint.pop_indent()
TealPrint.pop_indent()
def _matches_any_exclude(self, channel: Channel, video: Video) -> bool:
title = video.title.lower()
TealPrint.verbose(f"🚦 Check exclude filter", push_indent=True)
if len(channel.excludes) == 0:
TealPrint.verbose(f"🟢 Pass: no exclude filter",
color=LogColors.passed)
TealPrint.pop_indent()
return False
for filter in channel.excludes:
filter = filter.lower()
if re.search(filter, title):
TealPrint.verbose(f"🔴 Matched filter: {filter}",
color=LogColors.filtered)
TealPrint.pop_indent()
return True
else:
TealPrint.verbose(f"🟡 Didn't match filter: {filter}",
color=LogColors.no_match)
TealPrint.verbose(f"🟢 Didn't match any exclude filter",
color=LogColors.passed)
TealPrint.pop_indent()
return False
def _print_missing(self, section: str, varname: str) -> None:
TealPrint.warning(
f"Missing {varname} under section {section}. " + f"Please add it to your configuration file {self.path}",
exit=True,
)
def _print_section_not_found(section: str) -> None:
TealPrint.warning(f"⚠ [{section}] section not found!", indent=1)
def get_args(self) -> ConfigFileArgs:
args = ConfigFileArgs()
if not self.path.exists():
TealPrint.warning(f"Could not find config file {self.path}. Please add!", exit=True)
return args
config = ConfigParser()
config.read(self.path)
TealPrint.verbose(f"Reading configuration {self.path}", color=attr("bold"))
try:
config.to_object(
args.general,
"General",
"backup_location",
"int:days_to_keep",
)
except SectionNotFoundError:
ConfigFileParser._print_section_not_found("General")
try:
config.to_object(
args.backups,
"Backups",
"daily_alias",
"weekly_alias",
"monthly_alias",
"str_list:daily",
"str_list:weekly",
"str_list:monthly",
)
except SectionNotFoundError:
ConfigFileParser._print_section_not_found("Backups")
try:
config.to_object(
args.mysql,
"MySQL",
"username",
"password",
"address",
"int:port",
"str_list:databases",
)
except SectionNotFoundError:
ConfigFileParser._print_section_not_found("MySQL")
try:
config.to_object(
args.email,
"Email",
"to->to_address",
"from->from_address",
"int:disk_percentage",
)
except SectionNotFoundError:
ConfigFileParser._print_section_not_found("Email")
self._check_required(args)
return args
