def main():
config, backup_paths, key_paths = parse_config()
try:
check_should_backup(config)
except BackupPrerequisiteFailed as e:
print('Not backing up, for reason:')
print(e.args[0])
sys.exit(1)
notifier = Notifier()
notifier.notify('Starting backup')
for bp in backup_paths.values():
if bp.automount:
mount_path_if_necessary(bp.mount_path)
try:
for name, snapshot in search_snapshots(bp.path).items():
if snapshot.newest == snapshot.base:
message = "Most recent snapshot for '{}' ({}) already on remote system".format(
name,
snapshot.newest,
)
print(message)
else:
message = (
"Need to backup subvolume {} (base snapshot: {}, most recent: {})"
).format(
name,
snapshot.base,
snapshot.newest,
)
print(message)
backup_snapshot(snapshot, config['server']['host'],
key_paths)
finally:
if bp.automount:
umount_path(bp.mount_path)
notifier.notify('Backup complete')
def main():
config, backup_paths, key_paths = parse_config()
try:
check_should_backup(config)
except BackupPrerequisiteFailed as e:
print('Not backing up, for reason:')
print(e.args[0])
sys.exit(1)
notifier = Notifier()
notifier.notify('Starting backup')
for bp in backup_paths.values():
if bp.automount:
mount_path_if_necessary(bp.mount_path)
try:
for name, snapshot in search_snapshots(bp.path).items():
if snapshot.newest == snapshot.base:
message = "Most recent snapshot for '{}' ({}) already on remote system".format(
name,
snapshot.newest,
)
print(message)
else:
message = (
"Need to backup subvolume {} (base snapshot: {}, most recent: {})"
).format(
name,
snapshot.base,
snapshot.newest,
)
print(message)
backup_snapshot(snapshot, config['server']['host'], key_paths)
finally:
if bp.automount:
umount_path(bp.mount_path)
notifier.notify('Backup complete')
class Archeloos:
def __init__(self):
self.config = config.Config()
self.notifier = Notifier(**self.config.get("notify", None))
self.watch = self.config.get("torrents", "watch_dir")
self.torrents = []
self.shows = self.config.get("torrents", "shows").split(",")
self.quality = self.config.get("torrents", "quality")
def check_torrents(self, torrents):
l = []
for t in torrents:
st = "%s" % t
for s in self.shows:
if s.lower() in st.lower():
if self.quality in t.quality and t.download(self.watch):
print(t)
l.append(t)
#print(t.link)
#print(len(torrents))
return l
def run(self):
l = []
tracker = self.config.get("tracker", None)
while True:
new_torrents = []
feed = list_torrents(tracker["rss_url"], tracker["user_id"], tracker["user_pass"])
lt = parse_results(feed)
new_torrents = [t for t in lt if t not in self.torrents]
self.torrents = list(set(lt) | set(self.torrents))
l = self.check_torrents(new_torrents)
self.torrents = self.torrents + new_torrents
self.notifier.notify(l)
time.sleep(1800)
class Test_Notifier(unittest.TestCase):
"""
Unittest for the class Notifier.
"""
def setUp(self):
self.notifier = Notifier(['simple_key', 'key_with_message'])
self.mock_listener = Mock()
self.first_key_heared = self.mock_listener.first_key_heared
self.first_key_heared_again = self.mock_listener.first_key_heared_again
self.second_key_heared = self.mock_listener.second_key_heared
self.notifier.connect('simple_key', self.first_key_heared)
self.notifier.connect('simple_key', self.first_key_heared_again)
self.notifier.connect('key_with_message', self.second_key_heared)
def test_notifier(self):
self.mock_listener.expects(once()).first_key_heared()
self.mock_listener.expects(once()).first_key_heared_again()
self.mock_listener.expects(never()).second_key_heared()
self.notifier.notify('simple_key')
self.mock_listener.verify()
def test_notifier_with_message(self):
self.mock_listener.expects(never()).first_key_heared()
self.mock_listener.expects(never()).first_key_heared_again()
self.mock_listener.expects(once()).method('second_key_heared').\
with_at_least(same('little message'))
self.notifier.notify('key_with_message', 'little message')
self.mock_listener.verify()
def test_disconnect(self):
self.mock_listener.expects(once()).first_key_heared()
self.mock_listener.expects(never()).first_key_heared_again()
self.notifier.disconnect('simple_key', self.first_key_heared_again)
self.notifier.notify('simple_key')
self.mock_listener.verify()
def test_reconnect(self):
self.mock_listener.expects(once()).first_key_heared()
self.mock_listener.expects(once()).first_key_heared_again()
self.notifier.disconnect('simple_key', self.first_key_heared_again)
self.notifier.connect('simple_key', self.first_key_heared_again)
self.notifier.notify('simple_key')
self.mock_listener.verify()
class Archeloos:
def __init__(self):
self.config = config.Config()
self.notifier = Notifier(**self.config.get("notify", None))
self.watch = self.config.get("torrents", "watch_dir")
self.torrents = []
self.shows = self.config.get("torrents", "shows").split(",")
self.fast_list = self.config.get("torrents", "fast_list").split(",")
self.resolution = self.config.get("torrents", "resolution")
self.quality = self.config.get("torrents", "quality")
self.extra = self.config.get("torrents", "extra")
tracker = self.config.get("tracker", None)
self.downloader = Downloader(tracker["rss_url"], tracker["user_id"], tracker["user_pass"])
print("watch: '%s'" % self.watch)
print("shows: '%s'" % self.shows)
print("fast_list: '%s'" % self.fast_list)
print("resolution: '%s'" % self.resolution)
print("quality: '%s'" % self.quality)
print("extra: '%s'" % self.extra)
def pick_show(self, torrent):
s = ("%s" % torrent.name).lower()
shows = [name.lower() for name in self.shows]
fast_list = [name.lower() for name in self.fast_list]
print("new pick ", torrent)
if not any(ss in s for ss in shows):
return False
if self.resolution.lower() not in torrent.resolution.lower():
print(" - %s WRONG QUALITY '%s' not in '%s'" % (torrent, self.resolution, torrent.resolution))
return False
if s in fast_list:
print(" - %s FAST LIST" % torrent)
return True
if self.quality.lower() in torrent.quality.lower():
print(" - %s OK" % torrent)
return True
print(" - %s NOT SELECTED ('%s' and '%s') != ('%s' and '%s')" % (torrent, self.resolution, self.quality, torrent.resolution, torrent.quality))
return False
def check_torrents(self, torrents):
l = []
for t in torrents:
if self.pick_show(t) and self.downloader.download(t, self.watch):
l.append(t)
#print(len(torrents))
return l
def run(self):
l = []
while True:
new_torrents = []
feed = self.downloader.list_torrents()
lt = parse_results(feed)
new_torrents = [t for t in lt if t not in self.torrents]
self.torrents = list(set(lt) | set(self.torrents))
l = self.check_torrents(new_torrents)
self.torrents = self.torrents + new_torrents
self.notifier.notify(l)
time.sleep(900)
class Downloader:
"""A downloader which checks for new videos on the Digital Foundry homepage, and downloads them."""
__scheme = 'https://'
__domain = 'www.digitalfoundry.net'
__url = __scheme + __domain
__cache_file = 'cache'
__download_strings = {
'hevc': ' Download HEVC',
'now': 'Download now',
}
def __init__(self, browser: str, sid: str, token: str, to: str, from_: str,
output_dir: str):
self.__browser = browser
self.__load_cookie_jar()
self.__output_dir = output_dir
self.__notifier = Notifier(sid, token, to, from_)
self.__lock = Lock()
def __load_cookie_jar(self):
if self.__browser == 'chrome':
self.__cj = chrome()
elif self.__browser == 'safari':
self.__cj = safari()
elif self.__browser == 'firefox':
self.__cj = firefox()
else:
raise ValueError
def load_cookie_jar(self):
self.__load_cookie_jar()
def download(self) -> None:
"""Checks the Digital Foundry homepage for new videos, and downloads them."""
if not self.__has_valid_cookie():
return
self.__lock.acquire()
logging.info('Checking Digital Foundry Homepage...')
r = get(self.__url, cookies=self.__cj)
if not r.ok:
msg = 'Can\'t reach Digital Foundry Homepage.'
logging.warning(msg)
self.__notifier.notify(msg)
self.__lock.release()
return
hrefs = self.__get_links(r)
total_downloads = len(hrefs)
if total_downloads > 0:
logging.info(
f"Found {total_downloads} new video{'s' if total_downloads > 1 else ''}!"
)
for i in range(0, total_downloads):
self.__process_downloads(hrefs[i], i + 1, total_downloads)
logging.info("All videos downloaded.")
self.__lock.release()
return
def __has_valid_cookie(self) -> bool:
"""Checks if there is a valid digital foundry cookie in the cookie jar"""
df_cookie = None
for cookie in self.__cj:
if cookie.domain == self.__domain:
df_cookie = cookie
break
if df_cookie is None:
msg = 'No Digital Foundry cookie found. Please log in to Digital Foundry in your browser.'
logging.warning(msg)
self.__notifier.notify(msg)
return False
elif df_cookie.is_expired(time()):
msg = 'Digital Foundry cookie expired. Please log in to Digital Foundry in your browser.'
logging.warning(msg)
self.__notifier.notify(msg)
return False
return True
def __get_links(self, r: Response) -> List[Dict[str, str]]:
"""Gets all the download links from a given response. If link is in cache, it won't be added to list."""
soup = bs4.BeautifulSoup(r.content, 'html.parser')
if not _logged_in(soup):
msg = 'Subscribe button found. Make sure you are logged in to Digital Foundry in your browser.'
logging.warning(msg)
self.__notifier.notify(msg)
return []
all_videos = soup.find_all('div', {'class', 'video'})
hrefs = []
total_downloads_available = 0
cache = None
whole_file = ""
try:
cache = open(self.__cache_file, "r")
except Exception as ex:
logging.error(
f"Problem opening cache file from {self.__cache_file}: {ex}")
finally:
if cache is not None:
whole_file = cache.read()
for video in all_videos:
art_tag = video.find('a', {'class', 'cover'})
art = _get_art_link(art_tag)
total_downloads_available += 1
if (cache is not None and art_tag['href']
not in whole_file) or cache is None:
hrefs.append({'art': art, 'href': art_tag['href']})
if cache is not None:
cache.close()
return hrefs
def __process_downloads(self, href: Dict[str, str], current: int,
total: int) -> None:
"""Follows HEVC link on a page with two file types"""
r = get(self.__url + href['href'], cookies=self.__cj)
soup = bs4.BeautifulSoup(r.content, 'html.parser')
dl_buttons = soup.find_all('a', class_='button wide download', limit=2)
hevc_button = None
for button in dl_buttons:
if button.get_text() == self.__download_strings['hevc']:
hevc_button = button
break
if hevc_button is None:
return
self.__process_hevc_download(hevc_button['href'], href, current, total)
def __process_hevc_download(self, href: str, original_link: Dict[str, str],
current: int, total: int) -> None:
"""Follows Download Now link on HEVC download page"""
r = get(self.__url + href, cookies=self.__cj)
soup = bs4.BeautifulSoup(r.content, 'html.parser')
download_button = soup.find('a', text=self.__download_strings['now'])
self.__download_video(soup.title.get_text(), download_button['href'],
original_link, current, total)
def __download_video(self, title: str, href: str, original_link: Dict[str,
str],
current: int, total: int) -> None:
"""Downloads a file at the given href"""
# Get actual video
r = get(self.__url + href, cookies=self.__cj, stream=True)
total_length = r.headers.get('content-length')
title = _convert_title(title)
if r.status_code == 404:
logging.error(f"{self.__url}{href} returned 404")
self.__notifier.notify(f"{title} returned 404")
return
logging.info('Downloading...')
print(f'{current}/{total} {title}')
try:
with open(self.__output_dir + '/' + title + '.mp4', 'wb') as f:
if original_link['art'] != "":
self.__download_art(original_link['art'], title)
if total_length is None: # no content length header
f.write(r.content)
self.__notifier.notify(f'New video downloaded: {title}')
else:
_download_with_progress(r, f, int(total_length))
self.__notifier.notify(f'New video downloaded: {title}')
except Exception as ex:
logging.error(f"Failed to download {title}: {ex}")
else:
try:
with open(self.__cache_file, 'a') as f:
f.write(original_link['href'] + '\n')
except Exception as ex:
logging.error(
f"Could not open cache file at {self.__cache_file}: {ex}")
print()
def __download_art(self, href: str, title: str):
"""Downloads a jpg at the given href"""
art = get(href, cookies=self.__cj)
with open(self.__output_dir + '/' + title + '.jpg', 'wb') as f:
f.write(art.content)
