def main(): config, backup_paths, key_paths = parse_config() try: check_should_backup(config) except BackupPrerequisiteFailed as e: print('Not backing up, for reason:') print(e.args[0]) sys.exit(1) notifier = Notifier() notifier.notify('Starting backup') for bp in backup_paths.values(): if bp.automount: mount_path_if_necessary(bp.mount_path) try: for name, snapshot in search_snapshots(bp.path).items(): if snapshot.newest == snapshot.base: message = "Most recent snapshot for '{}' ({}) already on remote system".format( name, snapshot.newest, ) print(message) else: message = ( "Need to backup subvolume {} (base snapshot: {}, most recent: {})" ).format( name, snapshot.base, snapshot.newest, ) print(message) backup_snapshot(snapshot, config['server']['host'], key_paths) finally: if bp.automount: umount_path(bp.mount_path) notifier.notify('Backup complete')
class Archeloos: def __init__(self): self.config = config.Config() self.notifier = Notifier(**self.config.get("notify", None)) self.watch = self.config.get("torrents", "watch_dir") self.torrents = [] self.shows = self.config.get("torrents", "shows").split(",") self.quality = self.config.get("torrents", "quality") def check_torrents(self, torrents): l = [] for t in torrents: st = "%s" % t for s in self.shows: if s.lower() in st.lower(): if self.quality in t.quality and t.download(self.watch): print(t) l.append(t) #print(t.link) #print(len(torrents)) return l def run(self): l = [] tracker = self.config.get("tracker", None) while True: new_torrents = [] feed = list_torrents(tracker["rss_url"], tracker["user_id"], tracker["user_pass"]) lt = parse_results(feed) new_torrents = [t for t in lt if t not in self.torrents] self.torrents = list(set(lt) | set(self.torrents)) l = self.check_torrents(new_torrents) self.torrents = self.torrents + new_torrents self.notifier.notify(l) time.sleep(1800)
class Test_Notifier(unittest.TestCase): """ Unittest for the class Notifier. """ def setUp(self): self.notifier = Notifier(['simple_key', 'key_with_message']) self.mock_listener = Mock() self.first_key_heared = self.mock_listener.first_key_heared self.first_key_heared_again = self.mock_listener.first_key_heared_again self.second_key_heared = self.mock_listener.second_key_heared self.notifier.connect('simple_key', self.first_key_heared) self.notifier.connect('simple_key', self.first_key_heared_again) self.notifier.connect('key_with_message', self.second_key_heared) def test_notifier(self): self.mock_listener.expects(once()).first_key_heared() self.mock_listener.expects(once()).first_key_heared_again() self.mock_listener.expects(never()).second_key_heared() self.notifier.notify('simple_key') self.mock_listener.verify() def test_notifier_with_message(self): self.mock_listener.expects(never()).first_key_heared() self.mock_listener.expects(never()).first_key_heared_again() self.mock_listener.expects(once()).method('second_key_heared').\ with_at_least(same('little message')) self.notifier.notify('key_with_message', 'little message') self.mock_listener.verify() def test_disconnect(self): self.mock_listener.expects(once()).first_key_heared() self.mock_listener.expects(never()).first_key_heared_again() self.notifier.disconnect('simple_key', self.first_key_heared_again) self.notifier.notify('simple_key') self.mock_listener.verify() def test_reconnect(self): self.mock_listener.expects(once()).first_key_heared() self.mock_listener.expects(once()).first_key_heared_again() self.notifier.disconnect('simple_key', self.first_key_heared_again) self.notifier.connect('simple_key', self.first_key_heared_again) self.notifier.notify('simple_key') self.mock_listener.verify()
class Archeloos: def __init__(self): self.config = config.Config() self.notifier = Notifier(**self.config.get("notify", None)) self.watch = self.config.get("torrents", "watch_dir") self.torrents = [] self.shows = self.config.get("torrents", "shows").split(",") self.fast_list = self.config.get("torrents", "fast_list").split(",") self.resolution = self.config.get("torrents", "resolution") self.quality = self.config.get("torrents", "quality") self.extra = self.config.get("torrents", "extra") tracker = self.config.get("tracker", None) self.downloader = Downloader(tracker["rss_url"], tracker["user_id"], tracker["user_pass"]) print("watch: '%s'" % self.watch) print("shows: '%s'" % self.shows) print("fast_list: '%s'" % self.fast_list) print("resolution: '%s'" % self.resolution) print("quality: '%s'" % self.quality) print("extra: '%s'" % self.extra) def pick_show(self, torrent): s = ("%s" % torrent.name).lower() shows = [name.lower() for name in self.shows] fast_list = [name.lower() for name in self.fast_list] print("new pick ", torrent) if not any(ss in s for ss in shows): return False if self.resolution.lower() not in torrent.resolution.lower(): print(" - %s WRONG QUALITY '%s' not in '%s'" % (torrent, self.resolution, torrent.resolution)) return False if s in fast_list: print(" - %s FAST LIST" % torrent) return True if self.quality.lower() in torrent.quality.lower(): print(" - %s OK" % torrent) return True print(" - %s NOT SELECTED ('%s' and '%s') != ('%s' and '%s')" % (torrent, self.resolution, self.quality, torrent.resolution, torrent.quality)) return False def check_torrents(self, torrents): l = [] for t in torrents: if self.pick_show(t) and self.downloader.download(t, self.watch): l.append(t) #print(len(torrents)) return l def run(self): l = [] while True: new_torrents = [] feed = self.downloader.list_torrents() lt = parse_results(feed) new_torrents = [t for t in lt if t not in self.torrents] self.torrents = list(set(lt) | set(self.torrents)) l = self.check_torrents(new_torrents) self.torrents = self.torrents + new_torrents self.notifier.notify(l) time.sleep(900)
class Downloader: """A downloader which checks for new videos on the Digital Foundry homepage, and downloads them.""" __scheme = 'https://' __domain = 'www.digitalfoundry.net' __url = __scheme + __domain __cache_file = 'cache' __download_strings = { 'hevc': ' Download HEVC', 'now': 'Download now', } def __init__(self, browser: str, sid: str, token: str, to: str, from_: str, output_dir: str): self.__browser = browser self.__load_cookie_jar() self.__output_dir = output_dir self.__notifier = Notifier(sid, token, to, from_) self.__lock = Lock() def __load_cookie_jar(self): if self.__browser == 'chrome': self.__cj = chrome() elif self.__browser == 'safari': self.__cj = safari() elif self.__browser == 'firefox': self.__cj = firefox() else: raise ValueError def load_cookie_jar(self): self.__load_cookie_jar() def download(self) -> None: """Checks the Digital Foundry homepage for new videos, and downloads them.""" if not self.__has_valid_cookie(): return self.__lock.acquire() logging.info('Checking Digital Foundry Homepage...') r = get(self.__url, cookies=self.__cj) if not r.ok: msg = 'Can\'t reach Digital Foundry Homepage.' logging.warning(msg) self.__notifier.notify(msg) self.__lock.release() return hrefs = self.__get_links(r) total_downloads = len(hrefs) if total_downloads > 0: logging.info( f"Found {total_downloads} new video{'s' if total_downloads > 1 else ''}!" ) for i in range(0, total_downloads): self.__process_downloads(hrefs[i], i + 1, total_downloads) logging.info("All videos downloaded.") self.__lock.release() return def __has_valid_cookie(self) -> bool: """Checks if there is a valid digital foundry cookie in the cookie jar""" df_cookie = None for cookie in self.__cj: if cookie.domain == self.__domain: df_cookie = cookie break if df_cookie is None: msg = 'No Digital Foundry cookie found. Please log in to Digital Foundry in your browser.' logging.warning(msg) self.__notifier.notify(msg) return False elif df_cookie.is_expired(time()): msg = 'Digital Foundry cookie expired. Please log in to Digital Foundry in your browser.' logging.warning(msg) self.__notifier.notify(msg) return False return True def __get_links(self, r: Response) -> List[Dict[str, str]]: """Gets all the download links from a given response. If link is in cache, it won't be added to list.""" soup = bs4.BeautifulSoup(r.content, 'html.parser') if not _logged_in(soup): msg = 'Subscribe button found. Make sure you are logged in to Digital Foundry in your browser.' logging.warning(msg) self.__notifier.notify(msg) return [] all_videos = soup.find_all('div', {'class', 'video'}) hrefs = [] total_downloads_available = 0 cache = None whole_file = "" try: cache = open(self.__cache_file, "r") except Exception as ex: logging.error( f"Problem opening cache file from {self.__cache_file}: {ex}") finally: if cache is not None: whole_file = cache.read() for video in all_videos: art_tag = video.find('a', {'class', 'cover'}) art = _get_art_link(art_tag) total_downloads_available += 1 if (cache is not None and art_tag['href'] not in whole_file) or cache is None: hrefs.append({'art': art, 'href': art_tag['href']}) if cache is not None: cache.close() return hrefs def __process_downloads(self, href: Dict[str, str], current: int, total: int) -> None: """Follows HEVC link on a page with two file types""" r = get(self.__url + href['href'], cookies=self.__cj) soup = bs4.BeautifulSoup(r.content, 'html.parser') dl_buttons = soup.find_all('a', class_='button wide download', limit=2) hevc_button = None for button in dl_buttons: if button.get_text() == self.__download_strings['hevc']: hevc_button = button break if hevc_button is None: return self.__process_hevc_download(hevc_button['href'], href, current, total) def __process_hevc_download(self, href: str, original_link: Dict[str, str], current: int, total: int) -> None: """Follows Download Now link on HEVC download page""" r = get(self.__url + href, cookies=self.__cj) soup = bs4.BeautifulSoup(r.content, 'html.parser') download_button = soup.find('a', text=self.__download_strings['now']) self.__download_video(soup.title.get_text(), download_button['href'], original_link, current, total) def __download_video(self, title: str, href: str, original_link: Dict[str, str], current: int, total: int) -> None: """Downloads a file at the given href""" # Get actual video r = get(self.__url + href, cookies=self.__cj, stream=True) total_length = r.headers.get('content-length') title = _convert_title(title) if r.status_code == 404: logging.error(f"{self.__url}{href} returned 404") self.__notifier.notify(f"{title} returned 404") return logging.info('Downloading...') print(f'{current}/{total} {title}') try: with open(self.__output_dir + '/' + title + '.mp4', 'wb') as f: if original_link['art'] != "": self.__download_art(original_link['art'], title) if total_length is None: # no content length header f.write(r.content) self.__notifier.notify(f'New video downloaded: {title}') else: _download_with_progress(r, f, int(total_length)) self.__notifier.notify(f'New video downloaded: {title}') except Exception as ex: logging.error(f"Failed to download {title}: {ex}") else: try: with open(self.__cache_file, 'a') as f: f.write(original_link['href'] + '\n') except Exception as ex: logging.error( f"Could not open cache file at {self.__cache_file}: {ex}") print() def __download_art(self, href: str, title: str): """Downloads a jpg at the given href""" art = get(href, cookies=self.__cj) with open(self.__output_dir + '/' + title + '.jpg', 'wb') as f: f.write(art.content)