def sync(comic_id): dao, crawler = None, None if comic_id == 'xkcd': dao = WebcomicDao() crawler = XkcdComic(app.config.XKCD_CRAWLER_URL) else: raise Exception('Invalid webcomic id %s' % comic_id) urls = get_comic_urls(comic_id) only_urls = set([url.get('url') for url in urls]) comic_links = crawler.get_comics(only_urls) last_sync = time.time() links_count = len(comic_links) + len(urls) dao.add_links(comic_id, comic_links) dao.update_comic(comic_id, links_count=links_count, last_sync=last_sync) return { 'id': comic_id, 'links_count': links_count, 'last_sync': readable.from_ts(last_sync) }
def get_all_playlists_meta_info(self): """ Returns list of { 'name': '', 'updated_at': 'time', 'links_count': 'count' } """ playlists = [] ret = self.db.music.find({ 'is_active': True }, { '_id': 1, 'name': 1, 'updated_at': 1, 'links_count': 1 }) for playlist in ret: id = playlist.get('_id') name = playlist.get('name') links_count = playlist.get('links_count') updated_at = readable.from_ts(playlist.get('updated_at')) \ if playlist.get('updated_at') is not None else 'Never' playlists.append({ 'id': id, 'name': name, 'updated_at': updated_at, 'links_count': links_count }) return playlists
def get_all_comics_meta_info(self): """ Returns list of { 'name': '', 'last_sync': 'time', 'links_count': 'count' } """ comics = [] ret = self.db.webcomics.find({ 'is_active': True }, { '_id': 1, 'name': 1, 'last_sync': 1, 'links_count': 1 }) l = [] for comic in ret: id = comic.get('_id') name = comic.get('name') links_count = comic.get('links_count') last_sync = readable.from_ts(comic.get('last_sync')) \ if comic.get('last_sync') is not None else 'Never' comics.append({ 'id': id, 'name': name, 'last_sync': last_sync, 'links_count': links_count }) return comics
def get_all_astros_meta_info(self): """ Returns list of { 'name': '', 'last_sync': 'time', 'links_count': 'count' } """ astros = [] ret = self.db.astros.find({ 'is_active': True }, { '_id': 1, 'name': 1, 'last_sync': 1, 'links_count': 1 }) for astro in ret: id = astro.get('_id') name = astro.get('name') links_count = astro.get('links_count') last_sync = readable.from_ts(astro.get('last_sync')) \ if astro.get('last_sync') is not None else 'Never' astros.append({ 'id': id, 'name': name, 'last_sync': last_sync, 'links_count': links_count }) return astros
def add_links_to_playlist(playlist_id, link, site): dao = MusicDao() if site == 'youtube': videos = youtube_service.get_videos(link) else: raise Exception('Invalid site id %s' % site) error, existing_video_links = get_links_in_playlist(playlist_id) existing_video_urls = [link.get('url') for link in existing_video_links] links = [] for video in videos: if not video.get('url') in existing_video_urls: links.append(video) last_sync = time.time() links_count = len(existing_video_urls) + len(links) dao.add_links_to_playlist(playlist_id, links) dao.update_playlist(playlist_id, links_count=links_count, updated_at=last_sync) return None, { 'id': playlist_id, 'links_count': links_count, 'updated_at': readable.from_ts(last_sync) }
def sync(astro_id): dao, crawler = None, None if astro_id == "quarks": dao = AstroDao() crawler = QuarksCrawler(app.config.QUARKS_CRAWLER_URL) else: raise Exception("Invalid astro id %s" % astro_id) urls = get_astro_urls(astro_id) astro_links = crawler.get_astros(urls) last_sync = time.time() links_count = len(astro_links) + len(urls) dao.add_links(astro_id, astro_links) dao.update_astro(astro_id, links_count=links_count, last_sync=last_sync) return {"id": astro_id, "links_count": links_count, "last_sync": readable.from_ts(last_sync)}