def sync(comic_id):
dao, crawler = None, None
if comic_id == 'xkcd':
dao = WebcomicDao()
crawler = XkcdComic(app.config.XKCD_CRAWLER_URL)
else:
raise Exception('Invalid webcomic id %s' % comic_id)
urls = get_comic_urls(comic_id)
only_urls = set([url.get('url') for url in urls])
comic_links = crawler.get_comics(only_urls)
last_sync = time.time()
links_count = len(comic_links) + len(urls)
dao.add_links(comic_id, comic_links)
dao.update_comic(comic_id, links_count=links_count, last_sync=last_sync)
return {
'id': comic_id,
'links_count': links_count,
'last_sync': readable.from_ts(last_sync)
}
def get_all_playlists_meta_info(self):
"""
Returns list of {
'name': '',
'updated_at': 'time',
'links_count': 'count'
}
"""
playlists = []
ret = self.db.music.find({
'is_active': True
}, { '_id': 1, 'name': 1, 'updated_at': 1, 'links_count': 1 })
for playlist in ret:
id = playlist.get('_id')
name = playlist.get('name')
links_count = playlist.get('links_count')
updated_at = readable.from_ts(playlist.get('updated_at')) \
if playlist.get('updated_at') is not None else 'Never'
playlists.append({
'id': id,
'name': name,
'updated_at': updated_at,
'links_count': links_count
})
return playlists
def get_all_comics_meta_info(self):
"""
Returns list of {
'name': '',
'last_sync': 'time',
'links_count': 'count'
}
"""
comics = []
ret = self.db.webcomics.find({
'is_active': True
}, { '_id': 1, 'name': 1, 'last_sync': 1, 'links_count': 1 })
l = []
for comic in ret:
id = comic.get('_id')
name = comic.get('name')
links_count = comic.get('links_count')
last_sync = readable.from_ts(comic.get('last_sync')) \
if comic.get('last_sync') is not None else 'Never'
comics.append({
'id': id,
'name': name,
'last_sync': last_sync,
'links_count': links_count
})
return comics
def get_all_astros_meta_info(self):
"""
Returns list of {
'name': '',
'last_sync': 'time',
'links_count': 'count'
}
"""
astros = []
ret = self.db.astros.find({
'is_active': True
}, { '_id': 1, 'name': 1, 'last_sync': 1, 'links_count': 1 })
for astro in ret:
id = astro.get('_id')
name = astro.get('name')
links_count = astro.get('links_count')
last_sync = readable.from_ts(astro.get('last_sync')) \
if astro.get('last_sync') is not None else 'Never'
astros.append({
'id': id,
'name': name,
'last_sync': last_sync,
'links_count': links_count
})
return astros
def add_links_to_playlist(playlist_id, link, site):
dao = MusicDao()
if site == 'youtube':
videos = youtube_service.get_videos(link)
else:
raise Exception('Invalid site id %s' % site)
error, existing_video_links = get_links_in_playlist(playlist_id)
existing_video_urls = [link.get('url') for link in existing_video_links]
links = []
for video in videos:
if not video.get('url') in existing_video_urls:
links.append(video)
last_sync = time.time()
links_count = len(existing_video_urls) + len(links)
dao.add_links_to_playlist(playlist_id, links)
dao.update_playlist(playlist_id, links_count=links_count, updated_at=last_sync)
return None, {
'id': playlist_id,
'links_count': links_count,
'updated_at': readable.from_ts(last_sync)
}
def sync(astro_id):
dao, crawler = None, None
if astro_id == "quarks":
dao = AstroDao()
crawler = QuarksCrawler(app.config.QUARKS_CRAWLER_URL)
else:
raise Exception("Invalid astro id %s" % astro_id)
urls = get_astro_urls(astro_id)
astro_links = crawler.get_astros(urls)
last_sync = time.time()
links_count = len(astro_links) + len(urls)
dao.add_links(astro_id, astro_links)
dao.update_astro(astro_id, links_count=links_count, last_sync=last_sync)
return {"id": astro_id, "links_count": links_count, "last_sync": readable.from_ts(last_sync)}
