def episode_count_for_podcast(podcast, since=None, until={}, **kwargs):
if not podcast:
raise QueryParameterMissing('podcast')
if kwargs.get('descending', False):
since, until = until, since
if isinstance(since, datetime):
since = since.isoformat()
if isinstance(until, datetime):
until = until.isoformat()
db = get_main_database()
res = get_single_result(db, 'episodes/by_podcast',
startkey = [podcast.get_id(), since],
endkey = [podcast.get_id(), until],
reduce = True,
group_level = 1,
**kwargs
)
return res['value'] if res else 0
def get(self, request):
commit, msg = get_git_head()
base_dir = settings.BASE_DIR
hostname = socket.gethostname()
django_version = django.VERSION
main_db = get_main_database()
db_tasks = main_db.server.active_tasks()
i = celery.control.inspect()
scheduled = i.scheduled()
if not scheduled:
num_celery_tasks = None
else:
num_celery_tasks = sum(len(node) for node in scheduled.values())
feed_queue_status = self._get_feed_queue_status()
return self.render_to_response({
'git_commit': commit,
'git_msg': msg,
'base_dir': base_dir,
'hostname': hostname,
'django_version': django_version,
'main_db': main_db.uri,
'db_tasks': db_tasks,
'num_celery_tasks': num_celery_tasks,
'feed_queue_status': feed_queue_status,
})
def episode_count():
db = get_main_database()
r = get_single_result(db, 'episodes/by_podcast',
reduce = True,
stale = 'update_after',
)
return r['value'] if r else 0
def search(q, offset=0, num_results=20):
if not q:
return [], 0
db = get_main_database()
FIELDS = ['title', 'description']
q = lucene_query(FIELDS, q)
try:
res = db.search('podcasts/search',
wrapper = search_wrapper,
include_docs = True,
limit = num_results,
stale = 'update_after',
skip = offset,
q = q,
)
podcasts = list(res)
for podcast in podcasts:
if podcast.needs_update:
incomplete_obj.send_robust(sender=podcast)
return podcasts, res.total_rows
except RequestFailed:
return [], 0
def all_tags():
""" Returns all tags
Some tags might be returned twice """
db = get_main_database()
res = multi_request_view(db, 'podcasts/by_tag',
wrap = False,
reduce = True,
group = True,
group_level = 1
)
for r in res:
yield r['key'][0]
udb = get_userdata_database()
res = multi_request_view(udb, 'usertags/podcasts',
wrap = False,
reduce = True,
group = True,
group_level = 1
)
for r in res:
yield r['key'][0]
def tags_for_podcast(podcast):
""" all tags for the podcast, in decreasing order of importance """
if not podcast:
raise QueryParameterMissing('podcast')
db = get_main_database()
res = db.view('tags/by_podcast',
startkey = [podcast.get_id(), None],
endkey = [podcast.get_id(), {}],
reduce = True,
group = True,
group_level = 2,
stale = 'update_after',
)
tags = Counter(dict((x['key'][1], x['value']) for x in res))
udb = get_userdata_database()
res = udb.view('usertags/by_podcast',
startkey = [podcast.get_id(), None],
endkey = [podcast.get_id(), {}],
reduce = True,
group = True,
group_level = 2,
)
tags.update(Counter(dict( (x['key'][1], x['value']) for x in res)))
get_tag = itemgetter(0)
return map(get_tag, tags.most_common())
def podcasts_groups_by_id(ids):
""" gets podcast groups and top-level podcasts for the given ids """
if ids is None:
raise QueryParameterMissing('ids')
if not ids:
return
db = get_main_database()
res = db.view('podcasts/podcasts_groups',
keys = ids,
include_docs = True,
)
for r in res:
obj = _wrap_pg(r)
if not obj:
yield None
continue
if obj.needs_update:
incomplete_obj.send_robust(sender=obj)
yield obj
def handle(self, *args, **options):
docs = set()
progress(0, len(docs), '', stream=sys.stderr)
for username in options.get('users', []):
user = User.get_user(username)
self.add_user_recursive(user, docs)
if options.get('toplist', False):
toplist = PodcastToplist()
for n, podcast in toplist[:25]:
self.add_podcast_recursive(podcast, docs)
for podcast_url in options.get('podcasts'):
podcast = podcast_for_url(podcast_url, docs)
if not podcast:
logger.warn('podcast not found for URL "%s"', podcast_url)
else:
self.add_podcast_recursive(podcast, docs)
db = get_main_database()
docs = sorted(docs)
self.dump(docs, db)
def get_podcast_licenses():
""" returns the licenses that are assigned to podcasts """
db = get_main_database()
r = db.view('podcasts/license',
reduce = True,
group_level = 1,
)
return Counter({ x['key']: x['value'] for x in r })
def main(request, username, scope):
db = get_main_database()
udb = get_userdata_database()
def user_settings(user):
return user, user, db
def device_settings(user, uid):
device = user.get_device_by_uid(uid)
# get it from the user directly so that changes
# to settings_obj are reflected in user (bug 1344)
settings_obj = user.get_device_by_uid(uid)
return user, settings_obj, db
def podcast_settings(user, url):
podcast = podcast_for_url(url)
if not podcast:
raise Http404
obj = podcast_state_for_user_podcast(user, podcast)
return obj, obj, udb
def episode_settings(user, url, podcast_url):
episode = episode_for_podcast_url(podcast_url, url)
if episode is None:
raise Http404
episode_state = episode_state_for_user_episode(user, episode)
return episode_state, episode_state, udb
models = dict(
account = lambda: user_settings(request.user),
device = lambda: device_settings(request.user, request.GET.get('device', '')),
podcast = lambda: podcast_settings(request.user, request.GET.get('podcast', '')),
episode = lambda: episode_settings(request.user, request.GET.get('episode', ''), request.GET.get('podcast', ''))
)
if scope not in models.keys():
return HttpResponseBadRequest('undefined scope %s' % scope)
try:
base_obj, settings_obj, db = models[scope]()
except DeviceDoesNotExist as e:
return HttpResponseNotFound(str(e))
if request.method == 'GET':
return JsonResponse( settings_obj.settings )
elif request.method == 'POST':
actions = parse_request_body(request)
ret = update_settings(settings_obj, actions)
db.save_doc(base_obj)
return JsonResponse(ret)
def set_episode_listeners(episode, listeners):
if episode.listeners == listeners:
return False
episode.listeners = listeners
db = get_main_database()
db.save_doc(episode)
return True
def filetype_stats():
""" Returns a filetype counter over all episodes """
db = get_main_database()
r = db.view('episode_stats/filetypes',
stale = 'update_after',
reduce = True,
group_level = 1,
)
return Counter({x['key']: x['value'] for x in r})
def get_license_podcast_count(license_url=None):
""" returns the number of podcasts that contain license information """
kwargs = {}
if license_url:
kwargs['key'] = license_url
db = get_main_database()
r = get_single_result(db, 'podcasts/license', **kwargs)
return r['value'] if r else 0
def handle(self, *args, **options):
db = get_main_database()
status = self.get_cmd_status()
since = self.get_since(status, options)
objects = self.get_objects(db, since)
actions = Counter()
# create unfinished command run status
run_status = CommandRunStatus()
run_status.timestamp_started = datetime.utcnow()
run_status.start_seq = since
# add it to existing one (if any)
status.runs.append(run_status)
status.save()
total = db.info()['update_seq']
has_slug = lambda x: bool(x.slug)
for seq, obj in objects:
total = db.info()['update_seq']
if isinstance(obj, PodcastGroup):
podcasts = filter(has_slug, obj.podcasts)
if isinstance(obj, Podcast):
podcasts = filter(has_slug, [obj])
elif isinstance(obj, Episode):
if has_slug(obj):
continue
podcast = podcast_by_id(obj.podcast)
if not podcast:
continue
podcasts = filter(has_slug, [podcast])
updated = self.handle_podcasts(podcasts)
actions['updated'] += updated
if not options['silent']:
status_str = ', '.join('%s: %d' % x for x in actions.items())
progress(seq, total, status_str)
# finish command run status
run_status.timestamp_finished = datetime.utcnow()
run_status.end_seq = total
run_status.status_counter = dict(actions)
# and overwrite existing one (we could keep a longer log here)
status.runs = [run_status]
status.save()
def multi_request_view(cls, view, wrap=True, auto_advance=True,
*args, **kwargs):
"""
splits up a view request into several requests, which reduces
the server load of the number of returned objects is large.
NOTE: As such a split request is obviously not atomical anymore, results
might skip some elements of contain some twice
If auto_advance is False the method will always request the same range.
This can be useful when the view contain unprocessed items and the caller
processes the items, thus removing them from the view before the next
request.
"""
per_page = kwargs.get('limit', 1000)
kwargs['limit'] = per_page + 1
db = get_main_database()
wrapper = kwargs.pop('wrapper', False) or cls.wrap
cont = True
while cont:
resp = db.view(view, *args, **kwargs)
cont = False
for n, obj in enumerate(resp.iterator()):
key = obj['key']
if wrap:
doc = wrapper(obj['doc']) if wrapper else obj['doc']
docid = doc._id if wrapper else obj['id']
else:
docid = obj.get('id', None)
doc = obj
if n == per_page:
if auto_advance:
kwargs['startkey'] = key
if docid is not None:
kwargs['startkey_docid'] = docid
if 'skip' in kwargs:
del kwargs['skip']
# we reached the end of the page, load next one
cont = True
break
yield doc
def podcasts_need_update(limit=100):
db = get_main_database()
res = db.view('episodes/need_update',
group_level = 1,
reduce = True,
limit = limit,
)
# TODO: this method is only used for retrieving podcasts to update;
# should we really send 'incomplete_obj' signals here?
for r in res:
podcast_id = r['key']
podcast = podcast_by_id(podcast_id)
if podcast:
yield podcast
def podcastlist_for_user_slug(user_id, slug):
if not user_id:
raise QueryParameterMissing('user_id')
if not slug:
raise QueryParameterMissing('slug')
db = get_main_database()
l = get_single_result(db, 'podcastlists/by_user_slug',
key = [user_id, slug],
include_docs = True,
schema = PodcastList,
)
return l
def subscriberdata_for_podcast(podcast_id):
if not podcast_id:
raise QueryParameterMissing('podcast_id')
db = get_main_database()
data = get_single_result(db, 'podcasts/subscriber_data',
key = podcast_id,
include_docs = True,
schema = PodcastSubscriberData,
)
if not data:
data = PodcastSubscriberData()
data.podcast = podcast_id
return data
def suggestions_for_user(user):
if not user:
raise QueryParameterMissing('user')
from mygpo.users.models import Suggestions
db = get_main_database()
s = get_single_result(db, 'suggestions/by_user',
key = user._id,
include_docs = True,
schema = Suggestions,
)
if not s:
s = Suggestions()
s.user = user._id
return s
def podcastgroup_for_oldid(oldid):
if not oldid:
raise QueryParameterMissing('oldid')
db = get_main_database()
pg = get_single_result(db, 'podcasts/groups_by_oldid',
key = long(oldid),
include_docs = True,
schema = PodcastGroup,
)
if not pg:
return None
if pg.needs_update:
incomplete_obj.send_robust(sender=pg)
return pg
def podcast_for_oldid(oldid):
if oldid is None:
raise QueryParameterMissing('oldid')
db = get_main_database()
podcast = get_single_result(db, 'podcasts/by_oldid',
key = long(oldid),
include_docs = True,
wrapper = _wrap_podcast_group_key1,
)
if not podcast:
return None
if podcast.needs_update:
incomplete_obj.send_robust(sender=podcast)
return podcast
def episode_for_podcast_id_url(podcast_id, episode_url, create=False):
if not podcast_id:
raise QueryParameterMissing('podcast_id')
if not episode_url:
raise QueryParameterMissing('episode_url')
key = u'episode-podcastid-%s-url-%s' % (
sha1(podcast_id.encode('utf-8')).hexdigest(),
sha1(episode_url.encode('utf-8')).hexdigest())
# Disabled as cache invalidation is not working properly
# episode = cache.get(key)
# if episode:
# return episode
db = get_main_database()
episode = get_single_result(db, 'episodes/by_podcast_url',
key = [podcast_id, episode_url],
include_docs = True,
reduce = False,
schema = Episode,
)
if episode:
if episode.needs_update:
incomplete_obj.send_robust(sender=episode)
else:
cache.set(key, episode)
return episode
if create:
episode = Episode()
episode.created_timestamp = get_timestamp(datetime.utcnow())
episode.podcast = podcast_id
episode.urls = [episode_url]
episode.save()
incomplete_obj.send_robust(sender=episode)
return episode
return None
def podcast_by_id_uncached(podcast_id, current_id=False):
if not podcast_id:
raise QueryParameterMissing('podcast_id')
db = get_main_database()
podcast = get_single_result(db, 'podcasts/by_id',
key = podcast_id,
include_docs = True,
wrapper = _wrap_podcast_group,
)
if not podcast:
return None
if podcast.needs_update:
incomplete_obj.send_robust(sender=podcast)
return podcast
def podcast_for_slug(slug):
if not slug:
raise QueryParameterMissing('slug')
db = get_main_database()
obj = get_single_result(db, 'podcasts/by_slug',
startkey = [slug, None],
endkey = [slug, {}],
include_docs = True,
wrapper = _wrap_podcast_group_key1,
)
if not obj:
return None
if obj.needs_update:
incomplete_obj.send_robust(sender=obj)
return obj
def episode_for_oldid(oldid):
if not oldid:
raise QueryParameterMissing('oldid')
oldid = int(oldid)
db = get_main_database()
episode = get_single_result(db, 'episodes/by_oldid',
key = oldid,
limit = 1,
include_docs = True,
schema = Episode,
)
if not episode:
return None
if episode.needs_update:
incomplete_obj.send_robust(sender=episode)
return episode
def episode_for_slug(podcast_id, episode_slug):
if not podcast_id:
raise QueryParameterMissing('podcast_id')
if not episode_slug:
raise QueryParameterMissing('episode_slug')
db = get_main_database()
episode = get_single_result(db, 'episodes/by_slug',
key = [podcast_id, episode_slug],
include_docs = True,
schema = Episode,
)
if not episode:
return None
if episode.needs_update:
incomplete_obj.send_robust(sender=episode)
return episode
def missing_slug_count(doc_type, start, end):
from mygpo.db import get_single_result
if not doc_type:
raise QueryParameterMissing('doc_type')
if not start:
raise QueryParameterMissing('start')
if not end:
raise QueryParameterMissing('end')
db = get_main_database()
res = get_single_result(db, 'slugs/missing',
startkey = [doc_type] + end,
endkey = [doc_type] + start,
descending = True,
reduce = True,
group = True,
group_level = 1,
)
return res['value'] if res else 0
def missing_slugs(doc_type, start, end, wrapper, **kwargs):
if not doc_type:
raise QueryParameterMissing('doc_type')
if not start:
raise QueryParameterMissing('start')
if not end:
raise QueryParameterMissing('end')
db = get_main_database()
return multi_request_view(db, 'slugs/missing',
startkey = [doc_type] + end,
endkey = [doc_type] + start,
descending = True,
include_docs = True,
reduce = False,
wrapper = wrapper,
auto_advance = False,
**kwargs
)
def episode_by_id(episode_id, current_id=False):
if not episode_id:
raise QueryParameterMissing('episode_id')
db = get_main_database()
episode = get_single_result(db, 'episodes/by_id',
key = episode_id,
include_docs = True,
schema = Episode,
)
if not episode:
return None
if current_id and episode._id != episode_id:
raise MergedIdException(episode, episode._id)
if episode.needs_update:
incomplete_obj.send_robust(sender=episode)
return episode
def podcast_for_url(url, create=False):
if not url:
raise QueryParameterMissing('url')
key = 'podcast-by-url-%s' % sha1(url.encode('utf-8')).hexdigest()
podcast = cache.get(key)
if podcast:
return podcast
db = get_main_database()
podcast_group = get_single_result(db, 'podcasts/by_url',
key = url,
include_docs = True,
wrapper = _wrap_pg,
)
if podcast_group:
podcast = podcast_group.get_podcast_by_url(url)
if podcast.needs_update:
incomplete_obj.send_robust(sender=podcast)
else:
cache.set(key, podcast)
return podcast
if create:
podcast = Podcast()
podcast.created_timestamp = get_timestamp(datetime.utcnow())
podcast.urls = [url]
podcast.save()
incomplete_obj.send_robust(sender=podcast)
return podcast
return None