Пример #1
0
def episode_count_for_podcast(podcast, since=None, until={}, **kwargs):

    if not podcast:
        raise QueryParameterMissing('podcast')


    if kwargs.get('descending', False):
        since, until = until, since

    if isinstance(since, datetime):
        since = since.isoformat()

    if isinstance(until, datetime):
        until = until.isoformat()

    db = get_main_database()
    res = get_single_result(db, 'episodes/by_podcast',
            startkey     = [podcast.get_id(), since],
            endkey       = [podcast.get_id(), until],
            reduce       = True,
            group_level  = 1,
            **kwargs
        )

    return res['value'] if res else 0
Пример #2
0
    def get(self, request):
        commit, msg = get_git_head()
        base_dir = settings.BASE_DIR
        hostname = socket.gethostname()
        django_version = django.VERSION

        main_db = get_main_database()

        db_tasks = main_db.server.active_tasks()

        i = celery.control.inspect()
        scheduled = i.scheduled()
        if not scheduled:
            num_celery_tasks = None
        else:
            num_celery_tasks = sum(len(node) for node in scheduled.values())

        feed_queue_status = self._get_feed_queue_status()

        return self.render_to_response({
            'git_commit': commit,
            'git_msg': msg,
            'base_dir': base_dir,
            'hostname': hostname,
            'django_version': django_version,
            'main_db': main_db.uri,
            'db_tasks': db_tasks,
            'num_celery_tasks': num_celery_tasks,
            'feed_queue_status': feed_queue_status,
        })
Пример #3
0
def episode_count():
    db = get_main_database()
    r = get_single_result(db, 'episodes/by_podcast',
            reduce = True,
            stale  = 'update_after',
        )
    return r['value'] if r else 0
Пример #4
0
def search(q, offset=0, num_results=20):

    if not q:
        return [], 0

    db = get_main_database()

    FIELDS = ['title', 'description']
    q = lucene_query(FIELDS, q)

    try:
        res = db.search('podcasts/search',
                wrapper      = search_wrapper,
                include_docs = True,
                limit        = num_results,
                stale        = 'update_after',
                skip         = offset,
                q            = q,
            )

        podcasts = list(res)

        for podcast in podcasts:
            if podcast.needs_update:
                incomplete_obj.send_robust(sender=podcast)

        return podcasts, res.total_rows

    except RequestFailed:
        return [], 0
Пример #5
0
def all_tags():
    """ Returns all tags

    Some tags might be returned twice """
    db = get_main_database()
    res = multi_request_view(db, 'podcasts/by_tag',
            wrap        = False,
            reduce      = True,
            group       = True,
            group_level = 1
        )

    for r in res:
        yield r['key'][0]

    udb = get_userdata_database()
    res = multi_request_view(udb, 'usertags/podcasts',
            wrap        = False,
            reduce      = True,
            group       = True,
            group_level = 1
        )

    for r in res:
        yield r['key'][0]
Пример #6
0
def tags_for_podcast(podcast):
    """ all tags for the podcast, in decreasing order of importance """

    if not podcast:
        raise QueryParameterMissing('podcast')


    db = get_main_database()
    res = db.view('tags/by_podcast',
            startkey    = [podcast.get_id(), None],
            endkey      = [podcast.get_id(), {}],
            reduce      = True,
            group       = True,
            group_level = 2,
            stale       = 'update_after',
        )

    tags = Counter(dict((x['key'][1], x['value']) for x in res))

    udb = get_userdata_database()
    res = udb.view('usertags/by_podcast',
            startkey    = [podcast.get_id(), None],
            endkey      = [podcast.get_id(), {}],
            reduce      = True,
            group       = True,
            group_level = 2,
        )

    tags.update(Counter(dict( (x['key'][1], x['value']) for x in res)))

    get_tag = itemgetter(0)
    return map(get_tag, tags.most_common())
Пример #7
0
def podcasts_groups_by_id(ids):
    """ gets podcast groups and top-level podcasts for the given ids """

    if ids is None:
        raise QueryParameterMissing('ids')

    if not ids:
        return

    db = get_main_database()
    res = db.view('podcasts/podcasts_groups',
            keys         = ids,
            include_docs = True,
        )

    for r in res:
        obj = _wrap_pg(r)

        if not obj:
            yield None
            continue

        if obj.needs_update:
            incomplete_obj.send_robust(sender=obj)

        yield obj
Пример #8
0
    def handle(self, *args, **options):

        docs = set()
        progress(0, len(docs), '', stream=sys.stderr)

        for username in options.get('users', []):
            user = User.get_user(username)
            self.add_user_recursive(user, docs)

        if options.get('toplist', False):
            toplist = PodcastToplist()
            for n, podcast in toplist[:25]:
                self.add_podcast_recursive(podcast, docs)

        for podcast_url in options.get('podcasts'):
            podcast = podcast_for_url(podcast_url, docs)
            if not podcast:
                logger.warn('podcast not found for URL "%s"', podcast_url)

            else:
                self.add_podcast_recursive(podcast, docs)

        db = get_main_database()
        docs = sorted(docs)
        self.dump(docs, db)
Пример #9
0
def get_podcast_licenses():
    """ returns the licenses that are assigned to podcasts """
    db = get_main_database()
    r = db.view('podcasts/license',
            reduce = True,
            group_level = 1,
    )

    return Counter({ x['key']: x['value'] for x in r })
Пример #10
0
def main(request, username, scope):

    db = get_main_database()
    udb = get_userdata_database()

    def user_settings(user):
        return user, user, db

    def device_settings(user, uid):
        device = user.get_device_by_uid(uid)

        # get it from the user directly so that changes
        # to settings_obj are reflected in user (bug 1344)
        settings_obj = user.get_device_by_uid(uid)

        return user, settings_obj, db

    def podcast_settings(user, url):
        podcast = podcast_for_url(url)
        if not podcast:
            raise Http404
        obj = podcast_state_for_user_podcast(user, podcast)
        return obj, obj, udb

    def episode_settings(user, url, podcast_url):
        episode = episode_for_podcast_url(podcast_url, url)
        if episode is None:
            raise Http404

        episode_state = episode_state_for_user_episode(user, episode)
        return episode_state, episode_state, udb

    models = dict(
            account = lambda: user_settings(request.user),
            device  = lambda: device_settings(request.user, request.GET.get('device', '')),
            podcast = lambda: podcast_settings(request.user, request.GET.get('podcast', '')),
            episode = lambda: episode_settings(request.user, request.GET.get('episode', ''), request.GET.get('podcast', ''))
        )


    if scope not in models.keys():
        return HttpResponseBadRequest('undefined scope %s' % scope)

    try:
        base_obj, settings_obj, db = models[scope]()
    except DeviceDoesNotExist as e:
        return HttpResponseNotFound(str(e))

    if request.method == 'GET':
        return JsonResponse( settings_obj.settings )

    elif request.method == 'POST':
        actions = parse_request_body(request)
        ret = update_settings(settings_obj, actions)
        db.save_doc(base_obj)
        return JsonResponse(ret)
Пример #11
0
def set_episode_listeners(episode, listeners):

    if episode.listeners == listeners:
        return False

    episode.listeners = listeners

    db = get_main_database()
    db.save_doc(episode)
    return True
Пример #12
0
def filetype_stats():
    """ Returns a filetype counter over all episodes """

    db = get_main_database()
    r = db.view('episode_stats/filetypes',
        stale       = 'update_after',
        reduce      = True,
        group_level = 1,
    )

    return Counter({x['key']: x['value'] for x in r})
Пример #13
0
def get_license_podcast_count(license_url=None):
    """ returns the number of podcasts that contain license information """

    kwargs = {}
    if license_url:
        kwargs['key'] = license_url

    db = get_main_database()
    r = get_single_result(db, 'podcasts/license', **kwargs)

    return r['value'] if r else 0
Пример #14
0
    def handle(self, *args, **options):

        db = get_main_database()
        status = self.get_cmd_status()
        since = self.get_since(status, options)
        objects = self.get_objects(db, since)
        actions = Counter()


        # create unfinished command run status
        run_status = CommandRunStatus()
        run_status.timestamp_started = datetime.utcnow()
        run_status.start_seq = since
        # add it to existing one (if any)
        status.runs.append(run_status)
        status.save()

        total = db.info()['update_seq']

        has_slug = lambda x: bool(x.slug)

        for seq, obj in objects:
            total = db.info()['update_seq']

            if isinstance(obj, PodcastGroup):
                podcasts = filter(has_slug, obj.podcasts)

            if isinstance(obj, Podcast):
                podcasts = filter(has_slug, [obj])

            elif isinstance(obj, Episode):
                if has_slug(obj):
                    continue

                podcast = podcast_by_id(obj.podcast)
                if not podcast:
                    continue
                podcasts = filter(has_slug, [podcast])

            updated = self.handle_podcasts(podcasts)
            actions['updated'] += updated

            if not options['silent']:
                status_str = ', '.join('%s: %d' % x for x in actions.items())
                progress(seq, total, status_str)


        # finish command run status
        run_status.timestamp_finished = datetime.utcnow()
        run_status.end_seq = total
        run_status.status_counter = dict(actions)
        # and overwrite existing one (we could keep a longer log here)
        status.runs = [run_status]
        status.save()
Пример #15
0
def multi_request_view(cls, view, wrap=True, auto_advance=True,
        *args, **kwargs):
    """
    splits up a view request into several requests, which reduces
    the server load of the number of returned objects is large.

    NOTE: As such a split request is obviously not atomical anymore, results
    might skip some elements of contain some twice

    If auto_advance is False the method will always request the same range.
    This can be useful when the view contain unprocessed items and the caller
    processes the items, thus removing them from the view before the next
    request.
    """

    per_page = kwargs.get('limit', 1000)
    kwargs['limit'] = per_page + 1
    db = get_main_database()
    wrapper = kwargs.pop('wrapper', False) or cls.wrap
    cont = True

    while cont:

        resp = db.view(view, *args, **kwargs)
        cont = False

        for n, obj in enumerate(resp.iterator()):

            key = obj['key']

            if wrap:
                doc = wrapper(obj['doc']) if wrapper else obj['doc']
                docid = doc._id if wrapper else obj['id']
            else:
                docid = obj.get('id', None)
                doc = obj

            if n == per_page:
                if auto_advance:
                    kwargs['startkey'] = key
                    if docid is not None:
                        kwargs['startkey_docid'] = docid
                    if 'skip' in kwargs:
                        del kwargs['skip']

                # we reached the end of the page, load next one
                cont = True
                break

            yield doc
Пример #16
0
def podcasts_need_update(limit=100):
    db = get_main_database()
    res = db.view('episodes/need_update',
            group_level = 1,
            reduce      = True,
            limit       = limit,
        )

    # TODO: this method is only used for retrieving podcasts to update;
    #       should we really send 'incomplete_obj' signals here?

    for r in res:
        podcast_id = r['key']
        podcast = podcast_by_id(podcast_id)
        if podcast:
            yield podcast
Пример #17
0
def podcastlist_for_user_slug(user_id, slug):

    if not user_id:
        raise QueryParameterMissing('user_id')

    if not slug:
        raise QueryParameterMissing('slug')

    db = get_main_database()
    l = get_single_result(db, 'podcastlists/by_user_slug',
            key          = [user_id, slug],
            include_docs = True,
            schema       = PodcastList,
        )

    return l
Пример #18
0
def subscriberdata_for_podcast(podcast_id):

    if not podcast_id:
        raise QueryParameterMissing('podcast_id')

    db = get_main_database()
    data = get_single_result(db, 'podcasts/subscriber_data',
            key          = podcast_id,
            include_docs = True,
            schema       = PodcastSubscriberData,
        )

    if not data:
        data = PodcastSubscriberData()
        data.podcast = podcast_id

    return data
Пример #19
0
def suggestions_for_user(user):

    if not user:
        raise QueryParameterMissing('user')

    from mygpo.users.models import Suggestions
    db = get_main_database()
    s = get_single_result(db, 'suggestions/by_user',
                key          = user._id,
                include_docs = True,
                schema       = Suggestions,
            )

    if not s:
        s = Suggestions()
        s.user = user._id

    return s
Пример #20
0
def podcastgroup_for_oldid(oldid):

    if not oldid:
        raise QueryParameterMissing('oldid')

    db = get_main_database()
    pg = get_single_result(db, 'podcasts/groups_by_oldid',
            key          = long(oldid),
            include_docs = True,
            schema       = PodcastGroup,
        )

    if not pg:
        return None

    if pg.needs_update:
        incomplete_obj.send_robust(sender=pg)

    return pg
Пример #21
0
def podcast_for_oldid(oldid):

    if oldid is None:
        raise QueryParameterMissing('oldid')

    db = get_main_database()
    podcast = get_single_result(db, 'podcasts/by_oldid',
            key          = long(oldid),
            include_docs = True,
            wrapper      = _wrap_podcast_group_key1,
        )

    if not podcast:
        return None

    if podcast.needs_update:
        incomplete_obj.send_robust(sender=podcast)

    return podcast
Пример #22
0
def episode_for_podcast_id_url(podcast_id, episode_url, create=False):

    if not podcast_id:
        raise QueryParameterMissing('podcast_id')

    if not episode_url:
        raise QueryParameterMissing('episode_url')


    key = u'episode-podcastid-%s-url-%s' % (
            sha1(podcast_id.encode('utf-8')).hexdigest(),
            sha1(episode_url.encode('utf-8')).hexdigest())

#   Disabled as cache invalidation is not working properly
#   episode = cache.get(key)
#   if episode:
#       return episode

    db = get_main_database()
    episode = get_single_result(db, 'episodes/by_podcast_url',
            key          = [podcast_id, episode_url],
            include_docs = True,
            reduce       = False,
            schema       = Episode,
        )

    if episode:
        if episode.needs_update:
            incomplete_obj.send_robust(sender=episode)
        else:
            cache.set(key, episode)
        return episode

    if create:
        episode = Episode()
        episode.created_timestamp = get_timestamp(datetime.utcnow())
        episode.podcast = podcast_id
        episode.urls = [episode_url]
        episode.save()
        incomplete_obj.send_robust(sender=episode)
        return episode

    return None
Пример #23
0
def podcast_by_id_uncached(podcast_id, current_id=False):

    if not podcast_id:
        raise QueryParameterMissing('podcast_id')

    db = get_main_database()
    podcast = get_single_result(db, 'podcasts/by_id',
            key          = podcast_id,
            include_docs = True,
            wrapper      = _wrap_podcast_group,
        )

    if not podcast:
        return None

    if podcast.needs_update:
        incomplete_obj.send_robust(sender=podcast)

    return podcast
Пример #24
0
def podcast_for_slug(slug):

    if not slug:
        raise QueryParameterMissing('slug')

    db = get_main_database()
    obj = get_single_result(db, 'podcasts/by_slug',
            startkey     = [slug, None],
            endkey       = [slug, {}],
            include_docs = True,
            wrapper      = _wrap_podcast_group_key1,
        )

    if not obj:
        return None

    if obj.needs_update:
        incomplete_obj.send_robust(sender=obj)

    return obj
Пример #25
0
def episode_for_oldid(oldid):

    if not oldid:
        raise QueryParameterMissing('oldid')

    oldid = int(oldid)
    db = get_main_database()
    episode = get_single_result(db, 'episodes/by_oldid',
            key          = oldid,
            limit        = 1,
            include_docs = True,
            schema       = Episode,
        )

    if not episode:
        return None

    if episode.needs_update:
        incomplete_obj.send_robust(sender=episode)

    return episode
Пример #26
0
def episode_for_slug(podcast_id, episode_slug):

    if not podcast_id:
        raise QueryParameterMissing('podcast_id')

    if not episode_slug:
        raise QueryParameterMissing('episode_slug')

    db = get_main_database()
    episode = get_single_result(db, 'episodes/by_slug',
            key          = [podcast_id, episode_slug],
            include_docs = True,
            schema       = Episode,
        )

    if not episode:
        return None

    if episode.needs_update:
        incomplete_obj.send_robust(sender=episode)

    return episode
Пример #27
0
def missing_slug_count(doc_type, start, end):
    from mygpo.db import get_single_result

    if not doc_type:
        raise QueryParameterMissing('doc_type')

    if not start:
        raise QueryParameterMissing('start')

    if not end:
        raise QueryParameterMissing('end')


    db = get_main_database()
    res = get_single_result(db, 'slugs/missing',
            startkey     = [doc_type] + end,
            endkey       = [doc_type] + start,
            descending   = True,
            reduce       = True,
            group        = True,
            group_level  = 1,
        )
    return res['value'] if res else 0
Пример #28
0
def missing_slugs(doc_type, start, end, wrapper, **kwargs):

    if not doc_type:
        raise QueryParameterMissing('doc_type')

    if not start:
        raise QueryParameterMissing('start')

    if not end:
        raise QueryParameterMissing('end')


    db = get_main_database()
    return multi_request_view(db, 'slugs/missing',
            startkey     = [doc_type] + end,
            endkey       = [doc_type] + start,
            descending   = True,
            include_docs = True,
            reduce       = False,
            wrapper      = wrapper,
            auto_advance = False,
            **kwargs
        )
Пример #29
0
def episode_by_id(episode_id, current_id=False):

    if not episode_id:
        raise QueryParameterMissing('episode_id')

    db = get_main_database()

    episode = get_single_result(db, 'episodes/by_id',
            key          = episode_id,
            include_docs = True,
            schema       = Episode,
        )

    if not episode:
        return None

    if current_id and episode._id != episode_id:
        raise MergedIdException(episode, episode._id)

    if episode.needs_update:
        incomplete_obj.send_robust(sender=episode)

    return episode
Пример #30
0
def podcast_for_url(url, create=False):

    if not url:
        raise QueryParameterMissing('url')

    key = 'podcast-by-url-%s' % sha1(url.encode('utf-8')).hexdigest()

    podcast = cache.get(key)
    if podcast:
        return podcast

    db = get_main_database()
    podcast_group = get_single_result(db, 'podcasts/by_url',
            key          = url,
            include_docs = True,
            wrapper      = _wrap_pg,
        )

    if podcast_group:
        podcast = podcast_group.get_podcast_by_url(url)

        if podcast.needs_update:
            incomplete_obj.send_robust(sender=podcast)
        else:
            cache.set(key, podcast)

        return podcast

    if create:
        podcast = Podcast()
        podcast.created_timestamp = get_timestamp(datetime.utcnow())
        podcast.urls = [url]
        podcast.save()
        incomplete_obj.send_robust(sender=podcast)
        return podcast

    return None