コード例 #1
0
ファイル: search.py プロジェクト: Mic92/mygpo
def search_podcasts(q, limit=20, skip=0):

    if is_url(q):
        url = sanitize_url(q)

        podcast = Podcast.for_url(url, create=True)

        if not podcast.title:
            update_podcasts([podcast])

        podcast = Podcast.for_url(url)

        return [podcast], 1


    db = get_main_database()

    #FIXME current couchdbkit can't parse responses for multi-query searches
    q = q.replace(',', '')

    res = db.search('podcasts/search', wrapper=search_wrapper,
        include_docs=True, limit=limit, skip=skip, q=q,
        sort='\\subscribers<int>')

    #FIXME: return empty results in case of search backend error
    try:
        return list(res), res.total_rows
    except:
        return [], 0
コード例 #2
0
ファイル: __init__.py プロジェクト: Mic92/mygpo
def update_episodes(user, actions, now, ua_string):
    update_urls = []

    grouped_actions = defaultdict(list)

    # group all actions by their episode
    for action in actions:

        podcast_url = action['podcast']
        podcast_url = sanitize_append(podcast_url, 'podcast', update_urls)
        if podcast_url == '': continue

        episode_url = action['episode']
        episode_url = sanitize_append(episode_url, 'episode', update_urls)
        if episode_url == '': continue

        act = parse_episode_action(action, user, update_urls, now, ua_string)
        grouped_actions[ (podcast_url, episode_url) ].append(act)

    # Prepare the updates for each episode state
    obj_funs = []

    for (p_url, e_url), action_list in grouped_actions.iteritems():
        episode_state = EpisodeUserState.for_ref_urls(user, p_url, e_url)

        fun = partial(update_episode_actions, action_list=action_list)
        obj_funs.append( (episode_state, fun) )

    db = get_main_database()
    bulk_save_retry(db, obj_funs)

    return update_urls
コード例 #3
0
ファイル: heatmap.py プロジェクト: Mic92/mygpo
    def _query(self):
        """ Queries the database and stores the heatmap and its borders """

        db = get_main_database()

        group_level = len(filter(None, [self.podcast_id,
                    self.episode_id, self.user_id]))

        r = db.view('heatmap/by_episode',
                startkey    = [self.podcast_id, self.episode_id,
                                self.user_id],
                endkey      = [self.podcast_id, self.episode_id or {},
                                self.user_id or {}],
                reduce      = True,
                group       = True,
                group_level = group_level,
                stale       = 'update_after',
            )

        if not r:
            self.heatmap = []
            self.borders = []
        else:
            res = r.first()['value']
            self.heatmap = res['heatmap']
            self.borders = res['borders']

            # heatmap info doesn't reach until the end of the episode
            # so we extend it with 0 listeners
            if self.duration > self.borders[-1]:
                self.heatmap.append(0)
                self.borders.append(self.duration)
コード例 #4
0
ファイル: backend.py プロジェクト: Mic92/mygpo
    def execute(self):
        """ Executes all added actions in bulk """
        db = get_main_database()
        obj_funs = map(self._get_obj_fun, self.actions)
        bulk_save_retry(db, obj_funs)

        # prepare for another run
        self.actions = []
コード例 #5
0
ファイル: backend.py プロジェクト: Mic92/mygpo
def get_podcast_count_for_language():
    """ Returns a the number of podcasts for each language """

    counts = defaultdict(int)

    db = get_main_database()
    r = db.view("podcasts/by_language", reduce=True, group_level=1, stale="update_after")

    counts.update(dict((x["key"][0], x["value"]) for x in r))
    return counts
コード例 #6
0
ファイル: assign-episode-slugs.py プロジェクト: Mic92/mygpo
    def handle(self, *args, **options):

        db = get_main_database()
        status = self.get_cmd_status()
        since = self.get_since(status, options)
        objects = self.get_objects(db, since)
        actions = Counter()


        # create unfinished command run status
        run_status = CommandRunStatus()
        run_status.timestamp_started = datetime.utcnow()
        run_status.start_seq = since
        # add it to existing one (if any)
        status.runs.append(run_status)
        status.save()

        total = db.info()['update_seq']

        has_slug = lambda x: bool(x.slug)

        for seq, obj in objects:
            total = db.info()['update_seq']

            if isinstance(obj, PodcastGroup):
                podcasts = filter(has_slug, obj.podcasts)

            if isinstance(obj, Podcast):
                podcasts = filter(has_slug, [obj])

            elif isinstance(obj, Episode):
                if has_slug(obj):
                    continue

                podcast = Podcast.get(obj.podcast)
                if not podcast:
                    continue
                podcasts = filter(has_slug, [podcast])

            updated = self.handle_podcasts(podcasts)
            actions['updated'] += updated

            if not options['silent']:
                status_str = ', '.join('%s: %d' % x for x in actions.items())
                progress(seq, total, status_str)


        # finish command run status
        run_status.timestamp_finished = datetime.utcnow()
        run_status.end_seq = total
        run_status.status_counter = dict(actions)
        # and overwrite existing one (we could keep a longer log here)
        status.runs = [run_status]
        status.save()
コード例 #7
0
ファイル: podcastcmd.py プロジェクト: Mic92/mygpo
    def get_podcast_with_new_episodes(self):
        db = get_main_database()
        res = db.view('episodes/need_update',
                group_level = 1,
                reduce      = True,
            )

        for r in res:
            podcast_id = r['key']
            podcast = Podcast.get(podcast_id)
            if podcast:
                yield podcast
コード例 #8
0
ファイル: models.py プロジェクト: Mic92/mygpo
    def __init__(self, user, device):
        self.user = user
        self.device = device
        self._db = get_main_database()

        if device:
            self._view = 'history/by_device'
            self._startkey = [self.user._id, device.id, None]
            self._endkey   = [self.user._id, device.id, {}]
        else:
            self._view = 'history/by_user'
            self._startkey = [self.user._id, None]
            self._endkey   = [self.user._id, {}]
コード例 #9
0
ファイル: merge-episode-states.py プロジェクト: Mic92/mygpo
    def handle(self, *args, **options):

        skip = options.get('skip')
        total = EpisodeUserState.view('episode_states/by_user_episode',
                limit=0,
            ).total_rows
        db = get_main_database()

        actions = Counter()
        actions['merged'] = 0


        for n in count(skip):

            first = EpisodeUserState.view('episode_states/by_user_episode',
                    skip         = n,
                    include_docs = True,
                    limit        = 1,
                )
            first = list(first)
            if not first:
                break

            first = first[0]


            states = EpisodeUserState.view('episode_states/by_user_episode',
                    key          = [first.user, first.episode],
                    include_docs = True,
                )
            states = list(states)

            l1 = len(states)
            # we don't want to delete this one
            states.remove(first)

            assert len(states) == l1-1

            if states:
                updater = get_updater(states)

                obj_funs = [(first, updater)] + [(state, do_delete) for state in states]

                bulk_save_retry(db, obj_funs)

                merged = len(states)-1
                actions['merged'] += merged
                total -= merged

            status_str = ', '.join('%s: %d' % x for x in actions.items())
            progress(n+1, total, status_str)
コード例 #10
0
ファイル: models.py プロジェクト: Mic92/mygpo
    def get_episode_states(self, user_id):
        """ Returns the latest episode actions for the podcast's episodes """

        from mygpo.users.models import EpisodeUserState
        db = get_main_database()

        res = db.view('episode_states/by_user_podcast',
                startkey = [user_id, self.get_id(), None],
                endkey   = [user_id, self.get_id(), {}],
            )

        for r in res:
            action = r['value']
            yield action
コード例 #11
0
ファイル: tags.py プロジェクト: Mic92/mygpo
    def _query(self):
        db = get_main_database()
        res = db.view('categories/by_weight',
                descending = True,
                skip       = self.skip,
                limit      = self.count,
                stale      = 'update_after',
            )

        mk_entry = lambda r: TagCloudEntry(r['value'], r['key'])
        self._entries = map(mk_entry, res)

        if self.sort_by_name:
            self._entries.sort(key = lambda x: x.label.lower())
コード例 #12
0
ファイル: utils.py プロジェクト: Mic92/mygpo
def multi_request_view(cls, view, wrap=True, auto_advance=True,
        *args, **kwargs):
    """
    splits up a view request into several requests, which reduces
    the server load of the number of returned objects is large.

    NOTE: As such a split request is obviously not atomical anymore, results
    might skip some elements of contain some twice

    If auto_advance is False the method will always request the same range.
    This can be useful when the view contain unprocessed items and the caller
    processes the items, thus removing them from the view before the next
    request.
    """

    per_page = kwargs.get('limit', 1000)
    kwargs['limit'] = per_page + 1
    db = get_main_database()
    wrapper = kwargs.pop('wrapper', cls.wrap)
    cont = True

    while cont:

        resp = db.view(view, *args, **kwargs)
        cont = False

        for n, obj in enumerate(resp.iterator()):

            key = obj['key']

            if wrap:
                doc = wrapper(obj['doc']) if wrapper else obj['doc']
                docid = doc._id if wrapper else obj['id']
            else:
                docid = obj.get('id', None)
                doc = obj

            if n == per_page:
                if auto_advance:
                    kwargs['startkey'] = key
                    if docid is not None:
                        kwargs['startkey_docid'] = docid
                    if 'skip' in kwargs:
                        del kwargs['skip']

                # we reached the end of the page, load next one
                cont = True
                break

            yield doc
コード例 #13
0
ファイル: update-toplist.py プロジェクト: Mic92/mygpo
    def get_subscriber_count(podcast):
        db = get_main_database()
        subscriber_sum = 0

        for podcast_id in podcast.get_ids():
            x = db.view('subscriptions/by_podcast',
                    startkey    = [podcast_id, None],
                    endkey      = [podcast_id, {}],
                    reduce      = True,
                    group       = True,
                    group_level = 2,
                )
            subscriber_sum += x.count()

        return subscriber_sum
コード例 #14
0
ファイル: dump-sample.py プロジェクト: Mic92/mygpo
    def handle(self, *args, **options):

        docs = set()

        for username in options.get('users', []):
            user = User.get_user(username)

            # User
            docs.add(user._id)

            # Suggestions
            suggestions = Suggestions.for_user(user)
            docs.add(suggestions._id)

            # Podcast States
            for p_state in PodcastUserState.for_user(user):
                docs.add(p_state._id)

                # Categories
                for tag in p_state.tags:
                    c = Category.for_tag(tag)
                    if c: docs.add(c._id)

                # Podcast
                podcast = Podcast.get(p_state.podcast)
                docs.add(podcast._id)

                # Categories
                for s in podcast.tags:
                    for tag in podcast.tags[s]:
                        c = Category.for_tag(tag)
                        if c: docs.add(c._id)

                # Episodes
                for episode in podcast.get_episodes():
                    docs.add(episode._id)

                    # Episode States
                    e_state = episode.get_user_state(user)
                    if e_state._id:
                        docs.add(e_state._id)


        db = get_main_database()
        docs = sorted(docs)
        self.dump(docs, db)
コード例 #15
0
ファイル: models.py プロジェクト: Mic92/mygpo
    def filter(user_id, since=None, until={}, podcast_id=None,
               device_id=None):
        """ Returns Episode Actions for the given criteria"""

        since_str = since.strftime('%Y-%m-%dT%H:%M:%S') if since else None
        until_str = until.strftime('%Y-%m-%dT%H:%M:%S') if until else {}

        if since_str >= until_str:
            return

        if not podcast_id and not device_id:
            view = 'episode_actions/by_user'
            startkey = [user_id, since_str]
            endkey   = [user_id, until_str]

        elif podcast_id and not device_id:
            view = 'episode_actions/by_podcast'
            startkey = [user_id, podcast_id, since_str]
            endkey   = [user_id, podcast_id, until_str]

        elif device_id and not podcast_id:
            view = 'episode_actions/by_device'
            startkey = [user_id, device_id, since_str]
            endkey   = [user_id, device_id, until_str]

        else:
            view = 'episode_actions/by_podcast_device'
            startkey = [user_id, podcast_id, device_id, since_str]
            endkey   = [user_id, podcast_id, device_id, until_str]

        db = get_main_database()
        res = db.view(view,
                startkey = startkey,
                endkey   = endkey
            )

        for r in res:
            action = r['value']
            yield action
コード例 #16
0
ファイル: changescmd.py プロジェクト: Mic92/mygpo
 def get_db(self):
     return get_main_database()
コード例 #17
0
ファイル: sync-design-docs.py プロジェクト: Mic92/mygpo
    def handle(self, *args, **options):

        path = os.path.join(settings.BASE_DIR, '..', 'couchdb', '_design')
        db = get_main_database()
        loader = FileSystemDocsLoader(path)
        loader.sync(db, verbose=True)