예제 #1
0
파일: reader.py 프로젝트: jaramir/leselys
    def run(self):
        self.data = feedparser.parse(self.feed['url'])

        # Update title if it change (yes some guys change it...)
        if self.data.feed['title'] != self.feed['title']:
            self.feed['title'] = self.data.feed['title']
            storage.update_feed(self.feed['_id'], copy.copy(self.feed))

        local_update = get_datetime(self.feed['last_update'])
        remote_update = False
        if self.data.feed.get('updated_parsed'):
            remote_update = get_datetime(self.data.feed.updated_parsed)
            remote_update_raw = get_dicttime(self.data.feed.updated_parsed)
        if self.data.get('updated_parsed'):
            if remote_update:
                if get_datetime(self.data.updated_parsed) > remote_update:
                    remote_update = get_datetime(self.data.updated_parsed)
                    remote_update_raw = get_dicttime(self.data.updated_parsed)
        if self.data.feed.get('published_parsed'):
            if remote_update:
                if get_datetime(self.data.feed.published_parsed) > remote_update:
                    remote_update = get_datetime(self.data.feed.published_parsed)
                    remote_update_raw = get_dicttime(self.data.feed.published_parsed)
        if self.data.get('published_parsed'):
            if remote_update:
                if get_datetime(self.data.published_parsed > remote_update):
                    remote_update = get_datetime(self.data.published_parsed)
                    remote_update_raw = get_dicttime(self.data.published_parsed)

        if not remote_update:
            remote_update = datetime.datetime.now()
            remote_update_raw = get_dicttime(remote_update.timetuple())

        if remote_update > local_update:
            print('!! %s is outdated.' % self.feed['title'].encode('utf-8'))
            readed = []
            for entry in storage.get_stories(self.feed['_id']):
                if entry['read']:
                    readed.append(entry['title'])

            if len(self.data.entries) <= int(config.get('worker', 'story_before_retention')):
                do_retention = False
            else:
                do_retention = True

            retriever = Retriever(self.data, self.feed, do_retention=do_retention)
            retriever.start()
            retriever.join()

            for entry in readed:
                if storage.get_story_by_title(entry):
                    entry = storage.get_story_by_title(entry)
                    entry['read'] = True
                    storage.update_story(entry['_id'], copy.copy(entry))

            self.feed['last_update'] = remote_update_raw
            storage.update_feed(self.feed_id, self.feed)

        else:
            print('=> %s is up-to-date.' % self.feed['title'].encode('utf-8'))
예제 #2
0
    def run(self):
        # This feed comes from database
        feed = backend.get_feed_by_title(self.title)
        url = feed['url']

        for entry in self.data:
            title = entry['title']
            link = entry['link']
            try:
                description = entry['content'][0]['value']
            except KeyError:
                description = entry['summary']

            last_update = get_dicttime(entry.updated_parsed)

            if entry.get('published_parsed', False):
                published = get_dicttime(entry.published_parsed)
            else:
                published = None

            _id = backend.add_story({
                    'title': title,
                    'link': link,
                    'description': description,
                    'published': published,
                    'last_update': last_update,
                    'feed_id': feed['_id'],
                    'read': False})
예제 #3
0
    def run(self):
        # This feed comes from database
        feed = backend.get_feed_by_title(self.title)
        url = feed['url']

        for entry in self.data:
            title = entry['title']
            link = entry['link']
            try:
                description = entry['content'][0]['value']
            except KeyError:
                description = entry['summary']

            last_update = get_dicttime(entry.updated_parsed)

            if entry.get('published_parsed', False):
                published = get_dicttime(entry.published_parsed)
            else:
                published = None

            _id = backend.add_story({
                'title': title,
                'link': link,
                'description': description,
                'published': published,
                'last_update': last_update,
                'feed_id': feed['_id'],
                'read': False
            })
예제 #4
0
파일: reader.py 프로젝트: orva/leselys
    def run(self):
        # This feed comes from database
        feed = storage.get_feed_by_title(self.title)

        for entry in self.data:
            title = entry['title']
            link = entry['link']
            try:
                description = entry['content'][0]['value']
            except KeyError:
                description = entry['summary']

            if entry.get('updated_parsed'):
                last_update = get_dicttime(entry.updated_parsed)
            else:
                last_update = get_dicttime(datetime.datetime.now().timetuple())

            if entry.get('published_parsed', False):
                published = get_dicttime(entry.published_parsed)
            else:
                published = get_dicttime(datetime.datetime.now().timetuple())

            storage.add_story({
                'title': title,
                'link': link,
                'description': description,
                'published': published,
                'last_update': last_update,
                'feed_id': feed['_id'],
                'read': False})
예제 #5
0
    def add(self, url):
        url = url.strip()
        feed = feedparser.parse(url)

        # Bad feed
        if not feed.feed.get('title', False):
            return {'success': False, 'output':'Bad feed'}

        title = feed.feed['title']

        feed_id = backend.get_feed_by_title(title)
        if not feed_id:
            if feed.feed.get('updated_parsed'):
                feed_update = get_dicttime(feed.feed.updated_parsed)
            elif feed.get('updated_parsed'):
                feed_update = get_dicttime(feed.updated_parsed)
            elif feed.feed.get('published_parsed'):
                feed_update = get_dicttime(feed.feed.published_parsed)
            elif feed.get('published_parsed'):
                feed_update = get_dicttime(feed.published_parsed)
            else:
                return {'success':False, 'output':'Parsing error'}
            feed_id = backend.add_feed({'url':url, 'title': title, 'last_update': feed_update})
        else:
            return {'success':False, 'output':'Feed already exists'}

        retriever = Retriever(feed)
        retriever.start()

        return {
            'success': True,
            'title': title,
            'feed_id': feed_id,
            'output': 'Feed added',
            'counter': len(feed['entries'])}
예제 #6
0
파일: reader.py 프로젝트: Fingel/leselys
    def unread(self, story_id):
        story = storage.get_story_by_id(story_id)
        if not story['read']:
            story['published'] = get_dicttime(story['published'])
            story['last_update'] = get_dicttime(story['last_update'])
            return {'success': False, 'output': 'Story already unreaded'}
        story['read'] = False
        storage.update_story(story['_id'], copy.copy(story))

        story['published'] = get_dicttime(story['published'])
        story['last_update'] = get_dicttime(story['last_update'])

        return {'success': True, 'content': story}
예제 #7
0
파일: reader.py 프로젝트: forksbot/leselys
    def unread(self, story_id):
        story = storage.get_story_by_id(story_id)
        if not story['read']:
            story['published'] = get_dicttime(story['published'])
            story['last_update'] = get_dicttime(story['last_update'])
            return {'success': False, 'output': 'Story already unreaded'}
        story['read'] = False
        storage.update_story(story['_id'], copy.copy(story))

        story['published'] = get_dicttime(story['published'])
        story['last_update'] = get_dicttime(story['last_update'])

        return {'success': True, 'content': story}
예제 #8
0
    def get(self, feed_id=False, feed_type="combined-feed", order_type="user", start=0, stop=50):
        # Special feeds
        if not feed_id:
            if feed_type == "combined-feed":
                if order_type == "user" or order_type not in ['unreaded', 'published']:
                    order_type = storage.get_feed_setting('combined-feed' , 'ordering')
                    if not order_type:
                        storage.set_feed_setting('combined-feed', 'ordering', 'unreaded')
                        order_type = 'unreaded'
                    else:
                        order_type = order_type['value']
            elif feed_type == "stared-feed":
                if order_type == "user" or order_type not in ['unreaded', 'published']:
                    order_type = storage.get_feed_setting('stared-feed', 'ordering')
                    if not order_type:
                        storage.set_feed_setting('stared-feed', 'ordering', 'unreaded')
                        order_type = 'unreaded'
                    else:
                        order_type = order_type['value']
            else:
                raise Exception('Unknown feed type (%s)' % feed_type)
        # Normal feed
        else:
            if order_type == "user" or order_type not in ['unreaded', 'published']:
                order_type = storage.get_feed_setting(feed_id, 'ordering')
                if not order_type:
                    storage.set_feed_setting(feed_id, 'ordering', 'unreaded')
                    order_type = 'unreaded'
                else:
                    order_type = order_type['value']

        # Get stories
        if not feed_id:
            if feed_type == "combined-feed":
                stories = storage.all_stories(order_type, start, stop)
            elif feed_type == "stared-feed":
                stories = storage.all_stared(order_type, start, stop)
            else:
                raise Exception('Unknown feed type (%s)' % feed_type)
        else:
            stories = storage.get_stories(feed_id, order_type, start, stop)

        length = storage.get_feed_unread_count(feed_id)
        count = storage.get_feed_count(feed_id)
        res = []
        for story in stories:
            story['last_update'] = get_dicttime(story['last_update'])
            story['published'] = get_dicttime(story['published'])
            res.append(story)

        return {'entries': res, 'ordering': order_type, 'detail': {'start': start, 'stop': stop, 'length': length, 'count':count}}
예제 #9
0
    def run(self):
        self.data = feedparser.parse(self.feed['url'])

        local_update = get_datetime(self.feed['last_update'])
        remote_update = False
        if self.data.feed.get('updated_parsed'):
            remote_update = get_datetime(self.data.feed.updated_parsed)
            remote_update_raw = get_dicttime(self.data.feed.updated_parsed)
        if self.data.get('updated_parsed'):
            if remote_update:
                if get_datetime(self.data.updated_parsed) > remote_update:
                    remote_update = get_datetime(self.data.updated_parsed)
                    remote_update_raw = get_dicttime(self.data.updated_parsed)
        if self.data.feed.get('published_parsed'):
            if remote_update:
                if get_datetime(self.data.feed.published_parsed) > remote_update:
                    remote_update = get_datetime(self.data.feed.published_parsed)
                    remote_update_raw = get_dicttime(self.data.feed.published_parsed)
        if self.data.get('published_parsed'):
            if remote_update:
                if get_datetime(self.data.published_parsed > remote_update):
                    remote_update = get_datetime(self.data.published_parsed)
                    remote_update_raw = get_dicttime(self.data.published_parsed)

        if not remote_update:
            remote_update = datetime.datetime.now()
            remote_update_raw = get_dicttime(remote_update.timetuple())

        if remote_update > local_update:
            print(':: %s is outdated' % self.feed['title'].encode('utf-8'))
            readed = []
            for entry in storage.get_stories(self.feed['_id']):
                if entry['read']:
                    readed.append(entry['title'])
                storage.remove_story(entry['_id'])

            retriever = Retriever(self.data)
            retriever.start()
            retriever.join()

            for entry in readed:
                if storage.get_story_by_title(entry):
                    entry = storage.get_story_by_title(entry)
                    entry['read'] = True
                    storage.update_story(entry['_id'], copy.copy(entry))

            self.feed['last_update'] = remote_update_raw
            storage.update_feed(self.feed_id, self.feed)
예제 #10
0
파일: reader.py 프로젝트: jaramir/leselys
    def add(self, url):
        feed_guesser = self.get_feed(url)
        if feed_guesser['success']:
            feed, url = feed_guesser['output']
        else:
            return feed_guesser

        # Bad feed
        if feed.version == '' or not feed.feed.get('title'):
            return {'success': False, 'output': 'Bad feed'}

        title = feed.feed['title']
        feed_id = storage.get_feed_by_title(title)
        if not feed_id:
            if feed.feed.get('updated_parsed'):
                feed_update = get_dicttime(feed.feed.updated_parsed)
            elif feed.get('updated_parsed'):
                feed_update = get_dicttime(feed.updated_parsed)
            elif feed.feed.get('published_parsed'):
                feed_update = get_dicttime(feed.feed.published_parsed)
            elif feed.get('published_parsed'):
                feed_update = get_dicttime(feed.published_parsed)
            else:
                feed_update = get_dicttime(datetime.datetime.now().timetuple())

            feed_id = storage.add_feed({'url': url,
                                        'title': title,
                                        'last_update': feed_update})
        else:
            return {'success': False, 'output': 'Feed already exists'}

        retriever = Retriever(feed, do_retention=False)
        retriever.start()

        return {
            'success': True,
            'title': title,
            'url': url,
            'feed_id': feed_id,
            'output': 'Feed added',
            'counter': len(feed['entries'])}
예제 #11
0
파일: reader.py 프로젝트: Fingel/leselys
    def read(self, story_id):
        """
        Return story content, set it at readed state and give
        previous read state for counter
        """
        story = storage.get_story_by_id(story_id)
        if story['read']:
            story['published'] = get_dicttime(story['published'])
            story['last_update'] = get_dicttime(story['last_update'])
            return {'success': False,
                    'output': 'Story already readed',
                    'content': story}

        # Save read state before update it for javascript counter in UI
        story['read'] = True
        storage.update_story(story['_id'], copy.copy(story))

        story['published'] = get_dicttime(story['published'])
        story['last_update'] = get_dicttime(story['last_update'])

        return {'success': True, 'content': story}
예제 #12
0
파일: reader.py 프로젝트: jaramir/leselys
    def run(self):
        # This feed comes from database
        feed = storage.get_feed_by_title(self.title)

        for entry in self.data:
            title = entry['title']
            link = entry['link']

            if storage.get_story_by_title(title):
                storage.remove_story(storage.get_story_by_title(title)['_id'])

            try:
                description = entry['content'][0]['value']
            except KeyError:
                description = entry['summary']

            if entry.get('updated_parsed'):
                last_update = get_dicttime(entry.updated_parsed)
            else:
                last_update = get_dicttime(datetime.datetime.now().timetuple())
            if entry.get('published_parsed', False):
                published = get_dicttime(entry.published_parsed)
                published_datetime = get_datetime(entry.published_parsed)
            else:
                published = get_dicttime(datetime.datetime.now().timetuple())
                published_datetime = datetime.datetime.now()

            if self.do_retention:
                delta = datetime.datetime.now() - published_datetime
                if delta.days > int(config.get('worker', 'retention')):
                    continue

            storage.add_story({
                'title': title,
                'link': link,
                'description': description,
                'published': published,
                'last_update': last_update,
                'feed_id': feed['_id'],
                'read': False})
예제 #13
0
    def add(self, url):
        url = url.strip()
        feed = feedparser.parse(url)

        # Bad feed
        if not feed.feed.get('title', False):
            return {'success': False, 'output': 'Bad feed'}

        title = feed.feed['title']

        feed_id = backend.get_feed_by_title(title)
        if not feed_id:
            if feed.feed.get('updated_parsed'):
                feed_update = get_dicttime(feed.feed.updated_parsed)
            elif feed.get('updated_parsed'):
                feed_update = get_dicttime(feed.updated_parsed)
            elif feed.feed.get('published_parsed'):
                feed_update = get_dicttime(feed.feed.published_parsed)
            elif feed.get('published_parsed'):
                feed_update = get_dicttime(feed.published_parsed)
            else:
                return {'success': False, 'output': 'Parsing error'}
            feed_id = backend.add_feed({
                'url': url,
                'title': title,
                'last_update': feed_update
            })
        else:
            return {'success': False, 'output': 'Feed already exists'}

        retriever = Retriever(feed)
        retriever.start()

        return {
            'success': True,
            'title': title,
            'feed_id': feed_id,
            'output': 'Feed added',
            'counter': len(feed['entries'])
        }
예제 #14
0
파일: reader.py 프로젝트: Epictetus/leselys
    def run(self):
        self.data = feedparser.parse(self.feed['url'])

        local_update = get_datetime(self.feed['last_update'])
        if self.data.feed.get('updated_parsed'):
            remote_update = get_datetime(self.data.feed.updated_parsed)
            remote_update_raw = get_dicttime(self.data.feed.updated_parsed)
        elif self.data.get('updated_parsed'):
            remote_update = get_datetime(self.data.updated_parsed)
            remote_update_raw = get_dicttime(self.data.updated_parsed)
        elif self.data.feed.get('published_parsed'):
            remote_update = get_datetime(self.data.feed.published_parsed)
            remote_update_raw = get_dicttime(self.data.feed.published_parsed)
        elif self.data.get('published_parsed'):
            remote_update = get_datetime(self.data.published_parsed)
            remote_update_raw = get_dicttime(self.data.published_parsed)
        else:
            return

        if remote_update > local_update:
            print(':: %s is outdated' % self.feed['title'])
            readed = []
            for entry in backend.get_stories(self.feed['_id']):
                if entry['read']:
                    readed.append(entry['title'])
                backend.remove_story(entry['_id'])

            retriever = Retriever(self.data)
            retriever.start()
            retriever.join()

            for entry in readed:
                if backend.get_story_by_title(entry):
                    entry = backend.get_story_by_title(entry)
                    entry['read'] = True
                    backend.update_story(entry['_id'], copy.copy(entry))

            self.feed['last_update'] = remote_update_raw
            backend.update_feed(self.feed_id, self.feed)
예제 #15
0
    def run(self):
        self.data = feedparser.parse(self.feed['url'])

        local_update = get_datetime(self.feed['last_update'])
        if self.data.feed.get('updated_parsed'):
            remote_update = get_datetime(self.data.feed.updated_parsed)
            remote_update_raw = get_dicttime(self.data.feed.updated_parsed)
        elif self.data.get('updated_parsed'):
            remote_update = get_datetime(self.data.updated_parsed)
            remote_update_raw = get_dicttime(self.data.updated_parsed)
        elif self.data.feed.get('published_parsed'):
            remote_update = get_datetime(self.data.feed.published_parsed)
            remote_update_raw = get_dicttime(self.data.feed.published_parsed)
        elif self.data.get('published_parsed'):
            remote_update = get_datetime(self.data.published_parsed)
            remote_update_raw = get_dicttime(self.data.published_parsed)
        else:
            return

        if remote_update > local_update:
            print(':: %s is outdated' % self.feed['title'])
            readed = []
            for entry in backend.get_stories(self.feed['_id']):
                if entry['read']:
                    readed.append(entry['title'])
                backend.remove_story(entry['_id'])

            retriever = Retriever(self.data)
            retriever.start()
            retriever.join()

            for entry in readed:
                if backend.get_story_by_title(entry):
                    entry = backend.get_story_by_title(entry)
                    entry['read'] = True
                    backend.update_story(entry['_id'], copy.copy(entry))

            self.feed['last_update'] = remote_update_raw
            backend.update_feed(self.feed_id, self.feed)
예제 #16
0
파일: reader.py 프로젝트: forksbot/leselys
    def read(self, story_id):
        """
        Return story content, set it at readed state and give
        previous read state for counter
        """
        story = storage.get_story_by_id(story_id)
        if story['read']:
            story['published'] = get_dicttime(story['published'])
            story['last_update'] = get_dicttime(story['last_update'])
            return {
                'success': False,
                'output': 'Story already readed',
                'content': story
            }

        # Save read state before update it for javascript counter in UI
        story['read'] = True
        storage.update_story(story['_id'], copy.copy(story))

        story['published'] = get_dicttime(story['published'])
        story['last_update'] = get_dicttime(story['last_update'])

        return {'success': True, 'content': story}
예제 #17
0
파일: reader.py 프로젝트: forksbot/leselys
    def get(self,
            feed_id=False,
            feed_type="combined-feed",
            order_type="user",
            start=0,
            stop=50):
        # Special feeds
        if not feed_id:
            if feed_type == "combined-feed":
                if order_type == "user" or order_type not in [
                        'unreaded', 'published'
                ]:
                    order_type = storage.get_feed_setting(
                        'combined-feed', 'ordering')
                    if not order_type:
                        storage.set_feed_setting('combined-feed', 'ordering',
                                                 'unreaded')
                        order_type = 'unreaded'
                    else:
                        order_type = order_type['value']
            elif feed_type == "stared-feed":
                if order_type == "user" or order_type not in [
                        'unreaded', 'published'
                ]:
                    order_type = storage.get_feed_setting(
                        'stared-feed', 'ordering')
                    if not order_type:
                        storage.set_feed_setting('stared-feed', 'ordering',
                                                 'unreaded')
                        order_type = 'unreaded'
                    else:
                        order_type = order_type['value']
            else:
                raise Exception('Unknown feed type (%s)' % feed_type)
        # Normal feed
        else:
            if order_type == "user" or order_type not in [
                    'unreaded', 'published'
            ]:
                order_type = storage.get_feed_setting(feed_id, 'ordering')
                if not order_type:
                    storage.set_feed_setting(feed_id, 'ordering', 'unreaded')
                    order_type = 'unreaded'
                else:
                    order_type = order_type['value']

        # Get stories
        if not feed_id:
            if feed_type == "combined-feed":
                stories = storage.all_stories(order_type, start, stop)
            elif feed_type == "stared-feed":
                stories = storage.all_stared(order_type, start, stop)
            else:
                raise Exception('Unknown feed type (%s)' % feed_type)
        else:
            stories = storage.get_stories(feed_id, order_type, start, stop)

        length = storage.get_feed_unread_count(feed_id)
        res = []
        for story in stories:
            story['last_update'] = get_dicttime(story['last_update'])
            story['published'] = get_dicttime(story['published'])
            res.append(story)

        return {
            'entries': res,
            'ordering': order_type,
            'detail': {
                'start': start,
                'stop': stop,
                'length': length
            }
        }