Beispiel #1
0
 def _trimmDeleted(self, posts):
     '''
     Trim the information from the deleted posts.
     '''
     for post in posts:
         assert isinstance(post, BlogPost)
         if BlogPost.DeletedOn in post and post.DeletedOn is not None:
             trimmed = BlogPost()
             trimmed.Id = post.Id
             trimmed.CId = post.CId
             trimmed.DeletedOn = post.DeletedOn
             yield trimmed
         else:
             yield post
Beispiel #2
0
 def _trimmDeleted(self, posts):
     '''
     Trim the information from the deleted posts.
     '''
     for post in posts:
         assert isinstance(post, BlogPost)
         if BlogPost.DeletedOn in post and post.DeletedOn is not None:
             trimmed = BlogPost()
             trimmed.Id = post.Id
             trimmed.CId = post.CId
             trimmed.DeletedOn = post.DeletedOn
             yield trimmed
         else:
             yield post
Beispiel #3
0
 def _trimPosts(self, posts, deleted=True, unpublished=True, published=False):
     '''
     Trim the information from the deleted posts.
     '''
     for post in posts:
         assert isinstance(post, BlogPostMapped)
         if (deleted and BlogPost.DeletedOn in post and post.DeletedOn is not None) \
         or (unpublished and (BlogPost.PublishedOn not in post or post.PublishedOn is None)) \
         or (published and (BlogPost.PublishedOn in post and post.PublishedOn is not None)):
             trimmed = BlogPost()
             trimmed.Id = post.Id
             trimmed.CId = post.CId
             trimmed.IsPublished = post.IsPublished
             trimmed.DeletedOn = post.DeletedOn
             yield trimmed
         else:
             yield post
Beispiel #4
0
    def _syncBlog(self, blogSync):
        '''
        Synchronize the blog for the given sync entry.

        @param blogSync: BlogSync
            The blog sync entry declaring the blog and source from which the blog
            has to be updated.
        '''
        assert isinstance(blogSync, BlogSync), 'Invalid blog sync %s' % blogSync
        source = self.sourceService.getById(blogSync.Source)
        assert isinstance(source, Source)
        (scheme, netloc, path, params, query, fragment) = urlparse(source.URI)

        q = parse_qsl(query, keep_blank_values=True)
        q.append(('asc', 'cId'))
        q.append(('cId.since', blogSync.CId if blogSync.CId is not None else 0))
        if blogSync.SyncStart is not None:
            q.append(('publishedOn.since', blogSync.SyncStart.strftime(self.date_time_format)))
        url = urlunparse((scheme, netloc, path + '/' + self.published_posts_path, params, urlencode(q), fragment))
        req = Request(url, headers={'Accept' : self.acceptType, 'Accept-Charset' : self.encodingType,
                                    'X-Filter' : '*,Author.Source.*,Author.User.*'})
        try: resp = urlopen(req)
        except (HTTPError, socket.error) as e:
            log.error('Read error on %s: %s' % (source.URI, e))
            return

        try: msg = json.load(codecs.getreader(self.encodingType)(resp))
        except ValueError as e:
            log.error('Invalid JSON data %s' % e)
            return
        for post in msg['PostList']:
            try:
                if post['IsPublished'] != 'True' or 'DeletedOn' in post: continue

                lPost = BlogPost()
                lPost.Type = post['Type']['Key']
                lPost.Creator = blogSync.Creator
                lPost.Author = self._getCollaboratorForAuthor(post['Author'], source)
                lPost.Meta = post['Meta'] if 'Meta' in post else None
                lPost.ContentPlain = post['ContentPlain'] if 'ContentPlain' in post else None
                lPost.Content = post['Content'] if 'Content' in post else None
                lPost.CreatedOn = lPost.PublishedOn = current_timestamp()

                # prepare the blog sync model to update the change identifier
                blogSync.CId = int(post['CId']) if blogSync.CId is None or int(post['CId']) > blogSync.CId else blogSync.CId
                blogSync.SyncStart = datetime.strptime(post['PublishedOn'], '%m/%d/%y %I:%M %p')

                # insert post from remote source
                self.blogPostService.insert(blogSync.Blog, lPost)
                # update blog sync entry
                self.blogSyncService.update(blogSync)
            except KeyError as e:
                log.error('Post from source %s is missing attribute %s' % (source.URI, e))
            except Exception as e:
                log.error('Error in source %s post: %s' % (source.URI, e))
Beispiel #5
0
 def _trimPosts(self, posts, deleted=True, unpublished=True, published=False):
     '''
     Trim the information from the deleted posts.
     '''
     for post in posts:
         assert isinstance(post, BlogPostMapped)
         if (deleted and BlogPost.DeletedOn in post and post.DeletedOn is not None) \
         or (unpublished and (BlogPost.PublishedOn not in post or post.PublishedOn is None)) \
         or (published and (BlogPost.PublishedOn in post and post.PublishedOn is not None)):
             trimmed = BlogPost()
             trimmed.Id = post.Id
             trimmed.CId = post.CId
             trimmed.IsPublished = post.IsPublished
             trimmed.DeletedOn = post.DeletedOn
             yield trimmed
         else:
             yield post        
Beispiel #6
0
    def _syncBlog(self, blogSync):
        '''
        Synchronize the blog for the given sync entry.

        @param blogSync: BlogSync
            The blog sync entry declaring the blog and source from which the blog
            has to be updated.
        '''
        assert isinstance(blogSync,
                          BlogSync), 'Invalid blog sync %s' % blogSync
        source = self.sourceService.getById(blogSync.Source)
        assert isinstance(source, Source)
        (scheme, netloc, path, params, query, fragment) = urlparse(source.URI)

        q = parse_qsl(query, keep_blank_values=True)
        q.append(('asc', 'cId'))
        q.append(
            ('cId.since', blogSync.CId if blogSync.CId is not None else 0))
        if blogSync.SyncStart is not None:
            q.append(('publishedOn.since',
                      blogSync.SyncStart.strftime(self.date_time_format)))
        url = urlunparse(
            (scheme, netloc, path + '/' + self.published_posts_path, params,
             urlencode(q), fragment))
        req = Request(url,
                      headers={
                          'Accept': self.acceptType,
                          'Accept-Charset': self.encodingType,
                          'X-Filter': '*,Author.Source.*,Author.User.*',
                          'User-Agent': 'Magic Browser'
                      })
        try:
            resp = urlopen(req)
        except (HTTPError, socket.error) as e:
            log.error('Read error on %s: %s' % (source.URI, e))
            return
        if str(resp.status) != '200':
            log.error('Read problem on %s, status: %s' %
                      (source.URI, resp.status))
            return

        try:
            msg = json.load(codecs.getreader(self.encodingType)(resp))
        except ValueError as e:
            log.error('Invalid JSON data %s' % e)
            return

        usersForIcons = {}
        for post in msg['PostList']:
            try:
                if post['IsPublished'] != 'True' or 'DeletedOn' in post:
                    continue

                lPost = BlogPost()
                lPost.Type = post['Type']['Key']
                lPost.Creator = blogSync.Creator
                lPost.Author, userId = self._getCollaboratorForAuthor(
                    post['Author'], source)
                lPost.Meta = post['Meta'] if 'Meta' in post else None
                lPost.ContentPlain = post[
                    'ContentPlain'] if 'ContentPlain' in post else None
                lPost.Content = post['Content'] if 'Content' in post else None
                lPost.CreatedOn = lPost.PublishedOn = current_timestamp()

                if userId and (userId not in usersForIcons):
                    try:
                        usersForIcons[userId] = post['Author']['User']
                    except KeyError:
                        pass

                # prepare the blog sync model to update the change identifier
                blogSync.CId = int(post['CId']) if blogSync.CId is None or int(
                    post['CId']) > blogSync.CId else blogSync.CId
                blogSync.SyncStart = datetime.strptime(post['PublishedOn'],
                                                       '%m/%d/%y %I:%M %p')

                # insert post from remote source
                self.blogPostService.insert(blogSync.Blog, lPost)
                # update blog sync entry
                self.blogSyncService.update(blogSync)
            except KeyError as e:
                log.error('Post from source %s is missing attribute %s' %
                          (source.URI, e))
            except Exception as e:
                log.error('Error in source %s post: %s' % (source.URI, e))

        self._updateIcons(usersForIcons)