def _trimmDeleted(self, posts):
'''
Trim the information from the deleted posts.
'''
for post in posts:
assert isinstance(post, BlogPost)
if BlogPost.DeletedOn in post and post.DeletedOn is not None:
trimmed = BlogPost()
trimmed.Id = post.Id
trimmed.CId = post.CId
trimmed.DeletedOn = post.DeletedOn
yield trimmed
else:
yield post
def _trimmDeleted(self, posts):
'''
Trim the information from the deleted posts.
'''
for post in posts:
assert isinstance(post, BlogPost)
if BlogPost.DeletedOn in post and post.DeletedOn is not None:
trimmed = BlogPost()
trimmed.Id = post.Id
trimmed.CId = post.CId
trimmed.DeletedOn = post.DeletedOn
yield trimmed
else:
yield post
def _trimPosts(self, posts, deleted=True, unpublished=True, published=False):
'''
Trim the information from the deleted posts.
'''
for post in posts:
assert isinstance(post, BlogPostMapped)
if (deleted and BlogPost.DeletedOn in post and post.DeletedOn is not None) \
or (unpublished and (BlogPost.PublishedOn not in post or post.PublishedOn is None)) \
or (published and (BlogPost.PublishedOn in post and post.PublishedOn is not None)):
trimmed = BlogPost()
trimmed.Id = post.Id
trimmed.CId = post.CId
trimmed.IsPublished = post.IsPublished
trimmed.DeletedOn = post.DeletedOn
yield trimmed
else:
yield post
def _syncBlog(self, blogSync):
'''
Synchronize the blog for the given sync entry.
@param blogSync: BlogSync
The blog sync entry declaring the blog and source from which the blog
has to be updated.
'''
assert isinstance(blogSync, BlogSync), 'Invalid blog sync %s' % blogSync
source = self.sourceService.getById(blogSync.Source)
assert isinstance(source, Source)
(scheme, netloc, path, params, query, fragment) = urlparse(source.URI)
q = parse_qsl(query, keep_blank_values=True)
q.append(('asc', 'cId'))
q.append(('cId.since', blogSync.CId if blogSync.CId is not None else 0))
if blogSync.SyncStart is not None:
q.append(('publishedOn.since', blogSync.SyncStart.strftime(self.date_time_format)))
url = urlunparse((scheme, netloc, path + '/' + self.published_posts_path, params, urlencode(q), fragment))
req = Request(url, headers={'Accept' : self.acceptType, 'Accept-Charset' : self.encodingType,
'X-Filter' : '*,Author.Source.*,Author.User.*'})
try: resp = urlopen(req)
except (HTTPError, socket.error) as e:
log.error('Read error on %s: %s' % (source.URI, e))
return
try: msg = json.load(codecs.getreader(self.encodingType)(resp))
except ValueError as e:
log.error('Invalid JSON data %s' % e)
return
for post in msg['PostList']:
try:
if post['IsPublished'] != 'True' or 'DeletedOn' in post: continue
lPost = BlogPost()
lPost.Type = post['Type']['Key']
lPost.Creator = blogSync.Creator
lPost.Author = self._getCollaboratorForAuthor(post['Author'], source)
lPost.Meta = post['Meta'] if 'Meta' in post else None
lPost.ContentPlain = post['ContentPlain'] if 'ContentPlain' in post else None
lPost.Content = post['Content'] if 'Content' in post else None
lPost.CreatedOn = lPost.PublishedOn = current_timestamp()
# prepare the blog sync model to update the change identifier
blogSync.CId = int(post['CId']) if blogSync.CId is None or int(post['CId']) > blogSync.CId else blogSync.CId
blogSync.SyncStart = datetime.strptime(post['PublishedOn'], '%m/%d/%y %I:%M %p')
# insert post from remote source
self.blogPostService.insert(blogSync.Blog, lPost)
# update blog sync entry
self.blogSyncService.update(blogSync)
except KeyError as e:
log.error('Post from source %s is missing attribute %s' % (source.URI, e))
except Exception as e:
log.error('Error in source %s post: %s' % (source.URI, e))
def _trimPosts(self, posts, deleted=True, unpublished=True, published=False):
'''
Trim the information from the deleted posts.
'''
for post in posts:
assert isinstance(post, BlogPostMapped)
if (deleted and BlogPost.DeletedOn in post and post.DeletedOn is not None) \
or (unpublished and (BlogPost.PublishedOn not in post or post.PublishedOn is None)) \
or (published and (BlogPost.PublishedOn in post and post.PublishedOn is not None)):
trimmed = BlogPost()
trimmed.Id = post.Id
trimmed.CId = post.CId
trimmed.IsPublished = post.IsPublished
trimmed.DeletedOn = post.DeletedOn
yield trimmed
else:
yield post
def _syncBlog(self, blogSync):
'''
Synchronize the blog for the given sync entry.
@param blogSync: BlogSync
The blog sync entry declaring the blog and source from which the blog
has to be updated.
'''
assert isinstance(blogSync,
BlogSync), 'Invalid blog sync %s' % blogSync
source = self.sourceService.getById(blogSync.Source)
assert isinstance(source, Source)
(scheme, netloc, path, params, query, fragment) = urlparse(source.URI)
q = parse_qsl(query, keep_blank_values=True)
q.append(('asc', 'cId'))
q.append(
('cId.since', blogSync.CId if blogSync.CId is not None else 0))
if blogSync.SyncStart is not None:
q.append(('publishedOn.since',
blogSync.SyncStart.strftime(self.date_time_format)))
url = urlunparse(
(scheme, netloc, path + '/' + self.published_posts_path, params,
urlencode(q), fragment))
req = Request(url,
headers={
'Accept': self.acceptType,
'Accept-Charset': self.encodingType,
'X-Filter': '*,Author.Source.*,Author.User.*',
'User-Agent': 'Magic Browser'
})
try:
resp = urlopen(req)
except (HTTPError, socket.error) as e:
log.error('Read error on %s: %s' % (source.URI, e))
return
if str(resp.status) != '200':
log.error('Read problem on %s, status: %s' %
(source.URI, resp.status))
return
try:
msg = json.load(codecs.getreader(self.encodingType)(resp))
except ValueError as e:
log.error('Invalid JSON data %s' % e)
return
usersForIcons = {}
for post in msg['PostList']:
try:
if post['IsPublished'] != 'True' or 'DeletedOn' in post:
continue
lPost = BlogPost()
lPost.Type = post['Type']['Key']
lPost.Creator = blogSync.Creator
lPost.Author, userId = self._getCollaboratorForAuthor(
post['Author'], source)
lPost.Meta = post['Meta'] if 'Meta' in post else None
lPost.ContentPlain = post[
'ContentPlain'] if 'ContentPlain' in post else None
lPost.Content = post['Content'] if 'Content' in post else None
lPost.CreatedOn = lPost.PublishedOn = current_timestamp()
if userId and (userId not in usersForIcons):
try:
usersForIcons[userId] = post['Author']['User']
except KeyError:
pass
# prepare the blog sync model to update the change identifier
blogSync.CId = int(post['CId']) if blogSync.CId is None or int(
post['CId']) > blogSync.CId else blogSync.CId
blogSync.SyncStart = datetime.strptime(post['PublishedOn'],
'%m/%d/%y %I:%M %p')
# insert post from remote source
self.blogPostService.insert(blogSync.Blog, lPost)
# update blog sync entry
self.blogSyncService.update(blogSync)
except KeyError as e:
log.error('Post from source %s is missing attribute %s' %
(source.URI, e))
except Exception as e:
log.error('Error in source %s post: %s' % (source.URI, e))
self._updateIcons(usersForIcons)