def _syncChain(self, blogSync):
'''
Synchronize the blog for the given sync entry.
@param blogSync: BlogSync
The blog sync entry declaring the blog and source from which the blog
has to be updated.
'''
assert isinstance(blogSync,
BlogSync), 'Invalid blog sync %s' % blogSync
source = self.sourceService.getById(blogSync.Source)
log.info('_syncChain blogId=%d, sourceId=%d', blogSync.Blog,
blogSync.Source)
assert isinstance(source, Source)
(scheme, netloc, path, params, query, fragment) = urlparse(source.URI)
if not scheme: scheme = 'http'
q = parse_qsl(query, keep_blank_values=True)
q.append(('asc', 'cId'))
q.append(
('cId.since', blogSync.CId if blogSync.CId is not None else 0))
url = urlunparse(
(scheme, netloc, path + '/' + self.published_posts_path, params,
urlencode(q), fragment))
req = Request(url,
headers={
'Accept': self.acceptType,
'Accept-Charset': self.encodingType,
'X-Filter':
'*,Creator.*,Author.User.*,Author.Source.*',
'User-Agent': 'Magic Browser'
})
try:
resp = urlopen(req)
except (HTTPError, socket.error) as e:
log.error('Read error on %s: %s' % (source.URI, e))
blogSync.LastActivity = None
self.blogSyncService.update(blogSync)
return
if str(resp.status) != '200':
log.error('Read problem on %s, status: %s' %
(source.URI, resp.status))
blogSync.LastActivity = None
self.blogSyncService.update(blogSync)
return
try:
msg = json.load(codecs.getreader(self.encodingType)(resp))
except ValueError as e:
log.error('Invalid JSON data %s' % e)
blogSync.LastActivity = None
self.blogSyncService.update(blogSync)
return
usersForIcons = {}
for post in msg['PostList']:
try:
if post['IsPublished'] != 'True': continue
insert = False
if 'Uuid' in post:
uuid = post['Uuid']
localPost = self.postService.getByUuidAndSource(
uuid, source.Id)
else:
#To support old instances that don't have Uuid attribute
uuid = str(uuid4().hex)
localPost = None
if localPost == None:
if 'DeletedOn' in post: continue
localPost = Post()
localPost.Uuid = uuid
insert = True
if 'DeletedOn' not in post:
#TODO: workaround, read again the Author because sometimes we get access denied
post['Author'] = self._readAuthor(post['Author']['href'])
post['Creator'] = self._readCreator(
post['Creator']['href'])
#if exists local, update it, otherwise continue the original insert
localPost.Type = post['Type']['Key']
localPost.Author, localPost.Creator, needUpdate, isAuthor = self._getCollaboratorForAuthor(
post['Author'], post['Creator'], source)
localPost.Feed = source.Id
localPost.Meta = post['Meta'] if 'Meta' in post else None
localPost.ContentPlain = post[
'ContentPlain'] if 'ContentPlain' in post else None
localPost.Content = post[
'Content'] if 'Content' in post else None
localPost.Order = post['Order'] if 'Order' in post else None
localPost.CreatedOn = current_timestamp()
if blogSync.Auto:
localPost.PublishedOn = current_timestamp()
localPost.WasPublished = True
log.info("received post: %s", str(localPost))
if localPost.Creator and (
localPost.Creator
not in usersForIcons) and needUpdate:
try:
if isAuthor:
usersForIcons[
localPost.Creator] = post['Author']['User']
else:
usersForIcons[
localPost.Creator] = post['Creator']
except KeyError:
pass
else:
localPost.DeletedOn = datetime.strptime(
post['DeletedOn'], '%m/%d/%y %I:%M %p')
# prepare the blog sync model to update the change identifier
blogSync.CId = int(post['CId']) if blogSync.CId is None or int(
post['CId']) > blogSync.CId else blogSync.CId
if insert:
self.blogPostService.insert(blogSync.Blog, localPost)
else:
self.blogPostService.update(blogSync.Blog, localPost)
# update blog sync entry
blogSync.LastActivity = datetime.now().replace(microsecond=0)
self.blogSyncService.update(blogSync)
except KeyError as e:
log.error('Post from source %s is missing attribute %s' %
(source.URI, e))
except Exception as e:
log.error('Error in source %s post: %s' % (source.URI, e))
self._updateIcons(usersForIcons)
blogSync.LastActivity = None
self.blogSyncService.update(blogSync)
def _syncChain(self, blogSync):
'''
Synchronize the blog for the given sync entry.
@param blogSync: BlogSync
The blog sync entry declaring the blog and source from which the blog
has to be updated.
'''
assert isinstance(blogSync, BlogSync), 'Invalid blog sync %s' % blogSync
source = self.sourceService.getById(blogSync.Source)
log.info('_syncChain blogId=%d, sourceId=%d', blogSync.Blog, blogSync.Source)
assert isinstance(source, Source)
(scheme, netloc, path, params, query, fragment) = urlparse(source.URI)
q = parse_qsl(query, keep_blank_values=True)
q.append(('asc', 'cId'))
q.append(('cId.since', blogSync.CId if blogSync.CId is not None else 0))
url = urlunparse((scheme, netloc, path + '/' + self.published_posts_path, params, urlencode(q), fragment))
req = Request(url, headers={'Accept' : self.acceptType, 'Accept-Charset' : self.encodingType,
'X-Filter' : '*,Creator.*,Author.User.*,Author.Source.*', 'User-Agent' : 'Magic Browser'})
try: resp = urlopen(req)
except (HTTPError, socket.error) as e:
log.error('Read error on %s: %s' % (source.URI, e))
return
if str(resp.status) != '200':
log.error('Read problem on %s, status: %s' % (source.URI, resp.status))
return
try: msg = json.load(codecs.getreader(self.encodingType)(resp))
except ValueError as e:
log.error('Invalid JSON data %s' % e)
return
usersForIcons = {}
for post in msg['PostList']:
try:
if post['IsPublished'] != 'True': continue
insert = False
if 'Uuid' in post:
uuid = post['Uuid']
localPost = self.postService.getByUuidAndSource(uuid, source.Id)
else:
#To support old instances that don't have Uuid attribute
uuid = str(uuid4().hex)
localPost = None
if localPost == None:
if 'DeletedOn' in post: continue
localPost = Post()
localPost.Uuid = uuid
insert = True
if 'DeletedOn' not in post:
#TODO: workaround, read again the Author because sometimes we get access denied
post['Author'] = self._readAuthor(post['Author']['href'])
post['Creator'] = self._readCreator(post['Creator']['href'])
#if exists local, update it, otherwise continue the original insert
localPost.Type = post['Type']['Key']
localPost.Author, localPost.Creator, needUpdate, isAuthor = self._getCollaboratorForAuthor(post['Author'], post['Creator'], source)
localPost.Feed = source.Id
localPost.Meta = post['Meta'] if 'Meta' in post else None
localPost.ContentPlain = post['ContentPlain'] if 'ContentPlain' in post else None
localPost.Content = post['Content'] if 'Content' in post else None
localPost.Order = post['Order'] if 'Order' in post else None
localPost.CreatedOn = current_timestamp()
if blogSync.Auto:
localPost.PublishedOn = current_timestamp()
localPost.WasPublished = True
log.info("received post: %s", str(localPost))
if localPost.Creator and (localPost.Creator not in usersForIcons) and needUpdate:
try:
if isAuthor: usersForIcons[localPost.Creator] = post['Author']['User']
else: usersForIcons[localPost.Creator] = post['Creator']
except KeyError:
pass
else:
localPost.DeletedOn = datetime.strptime(post['DeletedOn'], '%m/%d/%y %I:%M %p')
# prepare the blog sync model to update the change identifier
blogSync.CId = int(post['CId']) if blogSync.CId is None or int(post['CId']) > blogSync.CId else blogSync.CId
if insert: self.blogPostService.insert(blogSync.Blog, localPost)
else: self.blogPostService.update(blogSync.Blog, localPost)
# update blog sync entry
blogSync.LastActivity = datetime.now().replace(microsecond=0)
self.blogSyncService.update(blogSync)
except KeyError as e:
log.error('Post from source %s is missing attribute %s' % (source.URI, e))
except Exception as e:
log.error('Error in source %s post: %s' % (source.URI, e))
self._updateIcons(usersForIcons)
blogSync.LastActivity = None
self.blogSyncService.update(blogSync)
