def _syncChain(self, blogSync): ''' Synchronize the blog for the given sync entry. @param blogSync: BlogSync The blog sync entry declaring the blog and source from which the blog has to be updated. ''' assert isinstance(blogSync, BlogSync), 'Invalid blog sync %s' % blogSync source = self.sourceService.getById(blogSync.Source) log.info('_syncChain blogId=%d, sourceId=%d', blogSync.Blog, blogSync.Source) assert isinstance(source, Source) (scheme, netloc, path, params, query, fragment) = urlparse(source.URI) if not scheme: scheme = 'http' q = parse_qsl(query, keep_blank_values=True) q.append(('asc', 'cId')) q.append( ('cId.since', blogSync.CId if blogSync.CId is not None else 0)) url = urlunparse( (scheme, netloc, path + '/' + self.published_posts_path, params, urlencode(q), fragment)) req = Request(url, headers={ 'Accept': self.acceptType, 'Accept-Charset': self.encodingType, 'X-Filter': '*,Creator.*,Author.User.*,Author.Source.*', 'User-Agent': 'Magic Browser' }) try: resp = urlopen(req) except (HTTPError, socket.error) as e: log.error('Read error on %s: %s' % (source.URI, e)) blogSync.LastActivity = None self.blogSyncService.update(blogSync) return if str(resp.status) != '200': log.error('Read problem on %s, status: %s' % (source.URI, resp.status)) blogSync.LastActivity = None self.blogSyncService.update(blogSync) return try: msg = json.load(codecs.getreader(self.encodingType)(resp)) except ValueError as e: log.error('Invalid JSON data %s' % e) blogSync.LastActivity = None self.blogSyncService.update(blogSync) return usersForIcons = {} for post in msg['PostList']: try: if post['IsPublished'] != 'True': continue insert = False if 'Uuid' in post: uuid = post['Uuid'] localPost = self.postService.getByUuidAndSource( uuid, source.Id) else: #To support old instances that don't have Uuid attribute uuid = str(uuid4().hex) localPost = None if localPost == None: if 'DeletedOn' in post: continue localPost = Post() localPost.Uuid = uuid insert = True if 'DeletedOn' not in post: #TODO: workaround, read again the Author because sometimes we get access denied post['Author'] = self._readAuthor(post['Author']['href']) post['Creator'] = self._readCreator( post['Creator']['href']) #if exists local, update it, otherwise continue the original insert localPost.Type = post['Type']['Key'] localPost.Author, localPost.Creator, needUpdate, isAuthor = self._getCollaboratorForAuthor( post['Author'], post['Creator'], source) localPost.Feed = source.Id localPost.Meta = post['Meta'] if 'Meta' in post else None localPost.ContentPlain = post[ 'ContentPlain'] if 'ContentPlain' in post else None localPost.Content = post[ 'Content'] if 'Content' in post else None localPost.Order = post['Order'] if 'Order' in post else None localPost.CreatedOn = current_timestamp() if blogSync.Auto: localPost.PublishedOn = current_timestamp() localPost.WasPublished = True log.info("received post: %s", str(localPost)) if localPost.Creator and ( localPost.Creator not in usersForIcons) and needUpdate: try: if isAuthor: usersForIcons[ localPost.Creator] = post['Author']['User'] else: usersForIcons[ localPost.Creator] = post['Creator'] except KeyError: pass else: localPost.DeletedOn = datetime.strptime( post['DeletedOn'], '%m/%d/%y %I:%M %p') # prepare the blog sync model to update the change identifier blogSync.CId = int(post['CId']) if blogSync.CId is None or int( post['CId']) > blogSync.CId else blogSync.CId if insert: self.blogPostService.insert(blogSync.Blog, localPost) else: self.blogPostService.update(blogSync.Blog, localPost) # update blog sync entry blogSync.LastActivity = datetime.now().replace(microsecond=0) self.blogSyncService.update(blogSync) except KeyError as e: log.error('Post from source %s is missing attribute %s' % (source.URI, e)) except Exception as e: log.error('Error in source %s post: %s' % (source.URI, e)) self._updateIcons(usersForIcons) blogSync.LastActivity = None self.blogSyncService.update(blogSync)
def _syncChain(self, blogSync): ''' Synchronize the blog for the given sync entry. @param blogSync: BlogSync The blog sync entry declaring the blog and source from which the blog has to be updated. ''' assert isinstance(blogSync, BlogSync), 'Invalid blog sync %s' % blogSync source = self.sourceService.getById(blogSync.Source) log.info('_syncChain blogId=%d, sourceId=%d', blogSync.Blog, blogSync.Source) assert isinstance(source, Source) (scheme, netloc, path, params, query, fragment) = urlparse(source.URI) q = parse_qsl(query, keep_blank_values=True) q.append(('asc', 'cId')) q.append(('cId.since', blogSync.CId if blogSync.CId is not None else 0)) url = urlunparse((scheme, netloc, path + '/' + self.published_posts_path, params, urlencode(q), fragment)) req = Request(url, headers={'Accept' : self.acceptType, 'Accept-Charset' : self.encodingType, 'X-Filter' : '*,Creator.*,Author.User.*,Author.Source.*', 'User-Agent' : 'Magic Browser'}) try: resp = urlopen(req) except (HTTPError, socket.error) as e: log.error('Read error on %s: %s' % (source.URI, e)) return if str(resp.status) != '200': log.error('Read problem on %s, status: %s' % (source.URI, resp.status)) return try: msg = json.load(codecs.getreader(self.encodingType)(resp)) except ValueError as e: log.error('Invalid JSON data %s' % e) return usersForIcons = {} for post in msg['PostList']: try: if post['IsPublished'] != 'True': continue insert = False if 'Uuid' in post: uuid = post['Uuid'] localPost = self.postService.getByUuidAndSource(uuid, source.Id) else: #To support old instances that don't have Uuid attribute uuid = str(uuid4().hex) localPost = None if localPost == None: if 'DeletedOn' in post: continue localPost = Post() localPost.Uuid = uuid insert = True if 'DeletedOn' not in post: #TODO: workaround, read again the Author because sometimes we get access denied post['Author'] = self._readAuthor(post['Author']['href']) post['Creator'] = self._readCreator(post['Creator']['href']) #if exists local, update it, otherwise continue the original insert localPost.Type = post['Type']['Key'] localPost.Author, localPost.Creator, needUpdate, isAuthor = self._getCollaboratorForAuthor(post['Author'], post['Creator'], source) localPost.Feed = source.Id localPost.Meta = post['Meta'] if 'Meta' in post else None localPost.ContentPlain = post['ContentPlain'] if 'ContentPlain' in post else None localPost.Content = post['Content'] if 'Content' in post else None localPost.Order = post['Order'] if 'Order' in post else None localPost.CreatedOn = current_timestamp() if blogSync.Auto: localPost.PublishedOn = current_timestamp() localPost.WasPublished = True log.info("received post: %s", str(localPost)) if localPost.Creator and (localPost.Creator not in usersForIcons) and needUpdate: try: if isAuthor: usersForIcons[localPost.Creator] = post['Author']['User'] else: usersForIcons[localPost.Creator] = post['Creator'] except KeyError: pass else: localPost.DeletedOn = datetime.strptime(post['DeletedOn'], '%m/%d/%y %I:%M %p') # prepare the blog sync model to update the change identifier blogSync.CId = int(post['CId']) if blogSync.CId is None or int(post['CId']) > blogSync.CId else blogSync.CId if insert: self.blogPostService.insert(blogSync.Blog, localPost) else: self.blogPostService.update(blogSync.Blog, localPost) # update blog sync entry blogSync.LastActivity = datetime.now().replace(microsecond=0) self.blogSyncService.update(blogSync) except KeyError as e: log.error('Post from source %s is missing attribute %s' % (source.URI, e)) except Exception as e: log.error('Error in source %s post: %s' % (source.URI, e)) self._updateIcons(usersForIcons) blogSync.LastActivity = None self.blogSyncService.update(blogSync)