def process_feed(self, appid, feed_url): appconf = M.AppConfig.query.get(_id=appid) if not appconf: return c.project = appconf.project app = ForgeBlogApp(c.project, appconf) c.app = app allura_base.log.info("Get feed: %s" % feed_url) f = feedparser.parse(feed_url) if f.bozo: allura_base.log.exception("%s: %s" % (feed_url, f.bozo_exception)) return for e in f.entries: title = e.title if 'content' in e: content = u'' for ct in e.content: if ct.type != 'text/html': content += '[plain]%s[/plain]' % ct.value else: parser = MDHTMLParser() parser.feed(ct.value) parser.close() # must be before using the result_doc markdown_content = html2text.html2text( parser.result_doc, baseurl=e.link) content += markdown_content else: content = '[plain]%s[/plain]' % getattr( e, 'summary', getattr(e, 'subtitle', getattr(e, 'title'))) content += u' [link](%s)' % e.link updated = datetime.utcfromtimestamp(mktime(e.updated_parsed)) base_slug = BM.BlogPost.make_base_slug(title, updated) b_count = BM.BlogPost.query.find( dict(slug=base_slug, app_config_id=appid)).count() if b_count == 0: post = BM.BlogPost(title=title, text=content, timestamp=updated, app_config_id=appid, tool_version={'blog': version.__version__}, state='published') post.neighborhood_id = c.project.neighborhood_id post.make_slug() post.commit() session(BM.BlogPost).flush()
def import_news(project, pid, frs_mapping, sf_project_shortname, nbhd): from forgeblog import model as BM posts = os.listdir(os.path.join(options.output_dir, pid, 'news')) if len(posts): news_app = project.app_instance('news') if not news_app: news_app = project.install_app('blog', 'news', mount_label='News') h.set_context(project.shortname, 'news', neighborhood=nbhd) # make all the blog posts for post in posts: if '.json' == post[-5:]: post_data = loadjson(pid, 'news', post) create_date = datetime.strptime(post_data.createdOn, '%Y-%m-%d %H:%M:%S') p = BM.BlogPost.query.get(title=post_data.title, timestamp=create_date, app_config_id=news_app.config._id) if not p: p = BM.BlogPost(title=post_data.title, timestamp=create_date, app_config_id=news_app.config._id) p.text = convert_post_content(frs_mapping, sf_project_shortname, post_data.body, nbhd) p.mod_date = create_date p.state = 'published' if not p.slug: p.make_slug() if not p.history().first(): p.commit() ThreadLocalORMSession.flush_all() M.Thread(discussion_id=p.app_config.discussion_id, ref_id=p.index_id(), subject='%s discussion' % p.title) user = get_user(post_data.createdByUsername) p.history().first().author = dict( id=user._id, username=user.username, display_name=user.get_pref('display_name')) ThreadLocalORMSession.flush_all()