def iter_threads(self):
with self.browser:
for story in self.browser.iter_stories():
thread = Thread(story.id)
thread.title = story.title
thread.date = story.date
yield thread
def get_thread(self, id):
if isinstance(id, Thread):
thread = id
id = thread.id
else:
thread = None
story = self.browser.get_story(id)
if not story:
return None
if not thread:
thread = Thread(story.id)
flags = 0
if thread.id not in self.storage.get('seen', default=[]):
flags |= Message.IS_UNREAD
thread.title = story.title
thread.date = story.date
thread.root = Message(thread=thread,
id=0,
title=story.title,
sender=story.author.name,
receivers=None,
date=thread.date,
parent=None,
content=story.body,
children=[],
signature=u'Written by a %s in category %s' %
(self.GENDERS[story.author.sex], story.category),
flags=flags)
return thread
def iter_threads(self):
table = self.parser.select(self.document.getroot(),
'table#listeMessages', 1)
for tr in table.xpath('./tr'):
if tr.attrib.get('class', '') not in ('msgLu', 'msgNonLu'):
continue
author = unicode(self.parser.select(tr, 'td.colEmetteur', 1).text)
link = self.parser.select(tr, 'td.colObjet a', 1)
date_raw = self.parser.select(tr, 'td.colDate1', 1).attrib['data']
jsparams = re.search('\((.+)\)',
link.attrib['onclick']).groups()[0]
jsparams = [i.strip('\'" ') for i in jsparams.split(',')]
page_id, _id, unread = jsparams
# this means unread on the website
unread = False if unread == "false" else True
# 2012/02/29:01h30min45sec
dt_match = re.match('(\d+)/(\d+)/(\d+):(\d+)h(\d+)min(\d+)sec',
date_raw).groups()
dt_match = [int(d) for d in dt_match]
thread = Thread(_id)
thread._link_id = (page_id, unread)
thread.date = datetime(*dt_match)
thread.title = unicode(link.text)
message = Message(thread, 0)
message.set_empty_fields(None)
message.flags = message.IS_HTML
message.title = thread.title
message.date = thread.date
message.sender = author
message.content = NotLoaded # This is the only thing we are missing
thread.root = message
yield thread
def iter_threads(self):
with self.browser:
for story in self.browser.iter_stories():
thread = Thread(story.id)
thread.title = story.title
thread.date = story.date
yield thread
def iter_threads(self):
table = self.parser.select(self.document.getroot(), "table#listeMessages", 1)
for tr in table.xpath("./tr"):
if tr.attrib.get("class", "") not in ("msgLu", "msgNonLu"):
continue
author = unicode(self.parser.select(tr, "td.colEmetteur", 1).text)
link = self.parser.select(tr, "td.colObjet a", 1)
date_raw = self.parser.select(tr, "td.colDate1", 1).attrib["data"]
jsparams = re.search("\((.+)\)", link.attrib["onclick"]).groups()[0]
jsparams = [i.strip("'\" ") for i in jsparams.split(",")]
page_id, _id, unread = jsparams
# this means unread on the website
unread = False if unread == "false" else True
# 2012/02/29:01h30min45sec
dt_match = re.match("(\d+)/(\d+)/(\d+):(\d+)h(\d+)min(\d+)sec", date_raw).groups()
dt_match = [int(d) for d in dt_match]
thread = Thread(_id)
thread._link_id = (page_id, unread)
thread.date = datetime(*dt_match)
thread.title = unicode(link.text)
message = Message(thread, 0)
message.set_empty_fields(None)
message.flags = message.IS_HTML
message.title = thread.title
message.date = thread.date
message.sender = author
message.content = NotLoaded # This is the only thing we are missing
thread.root = message
yield thread
def iter_threads(self):
for thread in self.browser.get_threads():
t = Thread(thread['id'])
t.flags = Thread.IS_DISCUSSION
t.title = u'Discussion with %s' % thread['name']
t.date = local2utc(datetime.datetime.fromtimestamp(thread['last_message']['utc_timestamp']))
yield t
def iter_threads(self):
for thread in self.browser.get_threads():
t = Thread(thread['id'])
t.flags = Thread.IS_DISCUSSION
t.title = u'Discussion with %s' % thread['name']
t.date = local2utc(datetime.datetime.fromtimestamp(thread['last_message']['utc_timestamp']))
yield t
def iter_threads(self):
for article in Newsfeed(self.RSS_FEED,
GenericNewspaperModule.RSSID).iter_entries():
thread = Thread(article.id)
thread.title = article.title
thread.date = article.datetime
yield (thread)
def get_thread(self, id):
if isinstance(id, Thread):
thread = id
id = thread.id
else:
thread = None
with self.browser:
story = self.browser.get_story(id)
if not story:
return None
if not thread:
thread = Thread(story.id)
flags = 0
if not thread.id in self.storage.get('seen', default=[]):
flags |= Message.IS_UNREAD
thread.title = story.title
thread.date = story.date
thread.root = Message(thread=thread,
id=0,
title=story.title,
sender=story.author.name,
receivers=None,
date=thread.date,
parent=None,
content=story.body,
children=[],
signature='Written by a %s (%s)' % (self.GENDERS[story.author.sex], story.author.email),
flags=flags)
return thread
def get_thread(self, id):
thr = Thread(id=id)
self.fill_thread(thr)
thr.date = thr.root.date
thr.title = thr.root.title
thr.url = thr.root.url
return thr
def _build_thread(self, data):
thread = Thread("%s.%s" % (data["commentable_id"], data["id"]))
thread.title = data["title"]
thread.date = dateutil.parser.parse(data["created_at"])
thread.url = self.browser.thread.build(course=self.browser.course, topic=data["commentable_id"], id=data["id"])
thread.root = self._build_message(data, thread)
thread._messages_count = data["comments_count"] + 1
return thread
def iter_threads(self):
for msg in self.browser.iter_dates():
thread = Thread(msg.id)
thread.title = msg.title
thread.date = msg.date
thread.root = msg
msg.thread = thread
yield thread
def iter_threads(self):
for msg in self.browser.iter_dates():
thread = Thread(msg.id)
thread.title = msg.title
thread.date = msg.date
thread.root = msg
msg.thread = thread
yield thread
def _build_thread(self, data):
thread = Thread("%s.%s" % (data["commentable_id"], data["id"]))
thread.title = data["title"]
thread.date = dateutil.parser.parse(data["created_at"])
thread.url = self.browser.thread.build(course=self.browser.course, topic=data["commentable_id"], id=data["id"])
thread.root = self._build_message(data, thread)
thread._messages_count = data["comments_count"] + 1
return thread
def iter_threads(self):
for thread in self.browser.get_threads():
t = Thread(thread['id'])
t.flags = Thread.IS_DISCUSSION
for user in thread['participants']:
if user['user']['id'] != self.browser.my_id:
t.title = u'Discussion with %s' % user['user']['display_name']
t.date = parse_date(thread['modification_date'])
yield t
def iter_threads(self):
threads = self.browser.get_threads_list()
for thread in threads:
t = Thread(thread['userid'])
t.flags = Thread.IS_DISCUSSION
t.title = u'Discussion with %s' % thread['user']['username']
t.date = datetime.fromtimestamp(thread['timestamp'])
yield t
def iter_threads(self):
for thread in self.browser.get_threads():
t = Thread(thread['id'])
t.flags = Thread.IS_DISCUSSION
for user in thread['participants']:
if user['user']['id'] != self.browser.my_id:
t.title = u'Discussion with %s' % user['user']['first_name']
t.date = parse_date(thread['modification_date'])
yield t
def iter_threads(self):
threads = self.browser.get_threads_list()
for thread in threads:
t = Thread(thread['userid'])
t.flags = Thread.IS_DISCUSSION
t.title = u'Discussion with %s' % thread['user']['username']
t.date = datetime.fromtimestamp(thread['timestamp'])
yield t
def _iter_threads(self, root_link=None):
links = list(self.browser.iter_links(root_link.url if root_link else None))
for link in links:
if link.type == link.FORUM:
link.title = '%s[%s]' % (root_link.title if root_link else '', link.title)
for thread in self._iter_threads(link):
yield thread
if link.type == link.TOPIC:
thread = Thread(url2id(link.url))
thread.title = ('%s ' % root_link.title if root_link else '') + link.title
thread.date = link.date
thread.flags = thread.IS_DISCUSSION
yield thread
def _iter_threads(self, root_link=None):
links = list(self.browser.iter_links(root_link.url if root_link else None))
for link in links:
if link.type == link.FORUM:
link.title = '%s[%s]' % (root_link.title if root_link else '', link.title)
for thread in self._iter_threads(link):
yield thread
if link.type == link.TOPIC:
thread = Thread(url2id(link.url))
thread.title = ('%s ' % root_link.title if root_link else '') + link.title
thread.date = link.date
thread.flags = thread.IS_DISCUSSION
yield thread
def get_thread(self, id, getseen=True):
if not isinstance(id, Thread):
thread = None
else:
thread = id
id = thread.id
# Check if we have seen all comments of this thread.
oldhash = self.storage.get('hash', id, default="")
newhash = self.browser.get_hash(thread._rsscomment)
if not getseen and oldhash == newhash:
return None
self.storage.set('hash', id, newhash)
if thread.date:
self.storage.set('date', id, thread.date)
self.storage.save()
with self.browser:
content = self.browser.get_content(id)
if not content:
return None
if not thread:
thread = Thread(content.id)
flags = Message.IS_HTML
if not thread.id in self.storage.get('seen', default={}):
flags |= Message.IS_UNREAD
thread.title = content.title
if not thread.date:
thread.date = content.date
thread.root = Message(thread=thread,
id='0', # root message
title=content.title,
sender=content.author or u'',
receivers=None,
date=thread.date,
parent=None,
content=content.body,
signature='URL: %s' % self.browser.absurl(id2url(content.id)),
children=[],
flags=flags)
for com in content.comments:
self._insert_comment(com, thread.root, getseen)
return thread
def iter_threads(self):
whats = set()
if self.config['get_news']:
whats.add(self.RSS_NEWSPAPERS)
if self.config['get_telegrams']:
whats.add(self.RSS_TELEGRAMS)
for what in whats:
for article in Newsfeed(what, url2id).iter_entries():
thread = Thread(article.id)
thread.title = article.title
if article.datetime:
thread.date = article.datetime
yield thread
def iter_threads(self):
daily = []
for article in Newsfeed(self.RSS_FEED, self.RSSID).iter_entries():
if "/news-brief/" in article.link:
day = self.browser.get_daily_date(article.link)
if day and (day not in daily):
localid = url2id(article.link)
daily.append(day)
id, title, date = self.browser.get_daily_infos(day)
id = id + "#" + localid
thread = Thread(id)
thread.title = title
thread.date = date
yield(thread)
elif day is None:
thread = Thread(article.link)
thread.title = article.title
thread.date = article.datetime
yield(thread)
else:
thread = Thread(article.link)
thread.title = article.title
thread.date = article.datetime
yield(thread)
def iter_threads(self):
daily = []
for article in Newsfeed(self.RSS_FEED, self.RSSID).iter_entries():
if "/news-brief/" in article.link:
day = self.browser.get_daily_date(article.link)
if day and (day not in daily):
localid = url2id(article.link)
daily.append(day)
id, title, date = self.browser.get_daily_infos(day)
id = id + "#" + localid
thread = Thread(id)
thread.title = title
thread.date = date
yield (thread)
elif day is None:
thread = Thread(article.link)
thread.title = article.title
thread.date = article.datetime
yield (thread)
else:
thread = Thread(article.link)
thread.title = article.title
thread.date = article.datetime
yield (thread)
def iter_threads(self):
whats = set()
for param, url in self.FEEDS.iteritems():
if self.config[param].get():
whats.add(url)
for what in whats:
for article in Newsfeed(what, rssid).iter_entries():
if article.datetime and (datetime.now() - article.datetime) > timedelta(days=60):
continue
thread = Thread(article.id, article.link)
thread.title = article.title
thread._rsscomment = article.rsscomment
if article.datetime:
thread.date = article.datetime
yield thread
def get_thread(self, id, getseen=True):
if not isinstance(id, Thread):
thread = None
else:
thread = id
id = thread.id
if thread.date:
self.storage.set("date", id, thread.date)
self.storage.save()
with self.browser:
content = self.browser.get_content(id)
if not content:
return None
if not thread:
thread = Thread(content.id)
flags = Message.IS_HTML
if not thread.id in self.storage.get("seen", default={}):
flags |= Message.IS_UNREAD
thread.title = content.title
if not thread.date:
thread.date = content.date
thread.root = Message(
thread=thread,
id="0", # root message
title=content.title,
sender=content.author or u"",
receivers=None,
date=thread.date,
parent=None,
content=content.body,
signature="URL: %s" % self.browser.absurl(id2url(content.id)),
children=[],
flags=flags,
)
for com in content.comments:
self._insert_comment(com, thread.root, getseen)
return thread
def get_thread(self, id, getseen=True):
if not isinstance(id, Thread):
thread = None
else:
thread = id
id = thread.id
if thread.date:
self.storage.set('date', id, thread.date)
self.storage.save()
content = self.browser.get_content(id)
if not content:
return None
if not thread:
thread = Thread(content.id)
flags = Message.IS_HTML
if thread.id not in self.storage.get('seen', default={}):
flags |= Message.IS_UNREAD
thread.title = content.title
if not thread.date:
thread.date = content.date
thread.root = Message(
thread=thread,
id='0', # root message
url=self.browser.absurl(id2url(content.id)),
title=content.title,
sender=content.author or u'',
receivers=None,
date=thread.date,
parent=None,
content=content.body,
signature='URL: %s' % self.browser.absurl(id2url(content.id)),
children=[],
flags=flags)
for com in content.comments:
self._insert_comment(com, thread.root, getseen)
return thread
def browse_forum_mode(forum, prefix, mode):
start = 0
count = 50
while True:
if mode:
topics = self._conn.get_topic(forum['forum_id'], start, start+count-1, mode)
else:
topics = self._conn.get_topic(forum['forum_id'], start, start+count-1)
all_ignored = True
for topic in topics['topics']:
t = Thread(topic['topic_id'])
t.title = unicode(str(topic['topic_title']), 'utf-8')
t.date = self._get_time(topic)
if not unread or topic.get('new_post'):
all_ignored = False
yield t
start += count
if start >= topics['total_topic_num'] or all_ignored:
break
def browse_forum_mode(forum, prefix, mode):
start = 0
count = 50
while True:
if mode:
topics = self._conn.get_topic(xmlrpc_str(forum['forum_id']), start, start+count-1, mode)
else:
topics = self._conn.get_topic(xmlrpc_str(forum['forum_id']), start, start+count-1)
all_ignored = True
for topic in topics['topics']:
t = Thread(xmlrpc_str(topic['topic_id']))
t.title = xmlrpc_str(topic['topic_title'])
t.date = self._get_time(topic)
if not unread or topic.get('new_post'):
all_ignored = False
yield t
start += count
if start >= topics['total_topic_num'] or all_ignored:
break
def get_thread(self, _id):
thread = Thread(_id)
thread.title = self.document.find('div', 'PADtitreBlanc_txt').find('center').string
thread.flags = Thread.IS_DISCUSSION
root = True
for message in self._get_messages(thread):
if root:
message.children = []
thread.root = message
thread.date = message.date
message.title = thread.title
root = False
else:
message.title = 'Re: %s' % thread.title
message.children = []
message.parent = thread.root
thread.root.children.append(message)
return thread
def get_thread(self, _id):
if isinstance(_id, Thread):
thread = _id
id = thread.id
else:
thread = find_object(self.iter_threads(), id=_id)
id = _id
content = self.browser.get_content(id)
if content is None:
return None
if not thread:
thread = Thread(id)
flags = Message.IS_HTML
if thread.id not in self.storage.get('seen', default={}):
flags |= Message.IS_UNREAD
thread.title = content.title
if not thread.date:
thread.date = content.date
thread.root = Message(thread=thread,
id=0,
url=content.url,
title=content.title,
sender=content.author,
receivers=None,
date=thread.date,
parent=None,
content=content.body,
signature=u'<a href="%s">URL</a> \n' %
content.url,
flags=flags,
children=[])
return thread
def get_thread(self, _id):
if isinstance(_id, Thread):
thread = _id
id = thread.id
else:
thread = find_object(self.iter_threads(), id=_id)
id = _id
with self.browser:
content = self.browser.get_content(id)
if content is None:
return None
if not thread:
thread = Thread(id)
flags = Message.IS_HTML
if not thread.id in self.storage.get('seen', default={}):
flags |= Message.IS_UNREAD
thread.title = content.title
if not thread.date:
thread.date = content.date
thread.root = Message(
thread=thread,
id=0,
title=content.title,
sender=content.author,
receivers=None,
date=thread.date,
parent=None,
content=content.body,
signature= u'<a href="%s">URL</a> \n' % content.url,
flags=flags,
children=[])
return thread
def get_thread(self, id):
if isinstance(id, Thread):
thread = id
id = thread.id
else:
thread = None
with self.browser:
content = self.browser.get_content(id)
if not thread:
thread = Thread(id)
flags = Message.IS_HTML
if not thread.id in self.storage.get('seen', default={}):
flags |= Message.IS_UNREAD
thread.title = content.title
if not thread.date:
thread.date = content.date
thread.root = Message(thread=thread,
id=0, # root message
title=content.title,
sender=content.author,
receivers=None,
date=thread.date, #TODO XXX WTF this is None
parent=None,
content=''.join([content.body, content.part2]),
signature='URL: %s' % content.url,
children=[],
flags=flags)
for com in content.comments:
self._insert_comment(com, thread.root)
return thread
def get_thread(self, _id):
if isinstance(_id, Thread):
thread = _id
id = thread.id
else:
thread = self._get_thread(_id)
id = _id
with self.browser:
content = self.browser.get_content(id)
if content is None:
return None
if not thread:
thread = Thread(id)
flags = Message.IS_HTML
if not thread.id in self.storage.get('seen', default={}):
flags |= Message.IS_UNREAD
thread.title = content.title
if not thread.date:
thread.date = content.date
thread.root = Message(thread=thread,
id=0,
title=content.title,
sender=content.author,
receivers=None,
date=thread.date,
parent=None,
content=content.body,
signature='URL: %s' % content.url,
flags=flags,
children=[])
return thread
def iter_threads(self):
for article in Newsfeed(self.RSS_FEED, GenericNewspaperBackend.RSSID).iter_entries():
thread = Thread(article.id)
thread.title = article.title
thread.date = article.datetime
yield(thread)
def iter_threads(self):
for article in Newsfeed(self.RSS_FEED, self.RSSID).iter_entries():
thread = Thread(article.link)
thread.title = article.title
thread.date = article.datetime
yield(thread)
def iter_threads(self):
for article in Newsfeed(self.RSS_FEED, self.RSSID).iter_entries():
thread = Thread(article.link)
thread.title = article.title
thread.date = article.datetime
yield (thread)