def iter_unread_messages(self): with self.browser: url = self.browser.get_root_feed_url() for article in Newsfeed(url, rssid).iter_entries(): id = url2id(article.link) thread = None try: last_seen_id = self.storage.get('seen', default={})[id2topic(id)] except KeyError: last_seen_id = 0 child = None iterator = self.browser.riter_posts(id, last_seen_id) if self.config['thread_unread_messages'].get() > 0: iterator = limit(iterator, self.config['thread_unread_messages'].get()) for post in iterator: if not thread: thread = Thread('%s.%s' % (post.forum_id, post.topic_id)) message = self._post2message(thread, post) if child: message.children.append(child) child.parent = message if post.parent: message.parent = Message(thread=thread, id=post.parent) else: thread.root = message yield message
def get_thread(self, id): if isinstance(id, Thread): thread = id id = thread.id else: thread = None story = self.browser.get_story(id) if not story: return None if not thread: thread = Thread(story.id) flags = 0 if thread.id not in self.storage.get('seen', default=[]): flags |= Message.IS_UNREAD thread.title = story.title thread.date = story.date thread.root = Message(thread=thread, id=0, title=story.title, sender=story.author.name, receivers=None, date=thread.date, parent=None, content=story.body, children=[], signature=u'Written by a %s in category %s' % (self.GENDERS[story.author.sex], story.category), flags=flags) return thread
def iter_unread_messages(self): with self.browser: url = self.browser.get_root_feed_url() for article in Newsfeed(url, rssid).iter_entries(): id = url2id(article.link) thread = None try: last_seen_id = self.storage.get('seen', default={})[id2topic(id)] except KeyError: last_seen_id = 0 child = None iterator = self.browser.riter_posts(id, last_seen_id) if self.config['thread_unread_messages'].get() > 0: iterator = limit( iterator, self.config['thread_unread_messages'].get()) for post in iterator: if not thread: thread = Thread('%s.%s' % (post.forum_id, post.topic_id)) message = self._post2message(thread, post) if child: message.children.append(child) child.parent = message if post.parent: message.parent = Message(thread=thread, id=post.parent) else: thread.root = message yield message
def iter_threads(self): table = self.parser.select(self.document.getroot(), 'table#listeMessages', 1) for tr in table.xpath('./tr'): if tr.attrib.get('class', '') not in ('msgLu', 'msgNonLu'): continue author = unicode(self.parser.select(tr, 'td.colEmetteur', 1).text) link = self.parser.select(tr, 'td.colObjet a', 1) date_raw = self.parser.select(tr, 'td.colDate1', 1).attrib['data'] jsparams = re.search('\((.+)\)', link.attrib['onclick']).groups()[0] jsparams = [i.strip('\'" ') for i in jsparams.split(',')] page_id, _id, unread = jsparams # this means unread on the website unread = False if unread == "false" else True # 2012/02/29:01h30min45sec dt_match = re.match('(\d+)/(\d+)/(\d+):(\d+)h(\d+)min(\d+)sec', date_raw).groups() dt_match = [int(d) for d in dt_match] thread = Thread(_id) thread._link_id = (page_id, unread) thread.date = datetime(*dt_match) thread.title = unicode(link.text) message = Message(thread, 0) message.set_empty_fields(None) message.flags = message.IS_HTML message.title = thread.title message.date = thread.date message.sender = author message.content = NotLoaded # This is the only thing we are missing thread.root = message yield thread
def get_thread(self, id): if isinstance(id, Thread): thread = id id = thread.id else: thread = None with self.browser: story = self.browser.get_story(id) if not story: return None if not thread: thread = Thread(story.id) flags = 0 if not thread.id in self.storage.get('seen', default=[]): flags |= Message.IS_UNREAD thread.title = story.title thread.date = story.date thread.root = Message(thread=thread, id=0, title=story.title, sender=story.author.name, receivers=None, date=thread.date, parent=None, content=story.body, children=[], signature='Written by a %s (%s)' % (self.GENDERS[story.author.sex], story.author.email), flags=flags) return thread
def get_thread(self, id, entry=None): if isinstance(id, Thread): thread = id id = thread.id else: thread = Thread(id) if entry is None: entry = Newsfeed(self.config['url'].get()).get_entry(id) if entry is None: return None flags = Message.IS_HTML if not thread.id in self.storage.get('seen', default=[]): flags |= Message.IS_UNREAD if len(entry.content) > 0: content = u"<p>Link %s</p> %s" % (entry.link, entry.content[0]) else: content = entry.link thread.title = entry.title thread.root = Message(thread=thread, id=0, title=entry.title, sender=entry.author, receivers=None, date=entry.datetime, parent=None, content=content, children=[], flags=flags) return thread
def get_thread(self, id): if isinstance(id, Thread): thread = id id = thread.id else: thread = Thread(id) entry = Newsfeed(self.config["url"]).get_entry(id) flags = Message.IS_HTML if not thread.id in self.storage.get('seen', default=[]): flags |= Message.IS_UNREAD if len(entry.content): content = entry.content[0] else: content = None thread.title = entry.title thread.root = Message(thread=thread, id=0, title=entry.title, sender=entry.author, receivers=None, date=entry.datetime, parent=None, content=content, children=[], flags=flags) return thread
def get_thread(self, id, entry=None): if isinstance(id, Thread): thread = id id = thread.id else: thread = Thread(id) if entry is None: entry = Newsfeed(self.config['url'].get()).get_entry(id) if entry is None: return None flags = Message.IS_HTML if thread.id not in self.storage.get('seen', default=[]): flags |= Message.IS_UNREAD if len(entry.content) > 0: content = u"<p>Link %s</p> %s" % (entry.link, entry.content[0]) else: content = entry.link thread.title = entry.title thread.root = Message(thread=thread, id=0, title=entry.title, sender=entry.author, receivers=None, date=entry.datetime, parent=None, content=content, children=[], flags=flags) return thread
def iter_threads(self): table = self.parser.select(self.document.getroot(), "table#listeMessages", 1) for tr in table.xpath("./tr"): if tr.attrib.get("class", "") not in ("msgLu", "msgNonLu"): continue author = unicode(self.parser.select(tr, "td.colEmetteur", 1).text) link = self.parser.select(tr, "td.colObjet a", 1) date_raw = self.parser.select(tr, "td.colDate1", 1).attrib["data"] jsparams = re.search("\((.+)\)", link.attrib["onclick"]).groups()[0] jsparams = [i.strip("'\" ") for i in jsparams.split(",")] page_id, _id, unread = jsparams # this means unread on the website unread = False if unread == "false" else True # 2012/02/29:01h30min45sec dt_match = re.match("(\d+)/(\d+)/(\d+):(\d+)h(\d+)min(\d+)sec", date_raw).groups() dt_match = [int(d) for d in dt_match] thread = Thread(_id) thread._link_id = (page_id, unread) thread.date = datetime(*dt_match) thread.title = unicode(link.text) message = Message(thread, 0) message.set_empty_fields(None) message.flags = message.IS_HTML message.title = thread.title message.date = thread.date message.sender = author message.content = NotLoaded # This is the only thing we are missing thread.root = message yield thread
def get_thread(self, id): thread = None parent = None if isinstance(id, Thread): thread = id id = thread.id thread_id = url2id(id, nopost=True) or id try: last_seen_id = self.storage.get('seen', default={})[id2topic(thread_id)] except KeyError: last_seen_id = 0 with self.browser: for post in self.browser.iter_posts(id): if not thread: thread = Thread(thread_id) thread.title = post.title m = self._post2message(thread, post) m.parent = parent if last_seen_id < post.id: m.flags |= Message.IS_UNREAD if parent: parent.children = [m] else: thread.root = m parent = m return thread
def iter_unread_messages(self): try: contacts = {} with self.browser: threads = self.browser.get_threads_list() for thread in threads: #if thread['member'].get('isBan', thread['member'].get('dead', False)): # with self.browser: # self.browser.delete_thread(int(thread['member']['id'])) # continue if self.antispam and not self.antispam.check_thread(thread): self.logger.info('Skipped a spam-unread-thread from %s' % thread['who']['pseudo']) self.report_spam(thread['member']['id']) continue s**t = self._get_slut(thread['who']['id']) if parse_dt(thread['date']) > s**t['lastmsg'] or thread['status'] != s**t['status']: t = self.get_thread(thread['who']['id'], contacts, get_profiles=True) for m in t.iter_all_messages(): if m.flags & m.IS_UNREAD: yield m if not self.config['baskets'].get(): return # Send mail when someone added me in her basket. # XXX possibly race condition if a s**t adds me in her basket # between the aum.nb_new_baskets() and aum.get_baskets(). with self.browser: s**t = self._get_slut(-self.MAGIC_ID_BASKET) new_baskets = self.browser.nb_new_baskets() if new_baskets > 0: baskets = self.browser.get_baskets() my_name = self.browser.get_my_name() for basket in baskets: if parse_dt(basket['date']) <= s**t['lastmsg']: continue contact = self.get_contact(basket['who']['id']) if self.antispam and not self.antispam.check_contact(contact): self.logger.info('Skipped a spam-basket from %s' % contact.name) self.report_spam(basket['who']['id']) continue thread = Thread(int(basket['who']['id'])) thread.title = 'Basket of %s' % contact.name thread.root = Message(thread=thread, id=self.MAGIC_ID_BASKET, title=thread.title, sender=contact.name, receivers=[my_name], date=parse_dt(basket['date']), content='You are taken in her basket!', signature=contact.get_text(), children=[], flags=Message.IS_UNREAD) yield thread.root except BrowserUnavailable as e: self.logger.debug('No messages, browser is unavailable: %s' % e) pass # don't care about waiting
def _build_thread(self, data): thread = Thread("%s.%s" % (data["commentable_id"], data["id"])) thread.title = data["title"] thread.date = dateutil.parser.parse(data["created_at"]) thread.url = self.browser.thread.build(course=self.browser.course, topic=data["commentable_id"], id=data["id"]) thread.root = self._build_message(data, thread) thread._messages_count = data["comments_count"] + 1 return thread
def iter_threads(self): for msg in self.browser.iter_dates(): thread = Thread(msg.id) thread.title = msg.title thread.date = msg.date thread.root = msg msg.thread = thread yield thread
def iter_threads(self): for thread in self.browser.get_threads(): if not "person" in thread: # The account has been removed, probably because it was a # spammer. continue t = Thread(thread["_id"]) t.flags = Thread.IS_DISCUSSION t.title = u"Discussion with %s" % thread["person"]["name"] contact = self.storage.get("contacts", t.id, default={"lastmsg": 0}) birthday = parse_date(thread["person"]["birth_date"]).date() signature = u"Age: %d (%s)" % ((datetime.date.today() - birthday).days / 365.25, birthday) signature += u"\nLast ping: %s" % parse_date(thread["person"]["ping_time"]).strftime("%Y-%m-%d %H:%M:%S") signature += u"\nPhotos:\n\t%s" % "\n\t".join([photo["url"] for photo in thread["person"]["photos"]]) signature += u"\n\n%s" % thread["person"]["bio"] t.root = Message( thread=t, id=1, title=t.title, sender=unicode(thread["person"]["name"]), receivers=[self.browser.my_name], date=parse_date(thread["created_date"]), content=u"Match!", children=[], signature=signature, flags=Message.IS_UNREAD if int(contact["lastmsg"]) < 1 else 0, ) parent = t.root for msg in thread["messages"]: flags = 0 if int(contact["lastmsg"]) < msg["timestamp"]: flags = Message.IS_UNREAD msg = Message( thread=t, id=msg["timestamp"], title=t.title, sender=unicode( self.browser.my_name if msg["from"] == self.browser.my_id else thread["person"]["name"] ), receivers=[ unicode(self.browser.my_name if msg["to"] == self.browser.my_id else thread["person"]["name"]) ], date=parse_date(msg["sent_date"]), content=unicode(msg["message"]), children=[], parent=parent, signature=signature if msg["to"] == self.browser.my_id else u"", flags=flags, ) parent.children.append(msg) parent = msg yield t
def get_thread(self, id): thread = None if isinstance(id, Thread): thread = id id = thread.id if '.' not in id: self.logger.warning('Malformated ID (%s)' % id) return board, thread_id = self._splitid(id) with self.browser: _thread = self.browser.get_thread(board, thread_id) flags = 0 if _thread.id not in self.storage.get('boards', board, default={}): flags |= Message.IS_UNREAD if not thread: thread = Thread(id) thread.title = _thread.filename thread.root = Message( thread=thread, id=0, # root message title=_thread.filename, sender=_thread.author, receivers=None, date=_thread.datetime, parent=None, content=_thread.text, signature=None, children=[], flags=flags | Message.IS_HTML) for comment in _thread.comments: flags = 0 if comment.id not in self.storage.get('boards', board, _thread.id, default=[]): flags |= Message.IS_UNREAD m = Message(thread=thread, id=comment.id, title=_thread.filename, sender=comment.author, receivers=None, date=comment.datetime, parent=thread.root, content=comment.text, signature=None, children=None, flags=flags | Message.IS_HTML) thread.root.children.append(m) return thread
def get_thread(self, id): thread = None if isinstance(id, Thread): thread = id id = thread.id if not '.' in id: self.logger.warning('Malformated ID (%s)' % id) return board, thread_id = self._splitid(id) with self.browser: _thread = self.browser.get_thread(board, thread_id) flags = 0 if not _thread.id in self.storage.get('boards', board, default={}): flags |= Message.IS_UNREAD if not thread: thread = Thread(id) thread.title = _thread.filename thread.root = Message(thread=thread, id=0, # root message title=_thread.filename, sender=_thread.author, receivers=None, date=_thread.datetime, parent=None, content=_thread.text, signature=None, children=None, flags=flags|Message.IS_HTML) parent = thread.root for comment in _thread.comments: flags = 0 if not comment.id in self.storage.get('boards', board, _thread.id, default=[]): flags |= Message.IS_UNREAD m = Message(thread=thread, id=comment.id, title=_thread.filename, sender=comment.author, receivers=None, date=comment.datetime, parent=parent, content=comment.text, signature=None, children=None, flags=flags|Message.IS_HTML) parent.children = [m] parent = m return thread
def get_thread(self, thread): if not isinstance(thread, Thread): thread = Thread(thread) thread.flags = Thread.IS_DISCUSSION info = self.browser.get_thread(thread.id) for user in info['participants']: if user['user']['fb_id'] is not None: user['user']['fb'] = self.browser.get_facebook( user['user']['fb_id']) if user['user']['id'] == self.browser.my_id: me = HappnContact(user['user']) else: other = HappnContact(user['user']) thread.title = u'Discussion with %s' % other.name contact = self.storage.get( 'contacts', thread.id, default={'lastmsg_date': '1970-01-01T01:01:01+00:00'}) child = None for msg in info['messages']: flags = 0 if parse_date(contact['lastmsg_date']) < parse_date( msg['creation_date']): flags = Message.IS_UNREAD if msg['sender']['id'] == me.id: sender = me receiver = other else: sender = other receiver = me msg = Message(thread=thread, id=msg['id'], title=thread.title, sender=sender.name, receivers=[receiver.name], date=parse_date(msg['creation_date']), content=msg['message'], children=[], parent=None, signature=sender.get_text(), flags=flags) if child: msg.children.append(child) child.parent = msg child = msg thread.root = child return thread
def get_thread(self, thread): if not isinstance(thread, Thread): thread = Thread(thread) thread.flags = Thread.IS_DISCUSSION info = self.browser.get_thread(thread.id) for user in info['participants']: if user['user']['fb_id'] is not None: user['user']['fb'] = self.browser.get_facebook(user['user']['fb_id']) if user['user']['id'] == self.browser.my_id: me = HappnContact(user['user']) else: other = HappnContact(user['user']) thread.title = u'Discussion with %s' % other.name contact = self.storage.get('contacts', thread.id, default={'lastmsg_date': '1970-01-01T01:01:01+00:00'}) child = None for msg in info['messages']: flags = 0 if parse_date(contact['lastmsg_date']) < parse_date(msg['creation_date']): flags = Message.IS_UNREAD if msg['sender']['id'] == me.id: sender = me receiver = other else: sender = other receiver = me msg = Message(thread=thread, id=msg['id'], title=thread.title, sender=sender.name, receivers=[receiver.name], date=parse_date(msg['creation_date']), content=msg['message'], children=[], parent=None, signature=sender.get_text(), flags=flags) if child: msg.children.append(child) child.parent = msg child = msg thread.root = child return thread
def get_thread(self, thread): if not isinstance(thread, Thread): thread = Thread(thread) thread.flags = Thread.IS_DISCUSSION user = self.browser.get_user(thread.id) thread.title = u'Discussion with %s' % user['name'] contact = self.storage.get('contacts', thread.id, default={'lastmsg': 0}) signature = u'Age: %s' % user['age'] signature += u'\nLast online: %s' % user['last_online'] signature += u'\nPhotos:\n\t%s' % '\n\t'.join([user['photo_host'] + photo['large'] for photo in user['photos']]) child = None for msg in self.browser.get_thread_messages(thread.id): flags = 0 if int(contact['lastmsg']) < msg['utc_timestamp']: flags = Message.IS_UNREAD if msg['type'] == 'msg': content = unicode(msg['msg']) elif msg['type'] == 'new_challenge': content = u'A new challenge has been proposed!' elif msg['type'] == 'serie': content = u"I've played" elif msg['type'] == 'end_game': content = u'%s is the winner! (%s VS %s)' % (self.browser.my_name if msg['score']['w'] == self.browser.my_id else user['name'], msg['score']['s'][0], msg['score']['s'][1]) else: content = u'Unknown action: %s' % msg['type'] msg = Message(thread=thread, id=msg['utc_timestamp'], title=thread.title, sender=unicode(self.browser.my_name if msg['from'] == self.browser.my_id else user['name']), receivers=[unicode(self.browser.my_name if msg['from'] != self.browser.my_id else user['name'])], date=local2utc(datetime.datetime.fromtimestamp(msg['utc_timestamp'])), content=content, children=[], parent=None, signature=signature if msg['from'] != self.browser.my_id else u'', flags=flags) if child: msg.children.append(child) child.parent = msg child = msg thread.root = child return thread
def get_thread(self, thread): if not isinstance(thread, Thread): thread = Thread(thread) thread.flags = Thread.IS_DISCUSSION messages = self.browser.get_thread_messages(thread.id) contact = self.storage.get('s***s', thread.id, default={'lastmsg': datetime(1970, 1, 1)}) thread.title = u'Discussion with %s' % messages['fields']['username'] me = OkcContact(self.browser.get_profile(self.browser.me['userid'])) other = OkcContact(self.browser.get_profile(thread.id)) parent = None for message in messages['messages']['messages']: date = datetime.fromtimestamp(message['timestamp']) flags = 0 if contact['lastmsg'] < date: flags = Message.IS_UNREAD if message['from'] == thread.id: sender = other receiver = me else: receiver = other sender = me msg = Message(thread=thread, id=message['id'], title=thread.title, sender=sender.name, receivers=[receiver.name], date=date, content=to_unicode(unescape(message['body'])), children=[], parent=parent, signature=sender.get_text(), flags=flags) if parent: parent.children = [msg] else: thread.root = msg parent = msg return thread
def get_thread(self, id, getseen=True): if not isinstance(id, Thread): thread = None else: thread = id id = thread.id # Check if we have seen all comments of this thread. oldhash = self.storage.get('hash', id, default="") newhash = self.browser.get_hash(thread._rsscomment) if not getseen and oldhash == newhash: return None self.storage.set('hash', id, newhash) if thread.date: self.storage.set('date', id, thread.date) self.storage.save() with self.browser: content = self.browser.get_content(id) if not content: return None if not thread: thread = Thread(content.id) flags = Message.IS_HTML if not thread.id in self.storage.get('seen', default={}): flags |= Message.IS_UNREAD thread.title = content.title if not thread.date: thread.date = content.date thread.root = Message(thread=thread, id='0', # root message title=content.title, sender=content.author or u'', receivers=None, date=thread.date, parent=None, content=content.body, signature='URL: %s' % self.browser.absurl(id2url(content.id)), children=[], flags=flags) for com in content.comments: self._insert_comment(com, thread.root, getseen) return thread
def get_thread(self, thread): if not isinstance(thread, Thread): thread = Thread(thread) thread.flags = Thread.IS_DISCUSSION messages = self.browser.get_thread_messages(thread.id) contact = self.storage.get('s***s', thread.id, default={'lastmsg': datetime(1970,1,1)}) thread.title = u'Discussion with %s' % messages['fields']['username'] me = OkcContact(self.browser.get_profile(self.browser.me['userid'])) other = OkcContact(self.browser.get_profile(thread.id)) parent = None for message in messages['messages']['messages']: date = datetime.fromtimestamp(message['timestamp']) flags = 0 if contact['lastmsg'] < date: flags = Message.IS_UNREAD if message['from'] == thread.id: sender = other receiver = me else: receiver = other sender = me msg = Message(thread=thread, id=message['id'], title=thread.title, sender=sender.name, receivers=[receiver.name], date=date, content=to_unicode(unescape(message['body'])), children=[], parent=parent, signature=sender.get_text(), flags=flags) if parent: parent.children = [msg] else: thread.root = msg parent = msg return thread
def iter_threads(self): for thread in self.browser.get_threads(): if 'person' not in thread: # The account has been removed, probably because it was a # spammer. continue t = Thread(thread['_id']) t.flags = Thread.IS_DISCUSSION t.title = u'Discussion with %s' % thread['person']['name'] contact = self.storage.get('contacts', t.id, default={'lastmsg': 0}) birthday = parse_date(thread['person']['birth_date']).date() signature = u'Age: %d (%s)' % ((datetime.date.today() - birthday).days / 365.25, birthday) signature += u'\nLast ping: %s' % parse_date(thread['person']['ping_time']).strftime('%Y-%m-%d %H:%M:%S') signature += u'\nPhotos:\n\t%s' % '\n\t'.join([photo['url'] for photo in thread['person']['photos']]) signature += u'\n\n%s' % thread['person'].get('bio', '') t.root = Message(thread=t, id=1, title=t.title, sender=unicode(thread['person']['name']), receivers=[self.browser.my_name], date=parse_date(thread['created_date']), content=u'Match!', children=[], signature=signature, flags=Message.IS_UNREAD if int(contact['lastmsg']) < 1 else 0) parent = t.root for msg in thread['messages']: flags = 0 if int(contact['lastmsg']) < msg['timestamp']: flags = Message.IS_UNREAD msg = Message(thread=t, id=msg['timestamp'], title=t.title, sender=unicode(self.browser.my_name if msg['from'] == self.browser.my_id else thread['person']['name']), receivers=[unicode(self.browser.my_name if msg['to'] == self.browser.my_id else thread['person']['name'])], date=parse_date(msg['sent_date']), content=unicode(msg['message']), children=[], parent=parent, signature=signature if msg['to'] == self.browser.my_id else u'', flags=flags) parent.children.append(msg) parent = msg yield t
def iter_threads(self): for thread in self.browser.get_threads(): if 'person' not in thread: # The account has been removed, probably because it was a # spammer. continue t = Thread(thread['_id']) t.flags = Thread.IS_DISCUSSION t.title = u'Discussion with %s' % thread['person']['name'] contact = self.storage.get('contacts', t.id, default={'lastmsg': 0}) birthday = parse_date(thread['person']['birth_date']).date() signature = u'Age: %d (%s)' % ((datetime.date.today() - birthday).days / 365.25, birthday) signature += u'\nLast ping: %s' % parse_date(thread['person']['ping_time']).strftime('%Y-%m-%d %H:%M:%S') signature += u'\nPhotos:\n\t%s' % '\n\t'.join([photo['url'] for photo in thread['person']['photos']]) signature += u'\n\n%s' % thread['person']['bio'] t.root = Message(thread=t, id=1, title=t.title, sender=unicode(thread['person']['name']), receivers=[self.browser.my_name], date=parse_date(thread['created_date']), content=u'Match!', children=[], signature=signature, flags=Message.IS_UNREAD if int(contact['lastmsg']) < 1 else 0) parent = t.root for msg in thread['messages']: flags = 0 if int(contact['lastmsg']) < msg['timestamp']: flags = Message.IS_UNREAD msg = Message(thread=t, id=msg['timestamp'], title=t.title, sender=unicode(self.browser.my_name if msg['from'] == self.browser.my_id else thread['person']['name']), receivers=[unicode(self.browser.my_name if msg['to'] == self.browser.my_id else thread['person']['name'])], date=parse_date(msg['sent_date']), content=unicode(msg['message']), children=[], parent=parent, signature=signature if msg['to'] == self.browser.my_id else u'', flags=flags) parent.children.append(msg) parent = msg yield t
def iter_threads(self): inbox = self.config['inbox'].get() if not inbox: raise NotImplementedError() else: for d in self.browser.get_mails(inbox): thread = Thread(d['id']) thread.title = d['subject'] thread.flags = thread.IS_DISCUSSION msg = self.make_message(d, thread) if not msg.content: self._fetch_content(msg) thread.root = msg yield thread
def iter_threads(self): inbox = self.config['inbox'].get() if not inbox: raise NotImplementedError() else: for d in self.browser.get_mails(inbox): thread = Thread(d['id']) thread.title = d['subject'] thread.flags = thread.IS_DISCUSSION msg = self.make_message(d, thread) if not msg.content: msg.content = self.browser.get_mail_content(msg.id) thread.root = msg yield thread
def get_thread(self, id, getseen=True): if not isinstance(id, Thread): thread = None else: thread = id id = thread.id if thread.date: self.storage.set("date", id, thread.date) self.storage.save() with self.browser: content = self.browser.get_content(id) if not content: return None if not thread: thread = Thread(content.id) flags = Message.IS_HTML if not thread.id in self.storage.get("seen", default={}): flags |= Message.IS_UNREAD thread.title = content.title if not thread.date: thread.date = content.date thread.root = Message( thread=thread, id="0", # root message title=content.title, sender=content.author or u"", receivers=None, date=thread.date, parent=None, content=content.body, signature="URL: %s" % self.browser.absurl(id2url(content.id)), children=[], flags=flags, ) for com in content.comments: self._insert_comment(com, thread.root, getseen) return thread
def get_thread(self, id, getseen=True): if not isinstance(id, Thread): thread = None else: thread = id id = thread.id if thread.date: self.storage.set('date', id, thread.date) self.storage.save() content = self.browser.get_content(id) if not content: return None if not thread: thread = Thread(content.id) flags = Message.IS_HTML if thread.id not in self.storage.get('seen', default={}): flags |= Message.IS_UNREAD thread.title = content.title if not thread.date: thread.date = content.date thread.root = Message( thread=thread, id='0', # root message url=self.browser.absurl(id2url(content.id)), title=content.title, sender=content.author or u'', receivers=None, date=thread.date, parent=None, content=content.body, signature='URL: %s' % self.browser.absurl(id2url(content.id)), children=[], flags=flags) for com in content.comments: self._insert_comment(com, thread.root, getseen) return thread
def get_thread(self, _id): t = Thread(_id) t.title = 'Mail for %s' % _id t.flags = t.IS_DISCUSSION first = True for d in self.browser.get_mails(_id): m = self.make_message(d, t) if not m.content: m.content = self.browser.get_mail_content(m.id) if first: first = False t.root = m else: m.parent = t.root m.parent.children.append(m) return t
def get_thread(self, _id): thread = Thread(_id) thread.title = self.document.find('div', 'PADtitreBlanc_txt').find('center').string thread.flags = Thread.IS_DISCUSSION root = True for message in self._get_messages(thread): if root: message.children = [] thread.root = message thread.date = message.date message.title = thread.title root = False else: message.title = 'Re: %s' % thread.title message.children = [] message.parent = thread.root thread.root.children.append(message) return thread
def get_thread(self, id, entry=None): if isinstance(id, Thread): thread = id id = thread.id else: thread = Thread(id) if entry is None: url = id.split('#')[0] for article in self.browser.get_unread_feed(url): if article.id == id: entry = article if entry is None: return None if thread.id not in self.storage.get('seen', default=[]): entry.flags = Message.IS_UNREAD entry.thread = thread thread.title = entry.title thread.root = entry return thread
def get_thread(self, id, entry=None): if isinstance(id, Thread): thread = id id = thread.id else: thread = Thread(id) if entry is None: url = id.split('#')[0] for article in self.browser.get_unread_feed(url): if article.id == id: entry = article if entry is None: return None if not thread.id in self.storage.get('seen', default=[]): entry.flags = Message.IS_UNREAD entry.thread = thread thread.title = entry.title thread.root = entry return thread
def get_thread(self, _id): if isinstance(_id, Thread): thread = _id id = thread.id else: thread = find_object(self.iter_threads(), id=_id) id = _id content = self.browser.get_content(id) if content is None: return None if not thread: thread = Thread(id) flags = Message.IS_HTML if thread.id not in self.storage.get('seen', default={}): flags |= Message.IS_UNREAD thread.title = content.title if not thread.date: thread.date = content.date thread.root = Message(thread=thread, id=0, url=content.url, title=content.title, sender=content.author, receivers=None, date=thread.date, parent=None, content=content.body, signature=u'<a href="%s">URL</a> \n' % content.url, flags=flags, children=[]) return thread
def get_thread(self, _id): if isinstance(_id, Thread): thread = _id id = thread.id else: thread = find_object(self.iter_threads(), id=_id) id = _id with self.browser: content = self.browser.get_content(id) if content is None: return None if not thread: thread = Thread(id) flags = Message.IS_HTML if not thread.id in self.storage.get('seen', default={}): flags |= Message.IS_UNREAD thread.title = content.title if not thread.date: thread.date = content.date thread.root = Message( thread=thread, id=0, title=content.title, sender=content.author, receivers=None, date=thread.date, parent=None, content=content.body, signature= u'<a href="%s">URL</a> \n' % content.url, flags=flags, children=[]) return thread
def get_thread(self, id): if isinstance(id, Thread): thread = id id = thread.id else: thread = None with self.browser: content = self.browser.get_content(id) if not thread: thread = Thread(id) flags = Message.IS_HTML if not thread.id in self.storage.get('seen', default={}): flags |= Message.IS_UNREAD thread.title = content.title if not thread.date: thread.date = content.date thread.root = Message(thread=thread, id=0, # root message title=content.title, sender=content.author, receivers=None, date=thread.date, #TODO XXX WTF this is None parent=None, content=''.join([content.body, content.part2]), signature='URL: %s' % content.url, children=[], flags=flags) for com in content.comments: self._insert_comment(com, thread.root) return thread
def get_thread(self, _id): if isinstance(_id, Thread): thread = _id id = thread.id else: thread = self._get_thread(_id) id = _id with self.browser: content = self.browser.get_content(id) if content is None: return None if not thread: thread = Thread(id) flags = Message.IS_HTML if not thread.id in self.storage.get('seen', default={}): flags |= Message.IS_UNREAD thread.title = content.title if not thread.date: thread.date = content.date thread.root = Message(thread=thread, id=0, title=content.title, sender=content.author, receivers=None, date=thread.date, parent=None, content=content.body, signature='URL: %s' % content.url, flags=flags, children=[]) return thread
def get_thread(self, id, contacts=None, get_profiles=False): """ Get a thread and its messages. The 'contacts' parameters is only used for internal calls. """ thread = None if isinstance(id, Thread): thread = id id = thread.id if not thread: thread = Thread(int(id)) thread.flags = Thread.IS_DISCUSSION full = False else: full = True with self.browser: mails = self.browser.get_thread_mails(id, 100) my_name = self.browser.get_my_name() child = None msg = None s**t = self._get_slut(id) if contacts is None: contacts = {} if not thread.title: thread.title = u'Discussion with %s' % mails['who']['pseudo'] self.storage.set('s***s', int(thread.id), 'status', mails['status']) self.storage.save() for mail in mails['results']: flags = 0 if self.antispam and not self.antispam.check_mail(mail): self.logger.info('Skipped a spam-mail from %s' % mails['who']['pseudo']) self.report_spam(thread.id) break if parse_dt(mail['date']) > s**t['lastmsg']: flags |= Message.IS_UNREAD if get_profiles: if not mail['from'] in contacts: try: with self.browser: contacts[mail['from']] = self.get_contact( mail['from']) except BrowserHTTPNotFound: pass if self.antispam and mail[ 'from'] in contacts and not self.antispam.check_contact( contacts[mail['from']]): self.logger.info( 'Skipped a spam-mail-profile from %s' % mails['who']['pseudo']) self.report_spam(thread.id) break if int(mail['from']) == self.browser.my_id: if mails['remote_status'] == 'new' and msg is None: flags |= Message.IS_NOT_RECEIVED else: flags |= Message.IS_RECEIVED signature = u'' #if mail.get('src', None): # signature += u'Sent from my %s\n\n' % mail['src'] if mail['from'] in contacts: signature += contacts[mail['from']].get_text() msg = Message( thread=thread, id=int( time.strftime('%Y%m%d%H%M%S', parse_dt(mail['date']).timetuple())), title=thread.title, sender=to_unicode(my_name if int(mail['from']) == self.browser. my_id else mails['who']['pseudo']), receivers=[ to_unicode(my_name if int(mail['from']) != self.browser. my_id else mails['who']['pseudo']) ], date=parse_dt(mail['date']), content=to_unicode(unescape(mail['message'] or '').strip()), signature=signature, children=[], flags=flags) if child: msg.children.append(child) child.parent = msg child = msg if full and msg: # If we have get all the messages, replace NotLoaded with None as # parent. msg.parent = None if not full and not msg: # Perhaps there are hidden messages msg = NotLoaded thread.root = msg return thread
def get_thread(self, id, contacts=None, get_profiles=False): """ Get a thread and its messages. The 'contacts' parameters is only used for internal calls. """ thread = None if isinstance(id, Thread): thread = id id = thread.id if not thread and isinstance(id, basestring) and not id.isdigit(): for t in self.browser.get_threads_list(): if t['username'] == id: id = t['id'] break else: return None if not thread: thread = Thread(int(id)) thread.flags = Thread.IS_DISCUSSION with self.browser: mails = self.browser.get_thread_mails(id) my_name = self.browser.get_my_name() child = None msg = None s**t = self._get_slut(thread.id) if contacts is None: contacts = {} if not thread.title: thread.title = u'Discussion with %s' % mails['member']['pseudo'] for mail in mails['messages']: flags = 0 if mail['date'] > s**t['lastmsg']: flags |= Message.IS_UNREAD if get_profiles: if mail['id_from'] not in contacts: with self.browser: contacts[mail['id_from']] = self.get_contact( mail['id_from']) signature = u'' if mail.get('src', None): signature += u'Sent from my %s\n\n' % mail['src'] if contacts.get(mail['id_from'], None) is not None: signature += contacts[mail['id_from']].get_text() msg = Message(thread=thread, id=int( time.strftime('%Y%m%d%H%M%S', mail['date'].timetuple())), title=thread.title, sender=mail['id_from'], receivers=[ my_name if mail['id_from'] != my_name else mails['member']['pseudo'] ], date=mail['date'], content=mail['message'], signature=signature, children=[], flags=flags) if child: msg.children.append(child) child.parent = msg child = msg if msg: msg.parent = None thread.root = msg return thread
def get_thread(self, id, contacts=None, get_profiles=False): """ Get a thread and its messages. The 'contacts' parameters is only used for internal calls. """ thread = None if isinstance(id, Thread): thread = id id = thread.id if not thread: thread = Thread(int(id)) thread.flags = Thread.IS_DISCUSSION full = False else: full = True with self.browser: mails = self.browser.get_thread_mails(id, 100) my_name = self.browser.get_my_name() child = None msg = None s**t = self._get_slut(mails['member']['pseudo']) if contacts is None: contacts = {} if not thread.title: thread.title = u'Discussion with %s' % mails['member']['pseudo'] for mail in mails['messages']: flags = Message.IS_HTML if parse_dt(mail['date']) > s**t['lastmsg'] and mail[ 'id_from'] != self.browser.get_my_name(): flags |= Message.IS_UNREAD if get_profiles: if not mail['id_from'] in contacts: with self.browser: contacts[mail['id_from']] = self.get_contact( mail['id_from']) signature = u'' if mail.get('src', None): signature += u'Sent from my %s\n\n' % mail['src'] if mail['id_from'] in contacts: signature += contacts[mail['id_from']].get_text() msg = Message(thread=thread, id=int( time.strftime('%Y%m%d%H%M%S', parse_dt( mail['date']).timetuple())), title=thread.title, sender=mail['id_from'], receivers=[ my_name if mail['id_from'] != my_name else mails['member']['pseudo'] ], date=parse_dt(mail['date']), content=unescape(mail['message']).strip(), signature=signature, children=[], flags=flags) if child: msg.children.append(child) child.parent = msg child = msg if full and msg: # If we have get all the messages, replace NotLoaded with None as # parent. msg.parent = None if not full and not msg: # Perhaps there are hidden messages msg = NotLoaded thread.root = msg return thread
def get_thread(self, id, contacts=None, get_profiles=False): """ Get a thread and its messages. The 'contacts' parameters is only used for internal calls. """ thread = None if isinstance(id, Thread): thread = id id = thread.id if not thread: thread = Thread(int(id)) thread.flags = Thread.IS_DISCUSSION full = False else: full = True with self.browser: mails = self.browser.get_thread_mails(id, 100) my_name = self.browser.get_my_name() child = None msg = None s**t = self._get_slut(id) if contacts is None: contacts = {} if not thread.title: thread.title = u'Discussion with %s' % mails['who']['pseudo'] self.storage.set('s***s', int(thread.id), 'status', mails['status']) self.storage.save() for mail in mails['results']: flags = 0 if self.antispam and not self.antispam.check_mail(mail): self.logger.info('Skipped a spam-mail from %s' % mails['who']['pseudo']) self.report_spam(thread.id) break if parse_dt(mail['date']) > s**t['lastmsg']: flags |= Message.IS_UNREAD if get_profiles: if not mail['from'] in contacts: try: with self.browser: contacts[mail['from']] = self.get_contact(mail['from']) except BrowserHTTPNotFound: pass if self.antispam and mail['from'] in contacts and not self.antispam.check_contact(contacts[mail['from']]): self.logger.info('Skipped a spam-mail-profile from %s' % mails['who']['pseudo']) self.report_spam(thread.id) break if int(mail['from']) == self.browser.my_id: if mails['remote_status'] == 'new' and msg is None: flags |= Message.IS_NOT_RECEIVED else: flags |= Message.IS_RECEIVED signature = u'' #if mail.get('src', None): # signature += u'Sent from my %s\n\n' % mail['src'] if mail['from'] in contacts: signature += contacts[mail['from']].get_text() msg = Message(thread=thread, id=int(time.strftime('%Y%m%d%H%M%S', parse_dt(mail['date']).timetuple())), title=thread.title, sender=to_unicode(my_name if int(mail['from']) == self.browser.my_id else mails['who']['pseudo']), receivers=[to_unicode(my_name if int(mail['from']) != self.browser.my_id else mails['who']['pseudo'])], date=parse_dt(mail['date']), content=to_unicode(unescape(mail['message'] or '').strip()), signature=signature, children=[], flags=flags) if child: msg.children.append(child) child.parent = msg child = msg if full and msg: # If we have get all the messages, replace NotLoaded with None as # parent. msg.parent = None if not full and not msg: # Perhaps there are hidden messages msg = NotLoaded thread.root = msg return thread
def get_thread(self, id, contacts=None, get_profiles=False): """ Get a thread and its messages. The 'contacts' parameters is only used for internal calls. """ thread = None if isinstance(id, Thread): thread = id id = thread.id if not thread and isinstance(id, basestring) and not id.isdigit(): for t in self.browser.get_threads_list(): if t['username'] == id: id = t['id'] break else: return None if not thread: thread = Thread(int(id)) thread.flags = Thread.IS_DISCUSSION with self.browser: mails = self.browser.get_thread_mails(id) my_name = self.browser.get_my_name() child = None msg = None s**t = self._get_slut(thread.id) if contacts is None: contacts = {} if not thread.title: thread.title = u'Discussion with %s' % mails['member']['pseudo'] for mail in mails['messages']: flags = 0 if mail['date'] > s**t['lastmsg']: flags |= Message.IS_UNREAD if get_profiles: if not mail['id_from'] in contacts: with self.browser: contacts[mail['id_from']] = self.get_contact(mail['id_from']) signature = u'' if mail.get('src', None): signature += u'Sent from my %s\n\n' % mail['src'] if contacts.get(mail['id_from'], None) is not None: signature += contacts[mail['id_from']].get_text() msg = Message(thread=thread, id=int(time.strftime('%Y%m%d%H%M%S', mail['date'].timetuple())), title=thread.title, sender=mail['id_from'], receivers=[my_name if mail['id_from'] != my_name else mails['member']['pseudo']], date=mail['date'], content=mail['message'], signature=signature, children=[], flags=flags) if child: msg.children.append(child) child.parent = msg child = msg if msg: msg.parent = None thread.root = msg return thread
def iter_unread_messages(self): try: contacts = {} with self.browser: threads = self.browser.get_threads_list() for thread in threads: #if thread['member'].get('isBan', thread['member'].get('dead', False)): # with self.browser: # self.browser.delete_thread(int(thread['member']['id'])) # continue if self.antispam and not self.antispam.check_thread(thread): self.logger.info('Skipped a spam-unread-thread from %s' % thread['who']['pseudo']) self.report_spam(thread['member']['id']) continue s**t = self._get_slut(thread['who']['id']) if parse_dt( thread['date'] ) > s**t['lastmsg'] or thread['status'] != s**t['status']: t = self.get_thread(thread['who']['id'], contacts, get_profiles=True) for m in t.iter_all_messages(): if m.flags & m.IS_UNREAD: yield m if not self.config['baskets'].get(): return # Send mail when someone added me in her basket. # XXX possibly race condition if a s**t adds me in her basket # between the aum.nb_new_baskets() and aum.get_baskets(). with self.browser: s**t = self._get_slut(-self.MAGIC_ID_BASKET) new_baskets = self.browser.nb_new_baskets() if new_baskets > 0: baskets = self.browser.get_baskets() my_name = self.browser.get_my_name() for basket in baskets: if parse_dt(basket['date']) <= s**t['lastmsg']: continue contact = self.get_contact(basket['who']['id']) if self.antispam and not self.antispam.check_contact( contact): self.logger.info('Skipped a spam-basket from %s' % contact.name) self.report_spam(basket['who']['id']) continue thread = Thread(int(basket['who']['id'])) thread.title = 'Basket of %s' % contact.name thread.root = Message( thread=thread, id=self.MAGIC_ID_BASKET, title=thread.title, sender=contact.name, receivers=[my_name], date=parse_dt(basket['date']), content='You are taken in her basket!', signature=contact.get_text(), children=[], flags=Message.IS_UNREAD) yield thread.root except BrowserUnavailable as e: self.logger.debug('No messages, browser is unavailable: %s' % e) pass # don't care about waiting
def get_thread(self, id, contacts=None, get_profiles=False): """ Get a thread and its messages. The 'contacts' parameters is only used for internal calls. """ thread = None if isinstance(id, Thread): thread = id id = thread.id if not thread: thread = Thread(int(id)) thread.flags = Thread.IS_DISCUSSION full = False else: full = True with self.browser: mails = self.browser.get_thread_mails(id, 100) my_name = self.browser.get_my_name() child = None msg = None s**t = self._get_slut(mails["member"]["pseudo"]) if contacts is None: contacts = {} if not thread.title: thread.title = u"Discussion with %s" % mails["member"]["pseudo"] for mail in mails["messages"]: flags = Message.IS_HTML if parse_dt(mail["date"]) > s**t["lastmsg"] and mail["id_from"] != self.browser.get_my_name(): flags |= Message.IS_UNREAD if get_profiles: if not mail["id_from"] in contacts: with self.browser: contacts[mail["id_from"]] = self.get_contact(mail["id_from"]) signature = u"" if mail.get("src", None): signature += u"Sent from my %s\n\n" % mail["src"] if mail["id_from"] in contacts: signature += contacts[mail["id_from"]].get_text() msg = Message( thread=thread, id=int(time.strftime("%Y%m%d%H%M%S", parse_dt(mail["date"]).timetuple())), title=thread.title, sender=mail["id_from"], receivers=[my_name if mail["id_from"] != my_name else mails["member"]["pseudo"]], date=parse_dt(mail["date"]), content=unescape(mail["message"]).strip(), signature=signature, children=[], flags=flags, ) if child: msg.children.append(child) child.parent = msg child = msg if full and msg: # If we have get all the messages, replace NotLoaded with None as # parent. msg.parent = None if not full and not msg: # Perhaps there are hidden messages msg = NotLoaded thread.root = msg return thread
def get_thread(self, thread): if not isinstance(thread, Thread): thread = Thread(thread) thread.flags = Thread.IS_DISCUSSION messages = self.browser.get_thread_messages(thread.id) contact = self.storage.get('s***s', thread.id, default={'lastmsg': datetime(1970, 1, 1)}) thread.title = u'Discussion with %s' % messages['fields']['username'] me = OkcContact(self.browser.get_profile(self.browser.me['userid'])) other = OkcContact(self.browser.get_profile(thread.id)) parent = None for message in messages['messages']: date = datetime.fromtimestamp(message['timestamp']) flags = 0 if contact['lastmsg'] < date: flags = Message.IS_UNREAD if message['from'] == thread.id: sender = other receiver = me else: receiver = other sender = me if message.get('read', False): flags |= Message.IS_RECEIVED # Apply that flag on all previous messages as the 'read' # attribute is only set on the last read message. pmsg = parent while pmsg: if pmsg.flags & Message.IS_NOT_RECEIVED: pmsg.flags |= Message.IS_RECEIVED pmsg.flags &= ~Message.IS_NOT_RECEIVED pmsg = pmsg.parent else: flags |= Message.IS_NOT_RECEIVED msg = Message(thread=thread, id=message['id'], title=thread.title, sender=sender.name, receivers=[receiver.name], date=date, content=to_unicode(HTMLParser().unescape( message['body'])), children=[], parent=parent, signature=sender.get_text(), flags=flags) if parent: parent.children = [msg] else: thread.root = msg parent = msg return thread