def _process_post(self, thread, post, is_root): # Tapatalk app seems to have hardcoded this construction... I don't think we can do better :( url = u'%s/index.php?/topic/%s-%s#entry%s' % ( self.config["url"].get().rstrip('/'), thread.id, re.sub(r'[^a-zA-Z0-9-]', '', re.sub(r'\s+', '-', thread.title)), post['post_id'] ) message = Message( id = is_root and "0" or str(post['post_id']), thread = thread, sender = unicode(str(post.get('post_author_name', 'Anonymous')), 'utf-8'), title = is_root and thread.title or u"Re: %s"%thread.title, url = url, receivers = None, date = self._get_time(post), content = self._format_content(post),#bbcode(), signature = None, parent = thread.root or None, children = [], flags = Message.IS_HTML) if thread.root: thread.root.children.append(message) elif is_root: thread.root = message else: # First message in the thread is not the root message, # because we asked only for unread messages. Create a non-loaded root # message to allow monboob to fill correctly the References: header thread.root = Message(id="0", parent=None, children=[message], thread=thread) message.parent = thread.root return message
def _insert_comment(self, com, parent, getseen=True): """" Insert 'com' comment and its children in the parent message. """ flags = Message.IS_HTML if com.id not in self.storage.get('seen', parent.thread.id, 'comments', default=[]): flags |= Message.IS_UNREAD if getseen or flags & Message.IS_UNREAD: com.parse() message = Message(thread=parent.thread, id=com.id, url=com.url, title=com.title, sender=com.author or u'', receivers=None, date=com.date, parent=parent, content=com.body, signature=com.signature + '<br />'.join(['Score: %d' % com.score, 'URL: %s' % com.url]), children=[], flags=flags) else: message = Message(thread=parent.thread, id=com.id, children=[], parent=parent, flags=flags) parent.children.append(message) for sub in com.comments: self._insert_comment(sub, message, getseen)
def get_thread(self, id): thread = None if isinstance(id, Thread): thread = id id = thread.id if '.' not in id: self.logger.warning('Malformated ID (%s)' % id) return board, thread_id = self._splitid(id) with self.browser: _thread = self.browser.get_thread(board, thread_id) flags = 0 if _thread.id not in self.storage.get('boards', board, default={}): flags |= Message.IS_UNREAD if not thread: thread = Thread(id) thread.title = _thread.filename thread.root = Message( thread=thread, id=0, # root message title=_thread.filename, sender=_thread.author, receivers=None, date=_thread.datetime, parent=None, content=_thread.text, signature=None, children=[], flags=flags | Message.IS_HTML) for comment in _thread.comments: flags = 0 if comment.id not in self.storage.get('boards', board, _thread.id, default=[]): flags |= Message.IS_UNREAD m = Message(thread=thread, id=comment.id, title=_thread.filename, sender=comment.author, receivers=None, date=comment.datetime, parent=thread.root, content=comment.text, signature=None, children=None, flags=flags | Message.IS_HTML) thread.root.children.append(m) return thread
def do_post(self, line): """ post RECEIVER@BACKEND[,RECEIVER@BACKEND[...]] [TEXT] Post a message to the specified receivers. Multiple receivers are separated by a comma. If no text is supplied on command line, the content of message is read on stdin. """ receivers, text = self.parse_command_args(line, 2, 1) if text is None: text = self.acquire_input() if not self.options.accept_empty and not text.strip(): self.logger.warning( u'The message body is empty, use option --accept_empty to send empty messages' ) return for receiver in receivers.strip().split(','): receiver, backend_name = self.parse_id(receiver.strip(), unique_backend=True) if not backend_name and len(self.enabled_backends) > 1: self.logger.warning( u'No backend specified for receiver "%s": message will be sent with all the ' 'enabled backends (%s)' % (receiver, ','.join(backend.name for backend in self.enabled_backends))) if '.' in receiver: # It's a reply thread_id, parent_id = receiver.rsplit('.', 1) else: # It's an original message thread_id = receiver parent_id = None try: thread_id = self.threads[int(thread_id) - 1].id except (IndexError, ValueError): pass thread = Thread(thread_id) message = Message( thread, 0, title=self.options.title, parent=Message(thread, parent_id) if parent_id else None, content=text) try: self.do('post_message', message, backends=backend_name).wait() except CallErrors as errors: self.bcall_errors_handler(errors) else: if self.interactive: print('Message sent sucessfully to %s' % receiver)
def iter_threads(self): table = self.parser.select(self.document.getroot(), 'table#listeMessages', 1) for tr in table.xpath('./tr'): if tr.attrib.get('class', '') not in ('msgLu', 'msgNonLu'): continue author = unicode(self.parser.select(tr, 'td.colEmetteur', 1).text) link = self.parser.select(tr, 'td.colObjet a', 1) date_raw = self.parser.select(tr, 'td.colDate1', 1).attrib['data'] jsparams = re.search('\((.+)\)', link.attrib['onclick']).groups()[0] jsparams = [i.strip('\'" ') for i in jsparams.split(',')] page_id, _id, unread = jsparams # this means unread on the website unread = False if unread == "false" else True # 2012/02/29:01h30min45sec dt_match = re.match('(\d+)/(\d+)/(\d+):(\d+)h(\d+)min(\d+)sec', date_raw).groups() dt_match = [int(d) for d in dt_match] thread = Thread(_id) thread._link_id = (page_id, unread) thread.date = datetime(*dt_match) thread.title = unicode(link.text) message = Message(thread, 0) message.set_empty_fields(None) message.flags = message.IS_HTML message.title = thread.title message.date = thread.date message.sender = author message.content = NotLoaded # This is the only thing we are missing thread.root = message yield thread
def iter_threads(self): table = self.parser.select(self.document.getroot(), "table#listeMessages", 1) for tr in table.xpath("./tr"): if tr.attrib.get("class", "") not in ("msgLu", "msgNonLu"): continue author = unicode(self.parser.select(tr, "td.colEmetteur", 1).text) link = self.parser.select(tr, "td.colObjet a", 1) date_raw = self.parser.select(tr, "td.colDate1", 1).attrib["data"] jsparams = re.search("\((.+)\)", link.attrib["onclick"]).groups()[0] jsparams = [i.strip("'\" ") for i in jsparams.split(",")] page_id, _id, unread = jsparams # this means unread on the website unread = False if unread == "false" else True # 2012/02/29:01h30min45sec dt_match = re.match("(\d+)/(\d+)/(\d+):(\d+)h(\d+)min(\d+)sec", date_raw).groups() dt_match = [int(d) for d in dt_match] thread = Thread(_id) thread._link_id = (page_id, unread) thread.date = datetime(*dt_match) thread.title = unicode(link.text) message = Message(thread, 0) message.set_empty_fields(None) message.flags = message.IS_HTML message.title = thread.title message.date = thread.date message.sender = author message.content = NotLoaded # This is the only thing we are missing thread.root = message yield thread
def make_message(self, d, thread): m = Message(thread, d['id']) m.children = [] m.sender = d['from'] m.flags = 0 if not d.get('read', True): m.flags = m.IS_UNREAD m.title = d['subject'] m.date = d['datetime'] m.receivers = [d['to']] return m
def iter_threads(self): for thread in self.browser.get_threads(): if 'person' not in thread: # The account has been removed, probably because it was a # spammer. continue t = Thread(thread['_id']) t.flags = Thread.IS_DISCUSSION t.title = u'Discussion with %s' % thread['person']['name'] contact = self.storage.get('contacts', t.id, default={'lastmsg': 0}) birthday = parse_date(thread['person']['birth_date']).date() signature = u'Age: %d (%s)' % ((datetime.date.today() - birthday).days / 365.25, birthday) signature += u'\nLast ping: %s' % parse_date(thread['person']['ping_time']).strftime('%Y-%m-%d %H:%M:%S') signature += u'\nPhotos:\n\t%s' % '\n\t'.join([photo['url'] for photo in thread['person']['photos']]) signature += u'\n\n%s' % thread['person']['bio'] t.root = Message(thread=t, id=1, title=t.title, sender=unicode(thread['person']['name']), receivers=[self.browser.my_name], date=parse_date(thread['created_date']), content=u'Match!', children=[], signature=signature, flags=Message.IS_UNREAD if int(contact['lastmsg']) < 1 else 0) parent = t.root for msg in thread['messages']: flags = 0 if int(contact['lastmsg']) < msg['timestamp']: flags = Message.IS_UNREAD msg = Message(thread=t, id=msg['timestamp'], title=t.title, sender=unicode(self.browser.my_name if msg['from'] == self.browser.my_id else thread['person']['name']), receivers=[unicode(self.browser.my_name if msg['to'] == self.browser.my_id else thread['person']['name'])], date=parse_date(msg['sent_date']), content=unicode(msg['message']), children=[], parent=parent, signature=signature if msg['to'] == self.browser.my_id else u'', flags=flags) parent.children.append(msg) parent = msg yield t
def get_thread(self, id): if isinstance(id, Thread): thread = id id = thread.id else: thread = None story = self.browser.get_story(id) if not story: return None if not thread: thread = Thread(story.id) flags = 0 if thread.id not in self.storage.get('seen', default=[]): flags |= Message.IS_UNREAD thread.title = story.title thread.date = story.date thread.root = Message(thread=thread, id=0, title=story.title, sender=story.author.name, receivers=None, date=thread.date, parent=None, content=story.body, children=[], signature=u'Written by a %s in category %s' % (self.GENDERS[story.author.sex], story.category), flags=flags) return thread
def get_thread(self, _id, thread=None, seen=None): splitted_id = _id.split('#') if not thread: thread = self.thread_page.go(_id=splitted_id[1].split('.')[-1], user=splitted_id[0]).get_thread(obj=thread) title_content = thread.title.split('\n\t')[-1] thread.root = Message(thread=thread, id=splitted_id[1].split('.')[-1], title=title_content[:50] if len(title_content) > 50 else title_content, sender=splitted_id[0], receivers=None, date=thread.date, parent=thread.root, content=title_content, signature=u'', children=[] ) if seen and (_id not in seen): thread.root.flags = Message.IS_UNREAD comments = self.thread_page.stay_or_go(_id=splitted_id[1].split('.')[-1], user=splitted_id[0]).iter_comments() for comment in comments: comment.thread = thread comment.parent = thread.root if seen and comment.id not in seen.keys(): comment.flags = Message.IS_UNREAD thread.root.children.append(comment) return thread
def get_thread(self, id, entry=None): if isinstance(id, Thread): thread = id id = thread.id else: thread = Thread(id) if entry is None: entry = Newsfeed(self.config['url'].get()).get_entry(id) if entry is None: return None flags = Message.IS_HTML if thread.id not in self.storage.get('seen', default=[]): flags |= Message.IS_UNREAD if len(entry.content) > 0: content = u"<p>Link %s</p> %s" % (entry.link, entry.content[0]) else: content = entry.link thread.title = entry.title thread.root = Message(thread=thread, id=0, title=entry.title, sender=entry.author, receivers=None, date=entry.datetime, parent=None, content=content, children=[], flags=flags) return thread
def iter_unread_messages(self): with self.browser: url = self.browser.get_root_feed_url() for article in Newsfeed(url, rssid).iter_entries(): id = url2id(article.link) thread = None try: last_seen_id = self.storage.get('seen', default={})[id2topic(id)] except KeyError: last_seen_id = 0 child = None iterator = self.browser.riter_posts(id, last_seen_id) if self.config['thread_unread_messages'].get() > 0: iterator = limit( iterator, self.config['thread_unread_messages'].get()) for post in iterator: if not thread: thread = Thread('%s.%s' % (post.forum_id, post.topic_id)) message = self._post2message(thread, post) if child: message.children.append(child) child.parent = message if post.parent: message.parent = Message(thread=thread, id=post.parent) else: thread.root = message yield message
def _sendPressed(self): if not self.ui.replyWidget.isVisible(): return text = unicode(self.ui.replyEdit.toPlainText()) title = unicode(self.ui.titleEdit.text()) self.ui.backendsList.setEnabled(False) self.ui.threadsList.setEnabled(False) self.ui.messagesTree.setEnabled(False) self.ui.replyButton.setEnabled(False) self.ui.replyWidget.setEnabled(False) self.ui.sendButton.setText(self.tr('Sending...')) flags = 0 if self.ui.htmlBox.currentIndex() == 0: flags = Message.IS_HTML m = Message(thread=self.thread, id=0, title=title, sender=None, receivers=None, content=text, parent=self.message, flags=flags) self.process_reply = QtDo(self.weboob, self._postReply_cb, self._postReply_eb) self.process_reply.do('post_message', m, backends=self.thread.backend)
def make_message(self, d, thread): msg = Message(thread, d['id']) msg.children = [] msg.sender = d['from'] msg.flags = 0 msg.title = d['subject'] msg.date = d['datetime'] msg.receivers = [d['to']] return msg
def build_article(self, j): m = self.article.match(j['url']) ret = Message(id=m.group('title')) ret.title = j['title'] ret.url = j['url'] ret.flags = Message.IS_HTML ret.date = date(int(m.group('year')), int(m.group('month')), 1) ret.children = [] ret._type = 'article' return ret
def _get_messages(self, thread): thread_div = self.document.find(True, "PADpost_txt") used_ids = set() rcpt = self.document.find("input", attrs={"type": "hidden", "name": "Dest"})["value"] sender_to_receiver = {rcpt: self.browser.username, self.browser.username: rcpt} # site is sorted from latest to oldest message for message_table in reversed(thread_div.findAll("table")): for td in message_table.findAll("td"): profile_a = td.find("a", href=re.compile(r"profil_read.php\?.*")) if not profile_a: continue first_br = td.find("br") assert first_br.nextSibling.name == "br" text_nodes = ovsparse.all_next_siblings(first_br.nextSibling.nextSibling) # TODO # ~ print text_nodes # date will be used as id sitedate = profile_a.findParent("div").find(text=re.compile(",.*")).replace(", ", "") sysdate = parse_french_date(sitedate) compactdate = datetime.datetime.strftime(sysdate, "%Y%m%dT%H%M%S") # but make it unique msg_id = ovsparse.create_unique_id(compactdate, used_ids) used_ids.add(msg_id) message = Message(thread, msg_id) message.sender = re.search(r"\?(.+)", profile_a["href"]).group(1) message.receivers = [sender_to_receiver[message.sender]] message.date = sysdate message.content = ovsparse.html_message_to_text(text_nodes) notread_self = bool(td.find("span", "ColorSurligne")) notread_other = bool(td.find("span", "new_sortiepartenaire")) if notread_other or notread_self: message.flags |= Message.IS_NOT_RECEIVED else: message.flags |= Message.IS_RECEIVED yield message
def get_thread(self, thread): if not isinstance(thread, Thread): thread = Thread(thread) thread.flags = Thread.IS_DISCUSSION info = self.browser.get_thread(thread.id) for user in info['participants']: if user['user']['fb_id'] is not None: user['user']['fb'] = self.browser.get_facebook( user['user']['fb_id']) if user['user']['id'] == self.browser.my_id: me = HappnContact(user['user']) else: other = HappnContact(user['user']) thread.title = u'Discussion with %s' % other.name contact = self.storage.get( 'contacts', thread.id, default={'lastmsg_date': '1970-01-01T01:01:01+00:00'}) child = None for msg in info['messages']: flags = 0 if parse_date(contact['lastmsg_date']) < parse_date( msg['creation_date']): flags = Message.IS_UNREAD if msg['sender']['id'] == me.id: sender = me receiver = other else: sender = other receiver = me msg = Message(thread=thread, id=msg['id'], title=thread.title, sender=sender.name, receivers=[receiver.name], date=parse_date(msg['creation_date']), content=msg['message'], children=[], parent=None, signature=sender.get_text(), flags=flags) if child: msg.children.append(child) child.parent = msg child = msg thread.root = child return thread
def build_date(self, k): ret = Message(id='%s.%s' % k) ret.title = '%s/%s' % k ret.content = '' ret.date = date(int(k[0]), int(k[1]), 1) ret._type = 'date' ret._key = k return ret
def postReply(self): text = unicode(self.ui.textEdit.toPlainText()) self.ui.textEdit.setEnabled(False) self.ui.sendButton.setEnabled(False) m = Message(thread=self.thread, id=0, title=u'', sender=None, receivers=None, content=text, parent=self.messages[0].message if len(self.messages) > 0 else None) self.process_reply = QtDo(self.weboob, self._postReply_cb, self._postReply_eb) self.process_reply.do('post_message', m, backends=self.contact.backend)
def get_thread(self, thread): if not isinstance(thread, Thread): thread = Thread(thread) thread.flags = Thread.IS_DISCUSSION user = self.browser.get_user(thread.id) thread.title = u'Discussion with %s' % user['name'] contact = self.storage.get('contacts', thread.id, default={'lastmsg': 0}) signature = u'Age: %s' % user['age'] signature += u'\nLast online: %s' % user['last_online'] signature += u'\nPhotos:\n\t%s' % '\n\t'.join([user['photo_host'] + photo['large'] for photo in user['photos']]) child = None for msg in self.browser.get_thread_messages(thread.id): flags = 0 if int(contact['lastmsg']) < msg['utc_timestamp']: flags = Message.IS_UNREAD if msg['type'] == 'msg': content = unicode(msg['msg']) elif msg['type'] == 'new_challenge': content = u'A new challenge has been proposed!' elif msg['type'] == 'serie': content = u"I've played" elif msg['type'] == 'end_game': content = u'%s is the winner! (%s VS %s)' % (self.browser.my_name if msg['score']['w'] == self.browser.my_id else user['name'], msg['score']['s'][0], msg['score']['s'][1]) else: content = u'Unknown action: %s' % msg['type'] msg = Message(thread=thread, id=msg['utc_timestamp'], title=thread.title, sender=unicode(self.browser.my_name if msg['from'] == self.browser.my_id else user['name']), receivers=[unicode(self.browser.my_name if msg['from'] != self.browser.my_id else user['name'])], date=local2utc(datetime.datetime.fromtimestamp(msg['utc_timestamp'])), content=content, children=[], parent=None, signature=signature if msg['from'] != self.browser.my_id else u'', flags=flags) if child: msg.children.append(child) child.parent = msg child = msg thread.root = child return thread
def get_thread(self, thread): if not isinstance(thread, Thread): thread = Thread(thread) thread.flags = Thread.IS_DISCUSSION messages = self.browser.get_thread_messages(thread.id) contact = self.storage.get('s***s', thread.id, default={'lastmsg': datetime(1970, 1, 1)}) thread.title = u'Discussion with %s' % messages['fields']['username'] me = OkcContact(self.browser.get_profile(self.browser.me['userid'])) other = OkcContact(self.browser.get_profile(thread.id)) parent = None for message in messages['messages']['messages']: date = datetime.fromtimestamp(message['timestamp']) flags = 0 if contact['lastmsg'] < date: flags = Message.IS_UNREAD if message['from'] == thread.id: sender = other receiver = me else: receiver = other sender = me msg = Message(thread=thread, id=message['id'], title=thread.title, sender=sender.name, receivers=[receiver.name], date=date, content=to_unicode(unescape(message['body'])), children=[], parent=parent, signature=sender.get_text(), flags=flags) if parent: parent.children = [msg] else: thread.root = msg parent = msg return thread
def get_thread(self, id, getseen=True): if not isinstance(id, Thread): thread = None else: thread = id id = thread.id # Check if we have seen all comments of this thread. oldhash = self.storage.get('hash', id, default="") newhash = self.browser.get_hash(thread._rsscomment) if not getseen and oldhash == newhash: return None self.storage.set('hash', id, newhash) if thread.date: self.storage.set('date', id, thread.date) self.storage.save() with self.browser: content = self.browser.get_content(id) if not content: return None if not thread: thread = Thread(content.id) flags = Message.IS_HTML if not thread.id in self.storage.get('seen', default={}): flags |= Message.IS_UNREAD thread.title = content.title if not thread.date: thread.date = content.date thread.root = Message(thread=thread, id='0', # root message title=content.title, sender=content.author or u'', receivers=None, date=thread.date, parent=None, content=content.body, signature='URL: %s' % self.browser.absurl(id2url(content.id)), children=[], flags=flags) for com in content.comments: self._insert_comment(com, thread.root, getseen) return thread
def _process_post(self, thread, post, is_root): # Tapatalk app seems to have hardcoded this construction... I don't think we can do better :( url = u'%s/index.php?/topic/%s-%s#entry%s' % ( self.config["url"].get().rstrip('/'), thread.id, re.sub(r'[^a-zA-Z0-9-]', '', re.sub( r'\s+', '-', thread.title)), post['post_id']) message = Message( id=is_root and "0" or str(post['post_id']), thread=thread, sender=unicode(str(post.get('post_author_name', 'Anonymous')), 'utf-8'), title=is_root and thread.title or u"Re: %s" % thread.title, url=url, receivers=None, date=self._get_time(post), content=self._format_content(post), #bbcode(), signature=None, parent=thread.root or None, children=[], flags=Message.IS_HTML) if thread.root: thread.root.children.append(message) elif is_root: thread.root = message else: # First message in the thread is not the root message, # because we asked only for unread messages. Create a non-loaded root # message to allow monboob to fill correctly the References: header thread.root = Message(id="0", parent=None, children=[message], thread=thread) message.parent = thread.root return message
def _process_post(self, thread, post, is_root): message_id = is_root and u'0' or xmlrpc_str(post['post_id']) message_title = is_root and thread.title or u'Re: %s' % thread.title # Tapatalk app seems to have hardcoded this construction... I don't think we can do better :( rel_url = self.config['message_url_format'].get().format( thread_id = urllib.parse.quote(thread.id.encode('utf-8')), thread_title = urllib.parse.quote(thread.title.encode('utf-8')), message_id = urllib.parse.quote(message_id.encode('utf-8')), message_title = urllib.parse.quote(message_title.encode('utf-8'))) message = Message( id = message_id, thread = thread, sender = xmlrpc_str(post.get('post_author_name', u'Anonymous')), title = message_title, url = urllib.parse.urljoin(self.config['url'].get(), rel_url), receivers = None, date = self._get_time(post), content = self._format_content(post), signature = None, parent = thread.root or None, children = [], flags = Message.IS_HTML) if thread.root: thread.root.children.append(message) elif is_root: thread.root = message else: # First message in the thread is not the root message, # because we asked only for unread messages. Create a non-loaded root # message to allow monboob to fill correctly the References: header thread.root = Message(id=u'0', parent=None, children=[message], thread=thread) message.parent = thread.root return message
def _post2message(self, thread, post): signature = post.signature if signature: signature += '<br />' signature += 'URL: %s' % self.browser.absurl(id2url('%s.%s' % (thread.id, post.id))) return Message(thread=thread, id=post.id, title=post.title, sender=post.author, receivers=None, date=post.date, parent=None, content=post.content, signature=signature, children=[], flags=Message.IS_HTML)
def _build_message(self, data, thread, parent=None): flags = self.default_flags if data["id"] not in self.storage.get("seen", thread.id, default=[]): flags |= Message.IS_UNREAD message = Message(thread=thread, id=data["id"], title=(parent and "Re: %s" or "%s") % thread.title, sender=data.get("username"), receivers=None, date=dateutil.parser.parse(data["created_at"]), content=self.formatter(data["body"]), flags=flags, parent=parent, url=thread.url) self._append_children(data, message, thread) return message
def get_thread(self, id, getseen=True): if not isinstance(id, Thread): thread = None else: thread = id id = thread.id if thread.date: self.storage.set('date', id, thread.date) self.storage.save() content = self.browser.get_content(id) if not content: return None if not thread: thread = Thread(content.id) flags = Message.IS_HTML if thread.id not in self.storage.get('seen', default={}): flags |= Message.IS_UNREAD thread.title = content.title if not thread.date: thread.date = content.date thread.root = Message( thread=thread, id='0', # root message url=self.browser.absurl(id2url(content.id)), title=content.title, sender=content.author or u'', receivers=None, date=thread.date, parent=None, content=content.body, signature='URL: %s' % self.browser.absurl(id2url(content.id)), children=[], flags=flags) for com in content.comments: self._insert_comment(com, thread.root, getseen) return thread
def get_thread(self, _id): if isinstance(_id, Thread): thread = _id id = thread.id else: thread = find_object(self.iter_threads(), id=_id) id = _id content = self.browser.get_content(id) if content is None: return None if not thread: thread = Thread(id) flags = Message.IS_HTML if thread.id not in self.storage.get('seen', default={}): flags |= Message.IS_UNREAD thread.title = content.title if not thread.date: thread.date = content.date thread.root = Message(thread=thread, id=0, url=content.url, title=content.title, sender=content.author, receivers=None, date=thread.date, parent=None, content=content.body, signature=u'<a href="%s">URL</a> \n' % content.url, flags=flags, children=[]) return thread
def _get_messages(self, thread): thread_div = self.document.find(True, 'PADpost_txt') used_ids = set() rcpt = self.document.find('input', attrs={'type': 'hidden', 'name': 'Dest'})['value'] sender_to_receiver = {rcpt: self.browser.username, self.browser.username: rcpt} # site is sorted from latest to oldest message for message_table in reversed(thread_div.findAll('table')): for td in message_table.findAll('td'): profile_a = td.find('a', href=re.compile(r'profil_read.php\?.*')) if not profile_a: continue first_br = td.find('br') assert first_br.nextSibling.name == 'br' text_nodes = ovsparse.all_next_siblings(first_br.nextSibling.nextSibling) # TODO #~ print text_nodes # date will be used as id sitedate = profile_a.findParent('div').find(text=re.compile(',.*')).replace(', ', '') sysdate = parse_french_date(sitedate) compactdate = datetime.datetime.strftime(sysdate, '%Y%m%dT%H%M%S') # but make it unique msg_id = ovsparse.create_unique_id(compactdate, used_ids) used_ids.add(msg_id) message = Message(thread, msg_id) message.sender = re.search(r'\?(.+)', profile_a['href']).group(1) message.receivers = [sender_to_receiver[message.sender]] message.date = sysdate message.content = ovsparse.html_message_to_text(text_nodes) notread_self = bool(td.find('span', 'ColorSurligne')) notread_other = bool(td.find('span', 'new_sortiepartenaire')) if notread_other or notread_self: message.flags |= Message.IS_NOT_RECEIVED else: message.flags |= Message.IS_RECEIVED yield message
def get_thread(self, _id): if isinstance(_id, Thread): thread = _id id = thread.id else: thread = self._get_thread(_id) id = _id with self.browser: content = self.browser.get_content(id) if content is None: return None if not thread: thread = Thread(id) flags = Message.IS_HTML if not thread.id in self.storage.get('seen', default={}): flags |= Message.IS_UNREAD thread.title = content.title if not thread.date: thread.date = content.date thread.root = Message(thread=thread, id=0, title=content.title, sender=content.author, receivers=None, date=thread.date, parent=None, content=content.body, signature='URL: %s' % content.url, flags=flags, children=[]) return thread
def get_thread(self, id, contacts=None, get_profiles=False): """ Get a thread and its messages. The 'contacts' parameters is only used for internal calls. """ thread = None if isinstance(id, Thread): thread = id id = thread.id if not thread: thread = Thread(int(id)) thread.flags = Thread.IS_DISCUSSION full = False else: full = True with self.browser: mails = self.browser.get_thread_mails(id, 100) my_name = self.browser.get_my_name() child = None msg = None s**t = self._get_slut(mails["member"]["pseudo"]) if contacts is None: contacts = {} if not thread.title: thread.title = u"Discussion with %s" % mails["member"]["pseudo"] for mail in mails["messages"]: flags = Message.IS_HTML if parse_dt(mail["date"]) > s**t["lastmsg"] and mail["id_from"] != self.browser.get_my_name(): flags |= Message.IS_UNREAD if get_profiles: if not mail["id_from"] in contacts: with self.browser: contacts[mail["id_from"]] = self.get_contact(mail["id_from"]) signature = u"" if mail.get("src", None): signature += u"Sent from my %s\n\n" % mail["src"] if mail["id_from"] in contacts: signature += contacts[mail["id_from"]].get_text() msg = Message( thread=thread, id=int(time.strftime("%Y%m%d%H%M%S", parse_dt(mail["date"]).timetuple())), title=thread.title, sender=mail["id_from"], receivers=[my_name if mail["id_from"] != my_name else mails["member"]["pseudo"]], date=parse_dt(mail["date"]), content=unescape(mail["message"]).strip(), signature=signature, children=[], flags=flags, ) if child: msg.children.append(child) child.parent = msg child = msg if full and msg: # If we have get all the messages, replace NotLoaded with None as # parent. msg.parent = None if not full and not msg: # Perhaps there are hidden messages msg = NotLoaded thread.root = msg return thread
def process_incoming_mail(self, msg): to = self.get_email_address_ident(msg, 'To') sender = msg.get('From') reply_to = self.get_email_address_ident(msg, 'In-Reply-To') title = msg.get('Subject') if title: new_title = u'' for part in decode_header(title): if part[1]: new_title += unicode(part[0], part[1]) else: new_title += unicode(part[0]) title = new_title content = u'' for part in msg.walk(): if part.get_content_type() == 'text/plain': s = part.get_payload(decode=True) charsets = part.get_charsets() + msg.get_charsets() for charset in charsets: try: if charset is not None: content += unicode(s, charset) else: content += unicode(s) except UnicodeError as e: self.logger.warning('Unicode error: %s' % e) continue except Exception as e: self.logger.exception(e) continue else: break if len(content) == 0: print('Unable to send an empty message', file=self.stderr) return 1 # remove signature content = content.split(u'\n-- \n')[0] parent_id = None if reply_to is None: # This is a new message if '.' in to: backend_name, thread_id = to.split('.', 1) else: backend_name = to thread_id = None else: # This is a reply try: backend_name, id = reply_to.split('.', 1) thread_id, parent_id = id.rsplit('.', 1) except ValueError: print( 'In-Reply-To header might be in form <backend.thread_id.message_id>', file=self.stderr) return 1 # Default use the To header field to know the backend to use. if to and backend_name != to: backend_name = to try: backend = self.weboob.backend_instances[backend_name] except KeyError: print('Backend %s not found' % backend_name, file=self.stderr) return 1 if not backend.has_caps(CapMessagesPost): print('The backend %s does not implement CapMessagesPost' % backend_name, file=self.stderr) return 1 thread = Thread(thread_id) message = Message( thread, 0, title=title, sender=sender, receivers=[to], parent=Message(thread, parent_id) if parent_id else None, content=content) try: backend.post_message(message) except Exception as e: content = u'Unable to send message to %s:\n' % thread_id content += u'\n\t%s\n' % to_unicode(e) if logging.root.level <= logging.DEBUG: content += u'\n%s\n' % to_unicode(get_backtrace(e)) self.send_email( backend.name, Message( thread, 0, title='Unable to send message', sender='Monboob', parent=Message(thread, parent_id) if parent_id else None, content=content))
def get_thread(self, id, contacts=None, get_profiles=False): """ Get a thread and its messages. The 'contacts' parameters is only used for internal calls. """ thread = None if isinstance(id, Thread): thread = id id = thread.id if not thread: thread = Thread(int(id)) thread.flags = Thread.IS_DISCUSSION full = False else: full = True with self.browser: mails = self.browser.get_thread_mails(id, 100) my_name = self.browser.get_my_name() child = None msg = None s**t = self._get_slut(id) if contacts is None: contacts = {} if not thread.title: thread.title = u'Discussion with %s' % mails['who']['pseudo'] self.storage.set('s***s', int(thread.id), 'status', mails['status']) self.storage.save() for mail in mails['results']: flags = 0 if self.antispam and not self.antispam.check_mail(mail): self.logger.info('Skipped a spam-mail from %s' % mails['who']['pseudo']) self.report_spam(thread.id) break if parse_dt(mail['date']) > s**t['lastmsg']: flags |= Message.IS_UNREAD if get_profiles: if not mail['from'] in contacts: try: with self.browser: contacts[mail['from']] = self.get_contact(mail['from']) except BrowserHTTPNotFound: pass if self.antispam and mail['from'] in contacts and not self.antispam.check_contact(contacts[mail['from']]): self.logger.info('Skipped a spam-mail-profile from %s' % mails['who']['pseudo']) self.report_spam(thread.id) break if int(mail['from']) == self.browser.my_id: if mails['remote_status'] == 'new' and msg is None: flags |= Message.IS_NOT_RECEIVED else: flags |= Message.IS_RECEIVED signature = u'' #if mail.get('src', None): # signature += u'Sent from my %s\n\n' % mail['src'] if mail['from'] in contacts: signature += contacts[mail['from']].get_text() msg = Message(thread=thread, id=int(time.strftime('%Y%m%d%H%M%S', parse_dt(mail['date']).timetuple())), title=thread.title, sender=to_unicode(my_name if int(mail['from']) == self.browser.my_id else mails['who']['pseudo']), receivers=[to_unicode(my_name if int(mail['from']) != self.browser.my_id else mails['who']['pseudo'])], date=parse_dt(mail['date']), content=to_unicode(unescape(mail['message'] or '').strip()), signature=signature, children=[], flags=flags) if child: msg.children.append(child) child.parent = msg child = msg if full and msg: # If we have get all the messages, replace NotLoaded with None as # parent. msg.parent = None if not full and not msg: # Perhaps there are hidden messages msg = NotLoaded thread.root = msg return thread
def get_thread(self, id, contacts=None, get_profiles=False): """ Get a thread and its messages. The 'contacts' parameters is only used for internal calls. """ thread = None if isinstance(id, Thread): thread = id id = thread.id if not thread and isinstance(id, basestring) and not id.isdigit(): for t in self.browser.get_threads_list(): if t['username'] == id: id = t['id'] break else: return None if not thread: thread = Thread(int(id)) thread.flags = Thread.IS_DISCUSSION with self.browser: mails = self.browser.get_thread_mails(id) my_name = self.browser.get_my_name() child = None msg = None s**t = self._get_slut(thread.id) if contacts is None: contacts = {} if not thread.title: thread.title = u'Discussion with %s' % mails['member']['pseudo'] for mail in mails['messages']: flags = 0 if mail['date'] > s**t['lastmsg']: flags |= Message.IS_UNREAD if get_profiles: if not mail['id_from'] in contacts: with self.browser: contacts[mail['id_from']] = self.get_contact(mail['id_from']) signature = u'' if mail.get('src', None): signature += u'Sent from my %s\n\n' % mail['src'] if contacts.get(mail['id_from'], None) is not None: signature += contacts[mail['id_from']].get_text() msg = Message(thread=thread, id=int(time.strftime('%Y%m%d%H%M%S', mail['date'].timetuple())), title=thread.title, sender=mail['id_from'], receivers=[my_name if mail['id_from'] != my_name else mails['member']['pseudo']], date=mail['date'], content=mail['message'], signature=signature, children=[], flags=flags) if child: msg.children.append(child) child.parent = msg child = msg if msg: msg.parent = None thread.root = msg return thread
def iter_unread_messages(self): try: contacts = {} with self.browser: threads = self.browser.get_threads_list() for thread in threads: #if thread['member'].get('isBan', thread['member'].get('dead', False)): # with self.browser: # self.browser.delete_thread(int(thread['member']['id'])) # continue if self.antispam and not self.antispam.check_thread(thread): self.logger.info('Skipped a spam-unread-thread from %s' % thread['who']['pseudo']) self.report_spam(thread['member']['id']) continue s**t = self._get_slut(thread['who']['id']) if parse_dt( thread['date'] ) > s**t['lastmsg'] or thread['status'] != s**t['status']: t = self.get_thread(thread['who']['id'], contacts, get_profiles=True) for m in t.iter_all_messages(): if m.flags & m.IS_UNREAD: yield m if not self.config['baskets'].get(): return # Send mail when someone added me in her basket. # XXX possibly race condition if a s**t adds me in her basket # between the aum.nb_new_baskets() and aum.get_baskets(). with self.browser: s**t = self._get_slut(-self.MAGIC_ID_BASKET) new_baskets = self.browser.nb_new_baskets() if new_baskets > 0: baskets = self.browser.get_baskets() my_name = self.browser.get_my_name() for basket in baskets: if parse_dt(basket['date']) <= s**t['lastmsg']: continue contact = self.get_contact(basket['who']['id']) if self.antispam and not self.antispam.check_contact( contact): self.logger.info('Skipped a spam-basket from %s' % contact.name) self.report_spam(basket['who']['id']) continue thread = Thread(int(basket['who']['id'])) thread.title = 'Basket of %s' % contact.name thread.root = Message( thread=thread, id=self.MAGIC_ID_BASKET, title=thread.title, sender=contact.name, receivers=[my_name], date=parse_dt(basket['date']), content='You are taken in her basket!', signature=contact.get_text(), children=[], flags=Message.IS_UNREAD) yield thread.root except BrowserUnavailable as e: self.logger.debug('No messages, browser is unavailable: %s' % e) pass # don't care about waiting
def get_thread(self, id, contacts=None, get_profiles=False): """ Get a thread and its messages. The 'contacts' parameters is only used for internal calls. """ thread = None if isinstance(id, Thread): thread = id id = thread.id if not thread: thread = Thread(int(id)) thread.flags = Thread.IS_DISCUSSION full = False else: full = True with self.browser: mails = self.browser.get_thread_mails(id, 100) my_name = self.browser.get_my_name() child = None msg = None s**t = self._get_slut(id) if contacts is None: contacts = {} if not thread.title: thread.title = u'Discussion with %s' % mails['who']['pseudo'] self.storage.set('s***s', int(thread.id), 'status', mails['status']) self.storage.save() for mail in mails['results']: flags = 0 if self.antispam and not self.antispam.check_mail(mail): self.logger.info('Skipped a spam-mail from %s' % mails['who']['pseudo']) self.report_spam(thread.id) break if parse_dt(mail['date']) > s**t['lastmsg']: flags |= Message.IS_UNREAD if get_profiles: if not mail['from'] in contacts: try: with self.browser: contacts[mail['from']] = self.get_contact( mail['from']) except BrowserHTTPNotFound: pass if self.antispam and mail[ 'from'] in contacts and not self.antispam.check_contact( contacts[mail['from']]): self.logger.info( 'Skipped a spam-mail-profile from %s' % mails['who']['pseudo']) self.report_spam(thread.id) break if int(mail['from']) == self.browser.my_id: if mails['remote_status'] == 'new' and msg is None: flags |= Message.IS_NOT_RECEIVED else: flags |= Message.IS_RECEIVED signature = u'' #if mail.get('src', None): # signature += u'Sent from my %s\n\n' % mail['src'] if mail['from'] in contacts: signature += contacts[mail['from']].get_text() msg = Message( thread=thread, id=int( time.strftime('%Y%m%d%H%M%S', parse_dt(mail['date']).timetuple())), title=thread.title, sender=to_unicode(my_name if int(mail['from']) == self.browser. my_id else mails['who']['pseudo']), receivers=[ to_unicode(my_name if int(mail['from']) != self.browser. my_id else mails['who']['pseudo']) ], date=parse_dt(mail['date']), content=to_unicode(unescape(mail['message'] or '').strip()), signature=signature, children=[], flags=flags) if child: msg.children.append(child) child.parent = msg child = msg if full and msg: # If we have get all the messages, replace NotLoaded with None as # parent. msg.parent = None if not full and not msg: # Perhaps there are hidden messages msg = NotLoaded thread.root = msg return thread