def _process_post(self, thread, post, is_root):
# Tapatalk app seems to have hardcoded this construction... I don't think we can do better :(
url = u'%s/index.php?/topic/%s-%s#entry%s' % (
self.config["url"].get().rstrip('/'),
thread.id,
re.sub(r'[^a-zA-Z0-9-]', '', re.sub(r'\s+', '-', thread.title)),
post['post_id']
)
message = Message(
id = is_root and "0" or str(post['post_id']),
thread = thread,
sender = unicode(str(post.get('post_author_name', 'Anonymous')), 'utf-8'),
title = is_root and thread.title or u"Re: %s"%thread.title,
url = url,
receivers = None,
date = self._get_time(post),
content = self._format_content(post),#bbcode(),
signature = None,
parent = thread.root or None,
children = [],
flags = Message.IS_HTML)
if thread.root:
thread.root.children.append(message)
elif is_root:
thread.root = message
else:
# First message in the thread is not the root message,
# because we asked only for unread messages. Create a non-loaded root
# message to allow monboob to fill correctly the References: header
thread.root = Message(id="0", parent=None, children=[message], thread=thread)
message.parent = thread.root
return message
def _insert_comment(self, com, parent, getseen=True):
""""
Insert 'com' comment and its children in the parent message.
"""
flags = Message.IS_HTML
if com.id not in self.storage.get('seen', parent.thread.id, 'comments', default=[]):
flags |= Message.IS_UNREAD
if getseen or flags & Message.IS_UNREAD:
com.parse()
message = Message(thread=parent.thread,
id=com.id,
url=com.url,
title=com.title,
sender=com.author or u'',
receivers=None,
date=com.date,
parent=parent,
content=com.body,
signature=com.signature +
'<br />'.join(['Score: %d' % com.score,
'URL: %s' % com.url]),
children=[],
flags=flags)
else:
message = Message(thread=parent.thread,
id=com.id,
children=[],
parent=parent,
flags=flags)
parent.children.append(message)
for sub in com.comments:
self._insert_comment(sub, message, getseen)
def get_thread(self, id):
thread = None
if isinstance(id, Thread):
thread = id
id = thread.id
if '.' not in id:
self.logger.warning('Malformated ID (%s)' % id)
return
board, thread_id = self._splitid(id)
with self.browser:
_thread = self.browser.get_thread(board, thread_id)
flags = 0
if _thread.id not in self.storage.get('boards', board, default={}):
flags |= Message.IS_UNREAD
if not thread:
thread = Thread(id)
thread.title = _thread.filename
thread.root = Message(
thread=thread,
id=0, # root message
title=_thread.filename,
sender=_thread.author,
receivers=None,
date=_thread.datetime,
parent=None,
content=_thread.text,
signature=None,
children=[],
flags=flags | Message.IS_HTML)
for comment in _thread.comments:
flags = 0
if comment.id not in self.storage.get('boards',
board,
_thread.id,
default=[]):
flags |= Message.IS_UNREAD
m = Message(thread=thread,
id=comment.id,
title=_thread.filename,
sender=comment.author,
receivers=None,
date=comment.datetime,
parent=thread.root,
content=comment.text,
signature=None,
children=None,
flags=flags | Message.IS_HTML)
thread.root.children.append(m)
return thread
def do_post(self, line):
"""
post RECEIVER@BACKEND[,RECEIVER@BACKEND[...]] [TEXT]
Post a message to the specified receivers.
Multiple receivers are separated by a comma.
If no text is supplied on command line, the content of message is read on stdin.
"""
receivers, text = self.parse_command_args(line, 2, 1)
if text is None:
text = self.acquire_input()
if not self.options.accept_empty and not text.strip():
self.logger.warning(
u'The message body is empty, use option --accept_empty to send empty messages'
)
return
for receiver in receivers.strip().split(','):
receiver, backend_name = self.parse_id(receiver.strip(),
unique_backend=True)
if not backend_name and len(self.enabled_backends) > 1:
self.logger.warning(
u'No backend specified for receiver "%s": message will be sent with all the '
'enabled backends (%s)' %
(receiver, ','.join(backend.name
for backend in self.enabled_backends)))
if '.' in receiver:
# It's a reply
thread_id, parent_id = receiver.rsplit('.', 1)
else:
# It's an original message
thread_id = receiver
parent_id = None
try:
thread_id = self.threads[int(thread_id) - 1].id
except (IndexError, ValueError):
pass
thread = Thread(thread_id)
message = Message(
thread,
0,
title=self.options.title,
parent=Message(thread, parent_id) if parent_id else None,
content=text)
try:
self.do('post_message', message, backends=backend_name).wait()
except CallErrors as errors:
self.bcall_errors_handler(errors)
else:
if self.interactive:
print('Message sent sucessfully to %s' % receiver)
def iter_threads(self):
table = self.parser.select(self.document.getroot(),
'table#listeMessages', 1)
for tr in table.xpath('./tr'):
if tr.attrib.get('class', '') not in ('msgLu', 'msgNonLu'):
continue
author = unicode(self.parser.select(tr, 'td.colEmetteur', 1).text)
link = self.parser.select(tr, 'td.colObjet a', 1)
date_raw = self.parser.select(tr, 'td.colDate1', 1).attrib['data']
jsparams = re.search('\((.+)\)',
link.attrib['onclick']).groups()[0]
jsparams = [i.strip('\'" ') for i in jsparams.split(',')]
page_id, _id, unread = jsparams
# this means unread on the website
unread = False if unread == "false" else True
# 2012/02/29:01h30min45sec
dt_match = re.match('(\d+)/(\d+)/(\d+):(\d+)h(\d+)min(\d+)sec',
date_raw).groups()
dt_match = [int(d) for d in dt_match]
thread = Thread(_id)
thread._link_id = (page_id, unread)
thread.date = datetime(*dt_match)
thread.title = unicode(link.text)
message = Message(thread, 0)
message.set_empty_fields(None)
message.flags = message.IS_HTML
message.title = thread.title
message.date = thread.date
message.sender = author
message.content = NotLoaded # This is the only thing we are missing
thread.root = message
yield thread
def iter_threads(self):
table = self.parser.select(self.document.getroot(), "table#listeMessages", 1)
for tr in table.xpath("./tr"):
if tr.attrib.get("class", "") not in ("msgLu", "msgNonLu"):
continue
author = unicode(self.parser.select(tr, "td.colEmetteur", 1).text)
link = self.parser.select(tr, "td.colObjet a", 1)
date_raw = self.parser.select(tr, "td.colDate1", 1).attrib["data"]
jsparams = re.search("\((.+)\)", link.attrib["onclick"]).groups()[0]
jsparams = [i.strip("'\" ") for i in jsparams.split(",")]
page_id, _id, unread = jsparams
# this means unread on the website
unread = False if unread == "false" else True
# 2012/02/29:01h30min45sec
dt_match = re.match("(\d+)/(\d+)/(\d+):(\d+)h(\d+)min(\d+)sec", date_raw).groups()
dt_match = [int(d) for d in dt_match]
thread = Thread(_id)
thread._link_id = (page_id, unread)
thread.date = datetime(*dt_match)
thread.title = unicode(link.text)
message = Message(thread, 0)
message.set_empty_fields(None)
message.flags = message.IS_HTML
message.title = thread.title
message.date = thread.date
message.sender = author
message.content = NotLoaded # This is the only thing we are missing
thread.root = message
yield thread
def make_message(self, d, thread):
m = Message(thread, d['id'])
m.children = []
m.sender = d['from']
m.flags = 0
if not d.get('read', True):
m.flags = m.IS_UNREAD
m.title = d['subject']
m.date = d['datetime']
m.receivers = [d['to']]
return m
def iter_threads(self):
for thread in self.browser.get_threads():
if 'person' not in thread:
# The account has been removed, probably because it was a
# spammer.
continue
t = Thread(thread['_id'])
t.flags = Thread.IS_DISCUSSION
t.title = u'Discussion with %s' % thread['person']['name']
contact = self.storage.get('contacts', t.id, default={'lastmsg': 0})
birthday = parse_date(thread['person']['birth_date']).date()
signature = u'Age: %d (%s)' % ((datetime.date.today() - birthday).days / 365.25, birthday)
signature += u'\nLast ping: %s' % parse_date(thread['person']['ping_time']).strftime('%Y-%m-%d %H:%M:%S')
signature += u'\nPhotos:\n\t%s' % '\n\t'.join([photo['url'] for photo in thread['person']['photos']])
signature += u'\n\n%s' % thread['person']['bio']
t.root = Message(thread=t, id=1, title=t.title,
sender=unicode(thread['person']['name']),
receivers=[self.browser.my_name],
date=parse_date(thread['created_date']),
content=u'Match!',
children=[],
signature=signature,
flags=Message.IS_UNREAD if int(contact['lastmsg']) < 1 else 0)
parent = t.root
for msg in thread['messages']:
flags = 0
if int(contact['lastmsg']) < msg['timestamp']:
flags = Message.IS_UNREAD
msg = Message(thread=t,
id=msg['timestamp'],
title=t.title,
sender=unicode(self.browser.my_name if msg['from'] == self.browser.my_id else thread['person']['name']),
receivers=[unicode(self.browser.my_name if msg['to'] == self.browser.my_id else thread['person']['name'])],
date=parse_date(msg['sent_date']),
content=unicode(msg['message']),
children=[],
parent=parent,
signature=signature if msg['to'] == self.browser.my_id else u'',
flags=flags)
parent.children.append(msg)
parent = msg
yield t
def get_thread(self, id):
if isinstance(id, Thread):
thread = id
id = thread.id
else:
thread = None
story = self.browser.get_story(id)
if not story:
return None
if not thread:
thread = Thread(story.id)
flags = 0
if thread.id not in self.storage.get('seen', default=[]):
flags |= Message.IS_UNREAD
thread.title = story.title
thread.date = story.date
thread.root = Message(thread=thread,
id=0,
title=story.title,
sender=story.author.name,
receivers=None,
date=thread.date,
parent=None,
content=story.body,
children=[],
signature=u'Written by a %s in category %s' %
(self.GENDERS[story.author.sex], story.category),
flags=flags)
return thread
def get_thread(self, _id, thread=None, seen=None):
splitted_id = _id.split('#')
if not thread:
thread = self.thread_page.go(_id=splitted_id[1].split('.')[-1], user=splitted_id[0]).get_thread(obj=thread)
title_content = thread.title.split('\n\t')[-1]
thread.root = Message(thread=thread,
id=splitted_id[1].split('.')[-1],
title=title_content[:50] if len(title_content) > 50 else title_content,
sender=splitted_id[0],
receivers=None,
date=thread.date,
parent=thread.root,
content=title_content,
signature=u'',
children=[]
)
if seen and (_id not in seen):
thread.root.flags = Message.IS_UNREAD
comments = self.thread_page.stay_or_go(_id=splitted_id[1].split('.')[-1], user=splitted_id[0]).iter_comments()
for comment in comments:
comment.thread = thread
comment.parent = thread.root
if seen and comment.id not in seen.keys():
comment.flags = Message.IS_UNREAD
thread.root.children.append(comment)
return thread
def get_thread(self, id, entry=None):
if isinstance(id, Thread):
thread = id
id = thread.id
else:
thread = Thread(id)
if entry is None:
entry = Newsfeed(self.config['url'].get()).get_entry(id)
if entry is None:
return None
flags = Message.IS_HTML
if thread.id not in self.storage.get('seen', default=[]):
flags |= Message.IS_UNREAD
if len(entry.content) > 0:
content = u"<p>Link %s</p> %s" % (entry.link, entry.content[0])
else:
content = entry.link
thread.title = entry.title
thread.root = Message(thread=thread,
id=0,
title=entry.title,
sender=entry.author,
receivers=None,
date=entry.datetime,
parent=None,
content=content,
children=[],
flags=flags)
return thread
def iter_unread_messages(self):
with self.browser:
url = self.browser.get_root_feed_url()
for article in Newsfeed(url, rssid).iter_entries():
id = url2id(article.link)
thread = None
try:
last_seen_id = self.storage.get('seen',
default={})[id2topic(id)]
except KeyError:
last_seen_id = 0
child = None
iterator = self.browser.riter_posts(id, last_seen_id)
if self.config['thread_unread_messages'].get() > 0:
iterator = limit(
iterator, self.config['thread_unread_messages'].get())
for post in iterator:
if not thread:
thread = Thread('%s.%s' %
(post.forum_id, post.topic_id))
message = self._post2message(thread, post)
if child:
message.children.append(child)
child.parent = message
if post.parent:
message.parent = Message(thread=thread, id=post.parent)
else:
thread.root = message
yield message
def _sendPressed(self):
if not self.ui.replyWidget.isVisible():
return
text = unicode(self.ui.replyEdit.toPlainText())
title = unicode(self.ui.titleEdit.text())
self.ui.backendsList.setEnabled(False)
self.ui.threadsList.setEnabled(False)
self.ui.messagesTree.setEnabled(False)
self.ui.replyButton.setEnabled(False)
self.ui.replyWidget.setEnabled(False)
self.ui.sendButton.setText(self.tr('Sending...'))
flags = 0
if self.ui.htmlBox.currentIndex() == 0:
flags = Message.IS_HTML
m = Message(thread=self.thread,
id=0,
title=title,
sender=None,
receivers=None,
content=text,
parent=self.message,
flags=flags)
self.process_reply = QtDo(self.weboob, self._postReply_cb, self._postReply_eb)
self.process_reply.do('post_message', m, backends=self.thread.backend)
def make_message(self, d, thread):
msg = Message(thread, d['id'])
msg.children = []
msg.sender = d['from']
msg.flags = 0
msg.title = d['subject']
msg.date = d['datetime']
msg.receivers = [d['to']]
return msg
def build_article(self, j):
m = self.article.match(j['url'])
ret = Message(id=m.group('title'))
ret.title = j['title']
ret.url = j['url']
ret.flags = Message.IS_HTML
ret.date = date(int(m.group('year')), int(m.group('month')), 1)
ret.children = []
ret._type = 'article'
return ret
def make_message(self, d, thread):
m = Message(thread, d['id'])
m.children = []
m.sender = d['from']
m.flags = 0
if not d.get('read', True):
m.flags = m.IS_UNREAD
m.title = d['subject']
m.date = d['datetime']
m.receivers = [d['to']]
return m
def _get_messages(self, thread):
thread_div = self.document.find(True, "PADpost_txt")
used_ids = set()
rcpt = self.document.find("input", attrs={"type": "hidden", "name": "Dest"})["value"]
sender_to_receiver = {rcpt: self.browser.username, self.browser.username: rcpt}
# site is sorted from latest to oldest message
for message_table in reversed(thread_div.findAll("table")):
for td in message_table.findAll("td"):
profile_a = td.find("a", href=re.compile(r"profil_read.php\?.*"))
if not profile_a:
continue
first_br = td.find("br")
assert first_br.nextSibling.name == "br"
text_nodes = ovsparse.all_next_siblings(first_br.nextSibling.nextSibling) # TODO
# ~ print text_nodes
# date will be used as id
sitedate = profile_a.findParent("div").find(text=re.compile(",.*")).replace(", ", "")
sysdate = parse_french_date(sitedate)
compactdate = datetime.datetime.strftime(sysdate, "%Y%m%dT%H%M%S")
# but make it unique
msg_id = ovsparse.create_unique_id(compactdate, used_ids)
used_ids.add(msg_id)
message = Message(thread, msg_id)
message.sender = re.search(r"\?(.+)", profile_a["href"]).group(1)
message.receivers = [sender_to_receiver[message.sender]]
message.date = sysdate
message.content = ovsparse.html_message_to_text(text_nodes)
notread_self = bool(td.find("span", "ColorSurligne"))
notread_other = bool(td.find("span", "new_sortiepartenaire"))
if notread_other or notread_self:
message.flags |= Message.IS_NOT_RECEIVED
else:
message.flags |= Message.IS_RECEIVED
yield message
def get_thread(self, thread):
if not isinstance(thread, Thread):
thread = Thread(thread)
thread.flags = Thread.IS_DISCUSSION
info = self.browser.get_thread(thread.id)
for user in info['participants']:
if user['user']['fb_id'] is not None:
user['user']['fb'] = self.browser.get_facebook(
user['user']['fb_id'])
if user['user']['id'] == self.browser.my_id:
me = HappnContact(user['user'])
else:
other = HappnContact(user['user'])
thread.title = u'Discussion with %s' % other.name
contact = self.storage.get(
'contacts',
thread.id,
default={'lastmsg_date': '1970-01-01T01:01:01+00:00'})
child = None
for msg in info['messages']:
flags = 0
if parse_date(contact['lastmsg_date']) < parse_date(
msg['creation_date']):
flags = Message.IS_UNREAD
if msg['sender']['id'] == me.id:
sender = me
receiver = other
else:
sender = other
receiver = me
msg = Message(thread=thread,
id=msg['id'],
title=thread.title,
sender=sender.name,
receivers=[receiver.name],
date=parse_date(msg['creation_date']),
content=msg['message'],
children=[],
parent=None,
signature=sender.get_text(),
flags=flags)
if child:
msg.children.append(child)
child.parent = msg
child = msg
thread.root = child
return thread
def build_date(self, k):
ret = Message(id='%s.%s' % k)
ret.title = '%s/%s' % k
ret.content = ''
ret.date = date(int(k[0]), int(k[1]), 1)
ret._type = 'date'
ret._key = k
return ret
def make_message(self, d, thread):
msg = Message(thread, d['id'])
msg.children = []
msg.sender = d['from']
msg.flags = 0
msg.title = d['subject']
msg.date = d['datetime']
msg.receivers = [d['to']]
return msg
def postReply(self):
text = unicode(self.ui.textEdit.toPlainText())
self.ui.textEdit.setEnabled(False)
self.ui.sendButton.setEnabled(False)
m = Message(thread=self.thread,
id=0,
title=u'',
sender=None,
receivers=None,
content=text,
parent=self.messages[0].message if len(self.messages) > 0 else None)
self.process_reply = QtDo(self.weboob, self._postReply_cb, self._postReply_eb)
self.process_reply.do('post_message', m, backends=self.contact.backend)
def get_thread(self, thread):
if not isinstance(thread, Thread):
thread = Thread(thread)
thread.flags = Thread.IS_DISCUSSION
user = self.browser.get_user(thread.id)
thread.title = u'Discussion with %s' % user['name']
contact = self.storage.get('contacts', thread.id, default={'lastmsg': 0})
signature = u'Age: %s' % user['age']
signature += u'\nLast online: %s' % user['last_online']
signature += u'\nPhotos:\n\t%s' % '\n\t'.join([user['photo_host'] + photo['large'] for photo in user['photos']])
child = None
for msg in self.browser.get_thread_messages(thread.id):
flags = 0
if int(contact['lastmsg']) < msg['utc_timestamp']:
flags = Message.IS_UNREAD
if msg['type'] == 'msg':
content = unicode(msg['msg'])
elif msg['type'] == 'new_challenge':
content = u'A new challenge has been proposed!'
elif msg['type'] == 'serie':
content = u"I've played"
elif msg['type'] == 'end_game':
content = u'%s is the winner! (%s VS %s)' % (self.browser.my_name if msg['score']['w'] == self.browser.my_id else user['name'], msg['score']['s'][0], msg['score']['s'][1])
else:
content = u'Unknown action: %s' % msg['type']
msg = Message(thread=thread,
id=msg['utc_timestamp'],
title=thread.title,
sender=unicode(self.browser.my_name if msg['from'] == self.browser.my_id else user['name']),
receivers=[unicode(self.browser.my_name if msg['from'] != self.browser.my_id else user['name'])],
date=local2utc(datetime.datetime.fromtimestamp(msg['utc_timestamp'])),
content=content,
children=[],
parent=None,
signature=signature if msg['from'] != self.browser.my_id else u'',
flags=flags)
if child:
msg.children.append(child)
child.parent = msg
child = msg
thread.root = child
return thread
def get_thread(self, thread):
if not isinstance(thread, Thread):
thread = Thread(thread)
thread.flags = Thread.IS_DISCUSSION
messages = self.browser.get_thread_messages(thread.id)
contact = self.storage.get('s***s',
thread.id,
default={'lastmsg': datetime(1970, 1, 1)})
thread.title = u'Discussion with %s' % messages['fields']['username']
me = OkcContact(self.browser.get_profile(self.browser.me['userid']))
other = OkcContact(self.browser.get_profile(thread.id))
parent = None
for message in messages['messages']['messages']:
date = datetime.fromtimestamp(message['timestamp'])
flags = 0
if contact['lastmsg'] < date:
flags = Message.IS_UNREAD
if message['from'] == thread.id:
sender = other
receiver = me
else:
receiver = other
sender = me
msg = Message(thread=thread,
id=message['id'],
title=thread.title,
sender=sender.name,
receivers=[receiver.name],
date=date,
content=to_unicode(unescape(message['body'])),
children=[],
parent=parent,
signature=sender.get_text(),
flags=flags)
if parent:
parent.children = [msg]
else:
thread.root = msg
parent = msg
return thread
def get_thread(self, id, getseen=True):
if not isinstance(id, Thread):
thread = None
else:
thread = id
id = thread.id
# Check if we have seen all comments of this thread.
oldhash = self.storage.get('hash', id, default="")
newhash = self.browser.get_hash(thread._rsscomment)
if not getseen and oldhash == newhash:
return None
self.storage.set('hash', id, newhash)
if thread.date:
self.storage.set('date', id, thread.date)
self.storage.save()
with self.browser:
content = self.browser.get_content(id)
if not content:
return None
if not thread:
thread = Thread(content.id)
flags = Message.IS_HTML
if not thread.id in self.storage.get('seen', default={}):
flags |= Message.IS_UNREAD
thread.title = content.title
if not thread.date:
thread.date = content.date
thread.root = Message(thread=thread,
id='0', # root message
title=content.title,
sender=content.author or u'',
receivers=None,
date=thread.date,
parent=None,
content=content.body,
signature='URL: %s' % self.browser.absurl(id2url(content.id)),
children=[],
flags=flags)
for com in content.comments:
self._insert_comment(com, thread.root, getseen)
return thread
def _process_post(self, thread, post, is_root):
# Tapatalk app seems to have hardcoded this construction... I don't think we can do better :(
url = u'%s/index.php?/topic/%s-%s#entry%s' % (
self.config["url"].get().rstrip('/'), thread.id,
re.sub(r'[^a-zA-Z0-9-]', '', re.sub(
r'\s+', '-', thread.title)), post['post_id'])
message = Message(
id=is_root and "0" or str(post['post_id']),
thread=thread,
sender=unicode(str(post.get('post_author_name', 'Anonymous')),
'utf-8'),
title=is_root and thread.title or u"Re: %s" % thread.title,
url=url,
receivers=None,
date=self._get_time(post),
content=self._format_content(post), #bbcode(),
signature=None,
parent=thread.root or None,
children=[],
flags=Message.IS_HTML)
if thread.root:
thread.root.children.append(message)
elif is_root:
thread.root = message
else:
# First message in the thread is not the root message,
# because we asked only for unread messages. Create a non-loaded root
# message to allow monboob to fill correctly the References: header
thread.root = Message(id="0",
parent=None,
children=[message],
thread=thread)
message.parent = thread.root
return message
def _process_post(self, thread, post, is_root):
message_id = is_root and u'0' or xmlrpc_str(post['post_id'])
message_title = is_root and thread.title or u'Re: %s' % thread.title
# Tapatalk app seems to have hardcoded this construction... I don't think we can do better :(
rel_url = self.config['message_url_format'].get().format(
thread_id = urllib.parse.quote(thread.id.encode('utf-8')),
thread_title = urllib.parse.quote(thread.title.encode('utf-8')),
message_id = urllib.parse.quote(message_id.encode('utf-8')),
message_title = urllib.parse.quote(message_title.encode('utf-8')))
message = Message(
id = message_id,
thread = thread,
sender = xmlrpc_str(post.get('post_author_name', u'Anonymous')),
title = message_title,
url = urllib.parse.urljoin(self.config['url'].get(), rel_url),
receivers = None,
date = self._get_time(post),
content = self._format_content(post),
signature = None,
parent = thread.root or None,
children = [],
flags = Message.IS_HTML)
if thread.root:
thread.root.children.append(message)
elif is_root:
thread.root = message
else:
# First message in the thread is not the root message,
# because we asked only for unread messages. Create a non-loaded root
# message to allow monboob to fill correctly the References: header
thread.root = Message(id=u'0', parent=None, children=[message], thread=thread)
message.parent = thread.root
return message
def _post2message(self, thread, post):
signature = post.signature
if signature:
signature += '<br />'
signature += 'URL: %s' % self.browser.absurl(id2url('%s.%s' % (thread.id, post.id)))
return Message(thread=thread,
id=post.id,
title=post.title,
sender=post.author,
receivers=None,
date=post.date,
parent=None,
content=post.content,
signature=signature,
children=[],
flags=Message.IS_HTML)
def _build_message(self, data, thread, parent=None):
flags = self.default_flags
if data["id"] not in self.storage.get("seen", thread.id, default=[]):
flags |= Message.IS_UNREAD
message = Message(thread=thread,
id=data["id"],
title=(parent and "Re: %s" or "%s") % thread.title,
sender=data.get("username"),
receivers=None,
date=dateutil.parser.parse(data["created_at"]),
content=self.formatter(data["body"]),
flags=flags,
parent=parent,
url=thread.url)
self._append_children(data, message, thread)
return message
def get_thread(self, id, getseen=True):
if not isinstance(id, Thread):
thread = None
else:
thread = id
id = thread.id
if thread.date:
self.storage.set('date', id, thread.date)
self.storage.save()
content = self.browser.get_content(id)
if not content:
return None
if not thread:
thread = Thread(content.id)
flags = Message.IS_HTML
if thread.id not in self.storage.get('seen', default={}):
flags |= Message.IS_UNREAD
thread.title = content.title
if not thread.date:
thread.date = content.date
thread.root = Message(
thread=thread,
id='0', # root message
url=self.browser.absurl(id2url(content.id)),
title=content.title,
sender=content.author or u'',
receivers=None,
date=thread.date,
parent=None,
content=content.body,
signature='URL: %s' % self.browser.absurl(id2url(content.id)),
children=[],
flags=flags)
for com in content.comments:
self._insert_comment(com, thread.root, getseen)
return thread
def get_thread(self, _id):
if isinstance(_id, Thread):
thread = _id
id = thread.id
else:
thread = find_object(self.iter_threads(), id=_id)
id = _id
content = self.browser.get_content(id)
if content is None:
return None
if not thread:
thread = Thread(id)
flags = Message.IS_HTML
if thread.id not in self.storage.get('seen', default={}):
flags |= Message.IS_UNREAD
thread.title = content.title
if not thread.date:
thread.date = content.date
thread.root = Message(thread=thread,
id=0,
url=content.url,
title=content.title,
sender=content.author,
receivers=None,
date=thread.date,
parent=None,
content=content.body,
signature=u'<a href="%s">URL</a> \n' %
content.url,
flags=flags,
children=[])
return thread
def _get_messages(self, thread):
thread_div = self.document.find(True, 'PADpost_txt')
used_ids = set()
rcpt = self.document.find('input', attrs={'type': 'hidden', 'name': 'Dest'})['value']
sender_to_receiver = {rcpt: self.browser.username, self.browser.username: rcpt}
# site is sorted from latest to oldest message
for message_table in reversed(thread_div.findAll('table')):
for td in message_table.findAll('td'):
profile_a = td.find('a', href=re.compile(r'profil_read.php\?.*'))
if not profile_a:
continue
first_br = td.find('br')
assert first_br.nextSibling.name == 'br'
text_nodes = ovsparse.all_next_siblings(first_br.nextSibling.nextSibling) # TODO
#~ print text_nodes
# date will be used as id
sitedate = profile_a.findParent('div').find(text=re.compile(',.*')).replace(', ', '')
sysdate = parse_french_date(sitedate)
compactdate = datetime.datetime.strftime(sysdate, '%Y%m%dT%H%M%S')
# but make it unique
msg_id = ovsparse.create_unique_id(compactdate, used_ids)
used_ids.add(msg_id)
message = Message(thread, msg_id)
message.sender = re.search(r'\?(.+)', profile_a['href']).group(1)
message.receivers = [sender_to_receiver[message.sender]]
message.date = sysdate
message.content = ovsparse.html_message_to_text(text_nodes)
notread_self = bool(td.find('span', 'ColorSurligne'))
notread_other = bool(td.find('span', 'new_sortiepartenaire'))
if notread_other or notread_self:
message.flags |= Message.IS_NOT_RECEIVED
else:
message.flags |= Message.IS_RECEIVED
yield message
def get_thread(self, _id):
if isinstance(_id, Thread):
thread = _id
id = thread.id
else:
thread = self._get_thread(_id)
id = _id
with self.browser:
content = self.browser.get_content(id)
if content is None:
return None
if not thread:
thread = Thread(id)
flags = Message.IS_HTML
if not thread.id in self.storage.get('seen', default={}):
flags |= Message.IS_UNREAD
thread.title = content.title
if not thread.date:
thread.date = content.date
thread.root = Message(thread=thread,
id=0,
title=content.title,
sender=content.author,
receivers=None,
date=thread.date,
parent=None,
content=content.body,
signature='URL: %s' % content.url,
flags=flags,
children=[])
return thread
def get_thread(self, id, contacts=None, get_profiles=False):
"""
Get a thread and its messages.
The 'contacts' parameters is only used for internal calls.
"""
thread = None
if isinstance(id, Thread):
thread = id
id = thread.id
if not thread:
thread = Thread(int(id))
thread.flags = Thread.IS_DISCUSSION
full = False
else:
full = True
with self.browser:
mails = self.browser.get_thread_mails(id, 100)
my_name = self.browser.get_my_name()
child = None
msg = None
s**t = self._get_slut(mails["member"]["pseudo"])
if contacts is None:
contacts = {}
if not thread.title:
thread.title = u"Discussion with %s" % mails["member"]["pseudo"]
for mail in mails["messages"]:
flags = Message.IS_HTML
if parse_dt(mail["date"]) > s**t["lastmsg"] and mail["id_from"] != self.browser.get_my_name():
flags |= Message.IS_UNREAD
if get_profiles:
if not mail["id_from"] in contacts:
with self.browser:
contacts[mail["id_from"]] = self.get_contact(mail["id_from"])
signature = u""
if mail.get("src", None):
signature += u"Sent from my %s\n\n" % mail["src"]
if mail["id_from"] in contacts:
signature += contacts[mail["id_from"]].get_text()
msg = Message(
thread=thread,
id=int(time.strftime("%Y%m%d%H%M%S", parse_dt(mail["date"]).timetuple())),
title=thread.title,
sender=mail["id_from"],
receivers=[my_name if mail["id_from"] != my_name else mails["member"]["pseudo"]],
date=parse_dt(mail["date"]),
content=unescape(mail["message"]).strip(),
signature=signature,
children=[],
flags=flags,
)
if child:
msg.children.append(child)
child.parent = msg
child = msg
if full and msg:
# If we have get all the messages, replace NotLoaded with None as
# parent.
msg.parent = None
if not full and not msg:
# Perhaps there are hidden messages
msg = NotLoaded
thread.root = msg
return thread
def process_incoming_mail(self, msg):
to = self.get_email_address_ident(msg, 'To')
sender = msg.get('From')
reply_to = self.get_email_address_ident(msg, 'In-Reply-To')
title = msg.get('Subject')
if title:
new_title = u''
for part in decode_header(title):
if part[1]:
new_title += unicode(part[0], part[1])
else:
new_title += unicode(part[0])
title = new_title
content = u''
for part in msg.walk():
if part.get_content_type() == 'text/plain':
s = part.get_payload(decode=True)
charsets = part.get_charsets() + msg.get_charsets()
for charset in charsets:
try:
if charset is not None:
content += unicode(s, charset)
else:
content += unicode(s)
except UnicodeError as e:
self.logger.warning('Unicode error: %s' % e)
continue
except Exception as e:
self.logger.exception(e)
continue
else:
break
if len(content) == 0:
print('Unable to send an empty message', file=self.stderr)
return 1
# remove signature
content = content.split(u'\n-- \n')[0]
parent_id = None
if reply_to is None:
# This is a new message
if '.' in to:
backend_name, thread_id = to.split('.', 1)
else:
backend_name = to
thread_id = None
else:
# This is a reply
try:
backend_name, id = reply_to.split('.', 1)
thread_id, parent_id = id.rsplit('.', 1)
except ValueError:
print(
'In-Reply-To header might be in form <backend.thread_id.message_id>',
file=self.stderr)
return 1
# Default use the To header field to know the backend to use.
if to and backend_name != to:
backend_name = to
try:
backend = self.weboob.backend_instances[backend_name]
except KeyError:
print('Backend %s not found' % backend_name, file=self.stderr)
return 1
if not backend.has_caps(CapMessagesPost):
print('The backend %s does not implement CapMessagesPost' %
backend_name,
file=self.stderr)
return 1
thread = Thread(thread_id)
message = Message(
thread,
0,
title=title,
sender=sender,
receivers=[to],
parent=Message(thread, parent_id) if parent_id else None,
content=content)
try:
backend.post_message(message)
except Exception as e:
content = u'Unable to send message to %s:\n' % thread_id
content += u'\n\t%s\n' % to_unicode(e)
if logging.root.level <= logging.DEBUG:
content += u'\n%s\n' % to_unicode(get_backtrace(e))
self.send_email(
backend.name,
Message(
thread,
0,
title='Unable to send message',
sender='Monboob',
parent=Message(thread, parent_id) if parent_id else None,
content=content))
def get_thread(self, id, contacts=None, get_profiles=False):
"""
Get a thread and its messages.
The 'contacts' parameters is only used for internal calls.
"""
thread = None
if isinstance(id, Thread):
thread = id
id = thread.id
if not thread:
thread = Thread(int(id))
thread.flags = Thread.IS_DISCUSSION
full = False
else:
full = True
with self.browser:
mails = self.browser.get_thread_mails(id, 100)
my_name = self.browser.get_my_name()
child = None
msg = None
s**t = self._get_slut(id)
if contacts is None:
contacts = {}
if not thread.title:
thread.title = u'Discussion with %s' % mails['who']['pseudo']
self.storage.set('s***s', int(thread.id), 'status', mails['status'])
self.storage.save()
for mail in mails['results']:
flags = 0
if self.antispam and not self.antispam.check_mail(mail):
self.logger.info('Skipped a spam-mail from %s' % mails['who']['pseudo'])
self.report_spam(thread.id)
break
if parse_dt(mail['date']) > s**t['lastmsg']:
flags |= Message.IS_UNREAD
if get_profiles:
if not mail['from'] in contacts:
try:
with self.browser:
contacts[mail['from']] = self.get_contact(mail['from'])
except BrowserHTTPNotFound:
pass
if self.antispam and mail['from'] in contacts and not self.antispam.check_contact(contacts[mail['from']]):
self.logger.info('Skipped a spam-mail-profile from %s' % mails['who']['pseudo'])
self.report_spam(thread.id)
break
if int(mail['from']) == self.browser.my_id:
if mails['remote_status'] == 'new' and msg is None:
flags |= Message.IS_NOT_RECEIVED
else:
flags |= Message.IS_RECEIVED
signature = u''
#if mail.get('src', None):
# signature += u'Sent from my %s\n\n' % mail['src']
if mail['from'] in contacts:
signature += contacts[mail['from']].get_text()
msg = Message(thread=thread,
id=int(time.strftime('%Y%m%d%H%M%S', parse_dt(mail['date']).timetuple())),
title=thread.title,
sender=to_unicode(my_name if int(mail['from']) == self.browser.my_id else mails['who']['pseudo']),
receivers=[to_unicode(my_name if int(mail['from']) != self.browser.my_id else mails['who']['pseudo'])],
date=parse_dt(mail['date']),
content=to_unicode(unescape(mail['message'] or '').strip()),
signature=signature,
children=[],
flags=flags)
if child:
msg.children.append(child)
child.parent = msg
child = msg
if full and msg:
# If we have get all the messages, replace NotLoaded with None as
# parent.
msg.parent = None
if not full and not msg:
# Perhaps there are hidden messages
msg = NotLoaded
thread.root = msg
return thread
def get_thread(self, id, contacts=None, get_profiles=False):
"""
Get a thread and its messages.
The 'contacts' parameters is only used for internal calls.
"""
thread = None
if isinstance(id, Thread):
thread = id
id = thread.id
if not thread and isinstance(id, basestring) and not id.isdigit():
for t in self.browser.get_threads_list():
if t['username'] == id:
id = t['id']
break
else:
return None
if not thread:
thread = Thread(int(id))
thread.flags = Thread.IS_DISCUSSION
with self.browser:
mails = self.browser.get_thread_mails(id)
my_name = self.browser.get_my_name()
child = None
msg = None
s**t = self._get_slut(thread.id)
if contacts is None:
contacts = {}
if not thread.title:
thread.title = u'Discussion with %s' % mails['member']['pseudo']
for mail in mails['messages']:
flags = 0
if mail['date'] > s**t['lastmsg']:
flags |= Message.IS_UNREAD
if get_profiles:
if not mail['id_from'] in contacts:
with self.browser:
contacts[mail['id_from']] = self.get_contact(mail['id_from'])
signature = u''
if mail.get('src', None):
signature += u'Sent from my %s\n\n' % mail['src']
if contacts.get(mail['id_from'], None) is not None:
signature += contacts[mail['id_from']].get_text()
msg = Message(thread=thread,
id=int(time.strftime('%Y%m%d%H%M%S', mail['date'].timetuple())),
title=thread.title,
sender=mail['id_from'],
receivers=[my_name if mail['id_from'] != my_name else mails['member']['pseudo']],
date=mail['date'],
content=mail['message'],
signature=signature,
children=[],
flags=flags)
if child:
msg.children.append(child)
child.parent = msg
child = msg
if msg:
msg.parent = None
thread.root = msg
return thread
def iter_unread_messages(self):
try:
contacts = {}
with self.browser:
threads = self.browser.get_threads_list()
for thread in threads:
#if thread['member'].get('isBan', thread['member'].get('dead', False)):
# with self.browser:
# self.browser.delete_thread(int(thread['member']['id']))
# continue
if self.antispam and not self.antispam.check_thread(thread):
self.logger.info('Skipped a spam-unread-thread from %s' %
thread['who']['pseudo'])
self.report_spam(thread['member']['id'])
continue
s**t = self._get_slut(thread['who']['id'])
if parse_dt(
thread['date']
) > s**t['lastmsg'] or thread['status'] != s**t['status']:
t = self.get_thread(thread['who']['id'],
contacts,
get_profiles=True)
for m in t.iter_all_messages():
if m.flags & m.IS_UNREAD:
yield m
if not self.config['baskets'].get():
return
# Send mail when someone added me in her basket.
# XXX possibly race condition if a s**t adds me in her basket
# between the aum.nb_new_baskets() and aum.get_baskets().
with self.browser:
s**t = self._get_slut(-self.MAGIC_ID_BASKET)
new_baskets = self.browser.nb_new_baskets()
if new_baskets > 0:
baskets = self.browser.get_baskets()
my_name = self.browser.get_my_name()
for basket in baskets:
if parse_dt(basket['date']) <= s**t['lastmsg']:
continue
contact = self.get_contact(basket['who']['id'])
if self.antispam and not self.antispam.check_contact(
contact):
self.logger.info('Skipped a spam-basket from %s' %
contact.name)
self.report_spam(basket['who']['id'])
continue
thread = Thread(int(basket['who']['id']))
thread.title = 'Basket of %s' % contact.name
thread.root = Message(
thread=thread,
id=self.MAGIC_ID_BASKET,
title=thread.title,
sender=contact.name,
receivers=[my_name],
date=parse_dt(basket['date']),
content='You are taken in her basket!',
signature=contact.get_text(),
children=[],
flags=Message.IS_UNREAD)
yield thread.root
except BrowserUnavailable as e:
self.logger.debug('No messages, browser is unavailable: %s' % e)
pass # don't care about waiting
def get_thread(self, id, contacts=None, get_profiles=False):
"""
Get a thread and its messages.
The 'contacts' parameters is only used for internal calls.
"""
thread = None
if isinstance(id, Thread):
thread = id
id = thread.id
if not thread:
thread = Thread(int(id))
thread.flags = Thread.IS_DISCUSSION
full = False
else:
full = True
with self.browser:
mails = self.browser.get_thread_mails(id, 100)
my_name = self.browser.get_my_name()
child = None
msg = None
s**t = self._get_slut(id)
if contacts is None:
contacts = {}
if not thread.title:
thread.title = u'Discussion with %s' % mails['who']['pseudo']
self.storage.set('s***s', int(thread.id), 'status', mails['status'])
self.storage.save()
for mail in mails['results']:
flags = 0
if self.antispam and not self.antispam.check_mail(mail):
self.logger.info('Skipped a spam-mail from %s' %
mails['who']['pseudo'])
self.report_spam(thread.id)
break
if parse_dt(mail['date']) > s**t['lastmsg']:
flags |= Message.IS_UNREAD
if get_profiles:
if not mail['from'] in contacts:
try:
with self.browser:
contacts[mail['from']] = self.get_contact(
mail['from'])
except BrowserHTTPNotFound:
pass
if self.antispam and mail[
'from'] in contacts and not self.antispam.check_contact(
contacts[mail['from']]):
self.logger.info(
'Skipped a spam-mail-profile from %s' %
mails['who']['pseudo'])
self.report_spam(thread.id)
break
if int(mail['from']) == self.browser.my_id:
if mails['remote_status'] == 'new' and msg is None:
flags |= Message.IS_NOT_RECEIVED
else:
flags |= Message.IS_RECEIVED
signature = u''
#if mail.get('src', None):
# signature += u'Sent from my %s\n\n' % mail['src']
if mail['from'] in contacts:
signature += contacts[mail['from']].get_text()
msg = Message(
thread=thread,
id=int(
time.strftime('%Y%m%d%H%M%S',
parse_dt(mail['date']).timetuple())),
title=thread.title,
sender=to_unicode(my_name if int(mail['from']) == self.browser.
my_id else mails['who']['pseudo']),
receivers=[
to_unicode(my_name if int(mail['from']) != self.browser.
my_id else mails['who']['pseudo'])
],
date=parse_dt(mail['date']),
content=to_unicode(unescape(mail['message'] or '').strip()),
signature=signature,
children=[],
flags=flags)
if child:
msg.children.append(child)
child.parent = msg
child = msg
if full and msg:
# If we have get all the messages, replace NotLoaded with None as
# parent.
msg.parent = None
if not full and not msg:
# Perhaps there are hidden messages
msg = NotLoaded
thread.root = msg
return thread
