def _text_sanitizer(self, text):
text = remove_non_asciiprintable(text, ' ')
text = remove_urls(text, ' ')
text = remove_twitter_pic_urls(text, ' ')
text = remove_emails(text, ' ')
text = text.lower()
text = text.strip()
return text
def _text_sanitizer(self, text):
text = html.unescape(text)
text = remove_non_asciiprintable(text, ' ')
text = remove_urls(text, ' ')
text = remove_emails(text, ' ')
text = text.lower()
text = text.strip()
return text
def _text_sanitizer(self, text):
text = remove_non_asciiprintable(text, ' ')
text = remove_urls(text, ' ')
text = remove_emails(text, ' ')
text = remove_html_comments(text, ' ')
text = remove_markdown_codeblocks(text, ' ')
text = text.lower()
text = text.strip()
return text
def render(self, streams, count = 100):
accounts = json.loads(self.model.accounts.List())
accounts = dict((a["id"], a) for a in accounts)
messages = []
seen = {}
time = 0
orderby = "time"
order = "desc"
limit = 50
if len(streams) == 1:
# get message data for selected stream
# stream, account, time, transient, recipient, orderby, order, limit
msgs = json.loads(self.model.streams.Messages(streams[0]["stream"] or "all", streams[0]["account"] or "all", time, streams[0]["transient"] or "0", streams[0].has_key("recipient") and streams[0]["recipient"] or "0", orderby, order, limit))
for item in msgs:
message = item
message["dupes"] = []
message["txtid"] = util.remove_urls(message["text"]).strip()[:MAX_MESSAGE_LENGTH] or None
message["color"] = util.Color(accounts.get(message["account"], {"color": "#5A5A5A"})["color"])
message["time_string"] = util.generate_time_string(message["time"])
try:
message["sender"]["image"] = resources.get_avatar_path(message["sender"]["image"])
except:
pass
messages.append(message)
def dupematch(item, message):
if item["service"] == message["service"] and item["mid"] == message["mid"]:
return True
for item in item["dupes"]:
if item["service"] == message["service"] and item["mid"] == message["mid"]:
return True
# Detect duplicates
for n, message in enumerate(messages):
message["is_dupe"] = message["txtid"] in seen
if message["is_dupe"]:
item = messages[seen[message["txtid"]]]
if not dupematch(item, message):
item["dupes"].append(message)
else:
if message["txtid"]:
seen[message["txtid"]] = n
messages = [m for m in messages if not m["is_dupe"]]
WebUi.render(self, self.model.settings["theme"], "template.mako",
message_store=messages,
preferences=self.model.settings,
services=self.model.services,
accounts=accounts)
return [m["txtid"] for m in messages]
def render(self, streams, count = 100):
accounts = json.loads(self.model.accounts.List())
accounts = dict((a["id"], a) for a in accounts)
messages = []
seen = {}
time = 0
orderby = "time"
order = "desc"
limit = 50
if len(streams) == 1:
# get message data for selected stream
# stream, account, time, transient, recipient, orderby, order, limit
msgs = json.loads(self.model.streams.Messages(streams[0]["stream"] or "all", streams[0]["account"] or "all", time, streams[0]["transient"] or "0", streams[0].has_key("recipient") and streams[0]["recipient"] or "0", orderby, order, limit))
for item in msgs:
message = item
message["dupes"] = []
message["txtid"] = util.remove_urls(message["text"]).strip()[:MAX_MESSAGE_LENGTH] or None
message["color"] = util.Color(accounts.get(message["account"], {"color": "#5A5A5A"})["color"])
message["time_string"] = util.generate_time_string(message["time"])
try:
message["sender"]["image"] = resources.get_avatar_path(message["sender"]["image"])
except:
pass
messages.append(message)
def dupematch(item, message):
if item["service"] == message["service"] and item["mid"] == message["mid"]:
return True
for item in item["dupes"]:
if item["service"] == message["service"] and item["mid"] == message["mid"]:
return True
# Detect duplicates
for n, message in enumerate(messages):
message["is_dupe"] = message["txtid"] in seen
if message["is_dupe"]:
item = messages[seen[message["txtid"]]]
if not dupematch(item, message):
item["dupes"].append(message)
else:
if message["txtid"]:
seen[message["txtid"]] = n
messages = [m for m in messages if not m["is_dupe"]]
WebUi.render(self, self.model.settings["theme"], "template.mako",
message_store=messages,
preferences=self.model.settings,
services=self.model.services,
accounts=accounts)
return [m["txtid"] for m in messages]
def render(self, streams, count = 100):
accounts = json.loads(self.model.accounts.List())
accounts = dict((a["id"], a) for a in accounts)
messages = []
seen = {}
if len(streams) == 1 and "sql" in streams[0]:
with sqlite3.connect(SQLITE_DB_FILENAME) as db:
data = db.execute("%s order by time desc limit %s" % (streams[0]["sql"], count)).fetchall()
msgs = json.loads("[%s]" % ",".join(i[0] for i in data))
for item in msgs:
message = item
message["dupes"] = []
message["txtid"] = util.remove_urls(message["text"]).strip()[:MAX_MESSAGE_LENGTH] or None
message["color"] = util.Color(accounts.get(message["account"], {"color": "#5A5A5A"})["color"])
message["time_string"] = util.generate_time_string(message["time"])
try:
message["sender"]["image"] = resources.get_avatar_path(message["sender"]["image"])
except:
pass
messages.append(message)
def dupematch(item, message):
if item["service"] == message["service"] and item["mid"] == message["mid"]:
return True
for item in item["dupes"]:
if item["service"] == message["service"] and item["mid"] == message["mid"]:
return True
# Detect duplicates
for n, message in enumerate(messages):
message["is_dupe"] = message["txtid"] in seen
if message["is_dupe"]:
item = messages[seen[message["txtid"]]]
if not dupematch(item, message):
item["dupes"].append(message)
else:
if message["txtid"]:
seen[message["txtid"]] = n
messages = [m for m in messages if not m["is_dupe"]]
WebUi.render(self, self.model.settings["theme"], "template.mako",
message_store=messages,
preferences=self.model.settings,
services=self.model.services,
accounts=accounts)
return [m["txtid"] for m in messages]
