def _text_sanitizer(self, text): text = remove_non_asciiprintable(text, ' ') text = remove_urls(text, ' ') text = remove_twitter_pic_urls(text, ' ') text = remove_emails(text, ' ') text = text.lower() text = text.strip() return text
def _text_sanitizer(self, text): text = html.unescape(text) text = remove_non_asciiprintable(text, ' ') text = remove_urls(text, ' ') text = remove_emails(text, ' ') text = text.lower() text = text.strip() return text
def _text_sanitizer(self, text): text = remove_non_asciiprintable(text, ' ') text = remove_urls(text, ' ') text = remove_emails(text, ' ') text = remove_html_comments(text, ' ') text = remove_markdown_codeblocks(text, ' ') text = text.lower() text = text.strip() return text
def render(self, streams, count = 100): accounts = json.loads(self.model.accounts.List()) accounts = dict((a["id"], a) for a in accounts) messages = [] seen = {} time = 0 orderby = "time" order = "desc" limit = 50 if len(streams) == 1: # get message data for selected stream # stream, account, time, transient, recipient, orderby, order, limit msgs = json.loads(self.model.streams.Messages(streams[0]["stream"] or "all", streams[0]["account"] or "all", time, streams[0]["transient"] or "0", streams[0].has_key("recipient") and streams[0]["recipient"] or "0", orderby, order, limit)) for item in msgs: message = item message["dupes"] = [] message["txtid"] = util.remove_urls(message["text"]).strip()[:MAX_MESSAGE_LENGTH] or None message["color"] = util.Color(accounts.get(message["account"], {"color": "#5A5A5A"})["color"]) message["time_string"] = util.generate_time_string(message["time"]) try: message["sender"]["image"] = resources.get_avatar_path(message["sender"]["image"]) except: pass messages.append(message) def dupematch(item, message): if item["service"] == message["service"] and item["mid"] == message["mid"]: return True for item in item["dupes"]: if item["service"] == message["service"] and item["mid"] == message["mid"]: return True # Detect duplicates for n, message in enumerate(messages): message["is_dupe"] = message["txtid"] in seen if message["is_dupe"]: item = messages[seen[message["txtid"]]] if not dupematch(item, message): item["dupes"].append(message) else: if message["txtid"]: seen[message["txtid"]] = n messages = [m for m in messages if not m["is_dupe"]] WebUi.render(self, self.model.settings["theme"], "template.mako", message_store=messages, preferences=self.model.settings, services=self.model.services, accounts=accounts) return [m["txtid"] for m in messages]
def render(self, streams, count = 100): accounts = json.loads(self.model.accounts.List()) accounts = dict((a["id"], a) for a in accounts) messages = [] seen = {} if len(streams) == 1 and "sql" in streams[0]: with sqlite3.connect(SQLITE_DB_FILENAME) as db: data = db.execute("%s order by time desc limit %s" % (streams[0]["sql"], count)).fetchall() msgs = json.loads("[%s]" % ",".join(i[0] for i in data)) for item in msgs: message = item message["dupes"] = [] message["txtid"] = util.remove_urls(message["text"]).strip()[:MAX_MESSAGE_LENGTH] or None message["color"] = util.Color(accounts.get(message["account"], {"color": "#5A5A5A"})["color"]) message["time_string"] = util.generate_time_string(message["time"]) try: message["sender"]["image"] = resources.get_avatar_path(message["sender"]["image"]) except: pass messages.append(message) def dupematch(item, message): if item["service"] == message["service"] and item["mid"] == message["mid"]: return True for item in item["dupes"]: if item["service"] == message["service"] and item["mid"] == message["mid"]: return True # Detect duplicates for n, message in enumerate(messages): message["is_dupe"] = message["txtid"] in seen if message["is_dupe"]: item = messages[seen[message["txtid"]]] if not dupematch(item, message): item["dupes"].append(message) else: if message["txtid"]: seen[message["txtid"]] = n messages = [m for m in messages if not m["is_dupe"]] WebUi.render(self, self.model.settings["theme"], "template.mako", message_store=messages, preferences=self.model.settings, services=self.model.services, accounts=accounts) return [m["txtid"] for m in messages]