def export_grid(grid, filename, title, db, sql_query="", table=""):
"""
Exports the current contents of the specified wx.Grid to file.
@param grid a wx.Grid object
@param filename full path and filename of resulting file, file extension
.html|.csv|.sql|.xslx determines file format
@param title title used in HTML
@param db SkypeDatabase instance
@param sql_query the SQL query producing the grid contents, if any
@param table name of the table producing the grid contents, if any
"""
result = False
f = None
is_html = filename.lower().endswith(".html")
is_csv = filename.lower().endswith(".csv")
is_sql = filename.lower().endswith(".sql")
is_xlsx = filename.lower().endswith(".xlsx")
try:
with open(filename, "w") as f:
columns = [c["name"] for c in grid.Table.columns]
def iter_rows():
"""Iterating row generator."""
row, index = grid.Table.GetRow(0), 0
while row:
yield row
index += 1
row = grid.Table.GetRow(index)
if is_csv or is_xlsx:
if is_csv:
dialect = csv.excel
dialect.delimiter, dialect.lineterminator = ";", "\r"
writer = csv.writer(f, dialect)
if sql_query:
flat = sql_query.replace("\r", " ").replace("\n", " ")
sql_query = flat.encode("latin1", "replace")
header = [c.encode("latin1", "replace") for c in columns]
else:
writer = xlsx_writer(filename, table or "SQL Query")
writer.set_header(True)
header = columns
if sql_query:
a = [[sql_query]] + (["bold", 0, False] if is_xlsx else [])
writer.writerow(*a)
writer.writerow(*([header, "bold"] if is_xlsx else [header]))
writer.set_header(False) if is_xlsx else 0
for row in iter_rows():
values = []
for col in columns:
val = "" if row[col] is None else row[col]
if is_csv:
val = val if isinstance(val, unicode) else str(val)
val = val.encode("latin1", "replace")
values.append(val)
writer.writerow(values)
writer.close() if is_xlsx else 0
else:
namespace = {
"db_filename": db.filename,
"title": title,
"columns": columns,
"row_count": grid.NumberRows,
"rows": iter_rows(),
"sql": sql_query,
"table": table,
"app": conf.Title,
}
if is_sql and table:
# Add CREATE TABLE statement.
create_sql = db.tables[table.lower()]["sql"] + ";"
re_sql = re.compile("^(CREATE\\s+TABLE\\s+)",
re.IGNORECASE)
replacer = lambda m: ("%sIF NOT EXISTS " % m.group(1))
namespace["create_sql"] = re_sql.sub(replacer, create_sql)
template = templates.GRID_HTML if is_html else templates.SQL_TXT
step.Template(template, strip=False).stream(f, namespace)
result = True
finally:
if f: util.try_until(f.close)
return result
def export_chat_template(chat, filename, db, messages):
"""
Exports the chat messages to file using templates.
@param chat chat data dict, as returned from SkypeDatabase
@param filename full path and filename of resulting file, file extension
.html|.txt determines file format
@param db SkypeDatabase instance
@param messages list of message data dicts
"""
tmpfile, tmpname = None, None # Temporary file for exported messages
try:
is_html = filename.lower().endswith(".html")
parser = skypedata.MessageParser(db, chat=chat, stats=True)
namespace = {
"db": db,
"chat": chat,
"messages": messages,
"parser": parser
}
# As HTML and TXT contain statistics in their headers before
# messages, write out all messages to a temporary file first,
# statistics will be available for the main file after parsing.
# Cannot keep all messages in memory at once - very large chats
# (500,000+ messages) can take gigabytes.
tmpname = util.unique_path("%s.messages" % filename)
tmpfile = open(tmpname, "w+")
mtemplate = templates.CHAT_MESSAGES_HTML if is_html \
else templates.CHAT_MESSAGES_TXT
step.Template(mtemplate, strip=False).stream(tmpfile, namespace)
namespace["stats"] = stats = parser.get_collected_stats()
namespace.update({
"date1":
stats["startdate"].strftime("%d.%m.%Y")
if stats.get("startdate") else "",
"date2":
stats["enddate"].strftime("%d.%m.%Y")
if stats.get("enddate") else "",
"emoticons_used":
[x for x in parser.emoticons_unique if hasattr(emoticons, x)],
"message_count":
stats.get("messages", 0),
})
if is_html:
# Collect chat and participant images.
namespace.update({
"participants": [],
"chat_picture_size": None,
"chat_picture_raw": None,
})
if chat["meta_picture"]:
raw = skypedata.fix_image_raw(chat["meta_picture"])
namespace["chat_picture_raw"] = raw
namespace["chat_picture_size"] = util.img_size(raw)
contacts = dict((c["skypename"], c) for c in db.get_contacts())
partics = dict((p["identity"], p) for p in chat["participants"])
# There can be authors not among participants, and vice versa
for author in stats["authors"].union(partics):
contact = partics.get(author, {}).get("contact")
contact = contact or contacts.get(author, {})
contact = contact or {"identity": author, "name": author}
bmp = contact.get("avatar_bitmap")
raw = contact.get("avatar_raw_small") or ""
raw_large = contact.get("avatar_raw_large") or ""
if not raw and not bmp:
raw = skypedata.get_avatar_raw(contact,
conf.AvatarImageSize)
raw = bmp and util.img_wx_to_raw(bmp) or raw
if raw:
raw_large = raw_large or skypedata.get_avatar_raw(
contact, conf.AvatarImageLargeSize)
contact["avatar_raw_small"] = raw
contact["avatar_raw_large"] = raw_large
contact["rank"] = partics.get(author, {}).get("rank")
namespace["participants"].append(contact)
tmpfile.flush(), tmpfile.seek(0)
namespace["message_buffer"] = iter(lambda: tmpfile.read(65536), "")
template = templates.CHAT_HTML if is_html else templates.CHAT_TXT
with open(filename, "w") as f:
step.Template(template, strip=False).stream(f, namespace)
finally:
if tmpfile: util.try_until(tmpfile.close)
if tmpname: util.try_until(lambda: os.unlink(tmpname))
def work_diff_left(self, params):
"""
Worker branch that compares all chats on the left side for differences,
posting results back to application.
"""
# {"output": "html result for db1, db2",
# "index": currently processed chat index,
# "chats": [differing chats in db1]}
result = {
"output": "",
"chats": [],
"params": params,
"index": 0,
"type": "diff_left"
}
db1, db2 = params["db1"], params["db2"]
chats1 = params.get("chats") or db1.get_conversations()
chats2 = db2.get_conversations()
c1map = dict((c["identity"], c) for c in chats1)
c2map = dict((c["identity"], c) for c in chats2)
compared = []
for c1 in chats1:
c2 = c2map.get(c1["identity"])
c = c1.copy()
c["messages1"] = c1["message_count"] or 0
c["messages2"] = c2["message_count"] or 0 if c2 else 0
c["c1"], c["c2"] = c1, c2
compared.append(c)
compared.sort(key=lambda x: x["title"].lower())
info_template = step.Template(templates.DIFF_RESULT_ITEM)
for index, chat in enumerate(compared):
diff = self.get_chat_diff_left(chat, db1, db2)
if self._stop_work:
break # break for index, chat in enumerate(compared)
if diff["messages"] \
or (chat["message_count"] and diff["participants"]):
new_chat = not chat["c2"]
newstr = "" if new_chat else "new "
info = info_template.expand(chat=chat)
if new_chat:
info += " - new chat"
if diff["messages"]:
info += ", %s" % util.plural("%smessage" % newstr,
diff["messages"])
else:
info += ", no messages"
if diff["participants"] and not new_chat:
info += ", %s" % (util.plural("%sparticipant" % newstr,
diff["participants"]))
info += ".<br />"
result["output"] += info
result["chats"].append({"chat": chat, "diff": diff})
result["index"] = index
if not self._drop_results:
if index < len(compared) - 1:
result["status"] = ("Scanning %s." %
compared[index + 1]["title_long_lc"])
self.postback(result)
result = {
"output": "",
"chats": [],
"index": index,
"params": params,
"type": "diff_left"
}
if not self._drop_results:
result["done"] = True
self.postback(result)
def run(self):
self._is_running = True
# For identifying "chat:xxx" and "from:xxx" keywords
query_parser = searchparser.SearchQueryParser()
result = None
while self._is_running:
try:
search = self._queue.get()
if not search:
continue # continue while self._is_running
is_text_output = ("text" == search.get("output"))
wrap_html = None # MessageParser wrap function, for HTML output
if is_text_output:
TEMPLATES = {
"chat": templates.SEARCH_ROW_CHAT_TXT,
"contact": templates.SEARCH_ROW_CONTACT_TXT,
"message": templates.SEARCH_ROW_MESSAGE_TXT,
"table": templates.SEARCH_ROW_TABLE_HEADER_TXT,
"row": templates.SEARCH_ROW_TABLE_TXT,
}
wrap_b = lambda x: "**%s**" % x.group(0)
output = {"format": "text"}
else:
TEMPLATES = {
"chat": templates.SEARCH_ROW_CHAT_HTML,
"contact": templates.SEARCH_ROW_CONTACT_HTML,
"message": templates.SEARCH_ROW_MESSAGE_HTML,
"table": templates.SEARCH_ROW_TABLE_HEADER_HTML,
"row": templates.SEARCH_ROW_TABLE_HTML,
}
wrap_b = lambda x: "<b>%s</b>" % x.group(0)
output = {"format": "html"}
width = search.get("width", -1)
if width > 0:
dc = wx.MemoryDC()
dc.SetFont(
wx.Font(8,
wx.SWISS,
wx.NORMAL,
wx.NORMAL,
face=conf.HistoryFontName))
wrap_html = lambda x: wx.lib.wordwrap.wordwrap(
x, width, dc)
output["wrap"] = True
main.log("Searching for \"%(text)s\" in %(table)s (%(db)s)." %
search)
self._stop_work = False
self._drop_results = False
parser = skypedata.MessageParser(search["db"],
wrapfunc=wrap_html)
# {"output": text with results, "map": link data map}
# map data: {"contact:666": {"contact": {contact data}}, }
result_type, result_count, count = None, 0, 0
result = {
"output": "",
"map": {},
"search": search,
"count": 0
}
sql, params, match_words = query_parser.Parse(search["text"])
# Turn wildcard characters * into regex-compatible .*
match_words_re = [
".*".join(map(re.escape, w.split("*")))
for w in match_words
]
patt = "(%s)" % "|".join(match_words_re)
# For replacing matching words with <b>words</b>
pattern_replace = re.compile(patt, re.IGNORECASE)
# Find chats with a matching title or matching participants
chats = search["db"].get_conversations()
chats.sort(key=lambda x: x["title"])
chat_map = {} # {chat id: {chat data}}
template_chat = step.Template(TEMPLATES["chat"])
for chat in chats:
chat_map[chat["id"]] = chat
if "conversations" == search["table"] and match_words:
title_matches = False
matching_authors = []
if self.match_all(chat["title"], match_words):
title_matches = True
for participant in chat["participants"]:
contact = participant["contact"]
if contact:
for n in filter(None, [
contact["fullname"],
contact["displayname"],
contact["identity"]
]):
if self.match_all(n, match_words) \
and contact not in matching_authors:
matching_authors.append(contact)
if title_matches or matching_authors:
count += 1
result_count += 1
result["output"] += template_chat.expand(locals())
key = "chat:%s" % chat["id"]
result["map"][key] = {"chat": chat["id"]}
if not count % conf.SearchResultsChunk \
and not self._drop_results:
result["count"] = result_count
self.postback(result)
result = {
"output": "",
"map": {},
"search": search,
"count": 0
}
if self._stop_work:
break # break for chat in chats
if result["output"] and not self._drop_results:
result["count"] = result_count
self.postback(result)
result = {
"output": "",
"map": {},
"search": search,
"count": 0
}
# Find contacts with a matching name
if not self._stop_work and "contacts" == search["table"] \
and match_words:
count = 0
contacts = search["db"].get_contacts()
# Possibly more: country (ISO code, need map), birthday
# (base has YYYYMMDD in integer field).
match_fields = [
"displayname",
"skypename",
"province",
"city",
"pstnnumber",
"phone_home",
"phone_office",
"phone_mobile",
"homepage",
"emails",
"about",
"mood_text",
]
template_contact = step.Template(TEMPLATES["contact"])
for contact in contacts:
match = False
fields_filled = {}
for field in match_fields:
if contact[field]:
val = contact[field]
if self.match_all(val, match_words):
match = True
val = pattern_replace.sub(wrap_b, val)
fields_filled[field] = val
if match:
count += 1
result_count += 1
result["output"] += template_contact.expand(
locals())
if not (self._drop_results
or count % conf.SearchResultsChunk):
result["count"] = result_count
self.postback(result)
result = {
"output": "",
"map": {},
"search": search,
"count": 0
}
if self._stop_work:
break # break for contact in contacts
if result["output"] and not self._drop_results:
result["count"] = result_count
self.postback(result)
result = {
"output": "",
"map": {},
"search": search,
"count": 0
}
# Find messages with a matching body
if not self._stop_work and "messages" == search["table"]:
template_message = step.Template(TEMPLATES["message"])
count, result_type = 0, "messages"
chat_messages = {} # {chat id: [message, ]}
chat_order = [] # [chat id, ]
messages = search["db"].get_messages(
additional_sql=sql,
additional_params=params,
ascending=False,
use_cache=False)
for m in messages:
chat = chat_map.get(m["convo_id"])
body = parser.parse(
m, pattern_replace if match_words else None,
output)
count += 1
result_count += 1
result["output"] += template_message.expand(locals())
key = "message:%s" % m["id"]
result["map"][key] = {
"chat": chat["id"],
"message": m["id"]
}
if is_text_output or (
not self._drop_results
and not count % conf.SearchResultsChunk):
result["count"] = result_count
self.postback(result)
result = {
"output": "",
"map": {},
"search": search,
"count": 0
}
if self._stop_work or (not is_text_output and count >=
conf.SearchMessagesMax):
break # break for m in messages
infotext = search["table"]
if not self._stop_work and "all tables" == search["table"]:
infotext, result_type = "", "table row"
# Search over all fields of all tables.
template_table = step.Template(TEMPLATES["table"])
template_row = step.Template(TEMPLATES["row"])
for table in search["db"].get_tables():
table["columns"] = search["db"].get_table_columns(
table["name"])
sql, params, words = \
query_parser.Parse(search["text"], table)
if not sql:
continue # continue for table in search["db"]..
infotext += (", " if infotext else "") + table["name"]
rows = search["db"].execute(sql, params)
row = rows.fetchone()
if not row:
continue # continue for table in search["db"]..
result["output"] = template_table.expand(locals())
count = 0
while row:
count += 1
result_count += 1
result["output"] += template_row.expand(locals())
key = "table:%s:%s" % (table["name"], count)
result["map"][key] = {
"table": table["name"],
"row": row
}
if not count % conf.SearchResultsChunk \
and not self._drop_results:
result["count"] = result_count
self.postback(result)
result = {
"output": "",
"map": {},
"search": search,
"count": 0
}
if self._stop_work or (not is_text_output
and result_count >=
conf.SearchTableRowsMax):
break # break while row
row = rows.fetchone()
if not self._drop_results:
if not is_text_output:
result["output"] += "</table>"
result["count"] = result_count
self.postback(result)
result = {
"output": "",
"map": {},
"search": search,
"count": 0
}
infotext += " (%s)" % util.plural("result", count)
if self._stop_work or (
not is_text_output
and result_count >= conf.SearchTableRowsMax):
break # break for table in search["db"]..
single_table = ("," not in infotext)
infotext = "table%s: %s" % \
("" if single_table else "s", infotext)
if not single_table:
infotext += "; %s in total" % \
util.plural("result", result_count)
final_text = "No matches found."
if self._drop_results:
result["output"] = ""
if result_count:
final_text = "Finished searching %s." % infotext
if self._stop_work:
final_text += " Stopped by user."
elif "messages" == result_type and not is_text_output \
and count >= conf.SearchMessagesMax:
final_text += " Stopped at %s limit %s." % \
(result_type, conf.SearchMessagesMax)
elif "table row" == result_type and not is_text_output \
and count >= conf.SearchTableRowsMax:
final_text += " Stopped at %s limit %s." % \
(result_type, conf.SearchTableRowsMax)
result[
"output"] += "</table><br /><br />%s</font>" % final_text
if is_text_output: result["output"] = ""
result["done"] = True
result["count"] = result_count
self.postback(result)
main.log("Search found %(count)s results." % result)
except Exception as e:
if not result:
result = {}
result["done"], result["error"] = True, traceback.format_exc()
result["error_short"] = "%s: %s" % (type(e).__name__,
e.message)
self.postback(result)
def export_chat_template(chat, filename, db, messages):
"""
Exports the chat messages to file using templates.
@param chat chat data dict, as returned from SkypeDatabase
@param filename full path and filename of resulting file, file extension
.html|.txt determines file format
@param db SkypeDatabase instance
@param messages list of message data dicts
"""
tmpfile, tmpname = None, None # Temporary file for exported messages
try:
is_html = filename.lower().endswith(".html")
parser = skypedata.MessageParser(db, chat=chat, stats=is_html)
namespace = {
"db": db,
"chat": chat,
"messages": messages,
"parser": parser
}
if is_html:
# Collect chat and participant images.
namespace.update({
"participants": [],
"chat_picture_size": None,
"chat_picture_raw": None,
})
if chat["meta_picture"]:
raw = skypedata.fix_image_raw(chat["meta_picture"])
imgparser = ImageFile.Parser()
imgparser.feed(raw)
img = imgparser.close()
namespace.update(chat_picture_size=img.size,
chat_picture_raw=raw)
for p in chat["participants"]:
contact = p["contact"].copy()
namespace["participants"].append(contact)
contact.update(avatar_raw_small="", avatar_raw_large="")
bmp = contact.get("avatar_bitmap")
raw = contact.get("avatar_raw_small")
raw_large = contact.get("avatar_raw_large")
if not raw and not bmp:
raw = skypedata.get_avatar_raw(contact,
conf.AvatarImageSize)
if raw:
p["contact"]["avatar_raw_small"] = raw
raw = bmp and util.wx_bitmap_to_raw(bmp) or raw
if raw:
if not raw_large:
size_large = conf.AvatarImageLargeSize
raw_large = skypedata.get_avatar_raw(
contact, size_large)
p["contact"]["avatar_raw_large"] = raw_large
contact["avatar_raw_small"] = raw
contact["avatar_raw_large"] = raw_large
# As HTML and TXT contain statistics in their headers before
# messages, write out all messages to a temporary file first,
# statistics will be available for the main file after parsing.
# Cannot keep all messages in memory at once - very large chats
# (500,000+ messages) can take gigabytes.
tmpname = util.unique_path("%s.messages" % filename)
tmpfile = open(tmpname, "w+")
mtemplate = templates.CHAT_MESSAGES_HTML if is_html \
else templates.CHAT_MESSAGES_TXT
step.Template(mtemplate, strip=False).stream(tmpfile, namespace)
namespace["stats"] = stats = parser.get_collected_stats()
namespace.update({
"date1":
stats["startdate"].strftime("%d.%m.%Y")
if stats.get("startdate") else "",
"date2":
stats["enddate"].strftime("%d.%m.%Y")
if stats.get("enddate") else "",
"emoticons_used":
list(
filter(lambda e: hasattr(emoticons, e),
parser.emoticons_unique)),
"message_count":
stats.get("messages", 0),
})
tmpfile.flush(), tmpfile.seek(0)
namespace["message_buffer"] = iter(lambda: tmpfile.read(65536), "")
template = templates.CHAT_HTML if is_html else templates.CHAT_TXT
with open(filename, "w") as f:
step.Template(template, strip=False).stream(f, namespace)
finally:
if tmpfile: util.try_until(tmpfile.close)
if tmpname: util.try_until(lambda: os.unlink(tmpname))