def to_templated_secular_display(self, fp, group, type='lecture'):
gt = GroupTemplate(None)
gt.add_patterns(fp)
gt.calculate_reduction(True)
out = []
for (_, _, (pattern, mult)) in gt.get_patterns_raw(False):
if pattern is None:
# just multiplier, should be just "x lectures"
row = "%d %s, %s" % (mult, util.plural(type), gt.template
) # XXX proper plurals
elif mult is None:
# traditional pattern, expand codes sensibly
row = "%s %s" % (util.plural(type),
self.atoms_to_secular(fp.patterns(), True))
else:
# start date and multiplier
pp = copy.deepcopy(gt.template)
pp.setAllYear()
row = "%s %s, starting on %s, %s" % (mult, util.plural(type),
self.atoms_to_secular(
[pattern], False), pp)
out.append(row)
prefix = ''
if group is not None:
prefix = "%s term, " % term_names[group.term]
return prefix + ", ".join(out)
def run_merge(filenames):
"""Merges all Skype databases to a new database."""
dbs = [skypedata.SkypeDatabase(f) for f in filenames]
db_base = dbs.pop()
counts = collections.defaultdict(lambda: collections.defaultdict(int))
postbacks = Queue.Queue()
postfunc = lambda r: postbacks.put(r)
worker = workers.MergeThread(postfunc)
name, ext = os.path.splitext(os.path.split(db_base.filename)[-1])
now = datetime.datetime.now().strftime("%Y%m%d")
filename_final = util.unique_path("%s.merged.%s%s" % (name, now, ext))
print("Creating %s, using %s as base." % (filename_final, db_base))
shutil.copyfile(db_base.filename, filename_final)
db2 = skypedata.SkypeDatabase(filename_final)
chats2 = db2.get_conversations()
db2.get_conversations_stats(chats2)
for db1 in dbs:
chats = db1.get_conversations()
db1.get_conversations_stats(chats)
bar_total = sum(c["message_count"] for c in chats)
bar_text = " Processing %.*s.." % (30, db1)
bar = ProgressBar(max=bar_total, afterword=bar_text)
bar.start()
args = {"db1": db1, "db2": db2, "chats": chats,
"type": "diff_merge_left"}
worker.work(args)
while True:
result = postbacks.get()
if "error" in result:
print("Error merging %s:\n\n%s" % (db1, result["error"]))
worker = None # Signal for global break
break # break while True
if "done" in result:
break # break while True
if "diff" in result:
counts[db1]["chats"] += 1
counts[db1]["msgs"] += len(result["diff"]["messages"])
msgcounts = sum(c["message_count"] for c in result["chats"])
bar.update(bar.value + msgcounts)
if result["output"]:
log(result["output"])
if not worker:
break # break for db1 in dbs
bar.stop()
bar.afterword = " Processed %s." % db1
bar.update(bar_total)
print
if not counts:
print("Nothing new to merge.")
db2.close()
os.unlink(filename_final)
else:
for db1 in dbs:
print("Merged %s in %s from %s." %
(util.plural("message", counts[db1]["msgs"]),
util.plural("chat", counts[db1]["chats"]), db1))
print("Merge into %s complete." % db2)
def work_diff_left(self, params):
"""
Worker branch that compares all chats on the left side for differences,
posting results back to application.
"""
# {"output": "html result for db1, db2",
# "index": currently processed chat index,
# "chats": [differing chats in db1]}
result = {"output": "", "chats": [],
"params": params, "index": 0, "type": "diff_left"}
db1, db2 = params["db1"], params["db2"]
chats1 = params.get("chats") or db1.get_conversations()
chats2 = db2.get_conversations()
c1map = dict((c["identity"], c) for c in chats1)
c2map = dict((c["identity"], c) for c in chats2)
compared = []
for c1 in chats1:
c2 = c2map.get(c1["identity"])
c = c1.copy()
c["messages1"] = c1["message_count"] or 0
c["messages2"] = c2["message_count"] or 0 if c2 else 0
c["c1"], c["c2"] = c1, c2
compared.append(c)
compared.sort(key=lambda x: x["title"].lower())
info_template = step.Template(templates.DIFF_RESULT_ITEM)
for index, chat in enumerate(compared):
diff = self.get_chat_diff_left(chat, db1, db2)
if self._stop_work:
break # break for index, chat in enumerate(compared)
if diff["messages"] \
or (chat["message_count"] and diff["participants"]):
new_chat = not chat["c2"]
newstr = "" if new_chat else "new "
info = info_template.expand(chat=chat)
if new_chat:
info += " - new chat"
if diff["messages"]:
info += ", %s" % util.plural("%smessage" % newstr,
diff["messages"])
else:
info += ", no messages"
if diff["participants"] and not new_chat:
info += ", %s" % (
util.plural("%sparticipant" % newstr,
diff["participants"]))
info += ".<br />"
result["output"] += info
result["chats"].append({"chat": chat, "diff": diff})
result["index"] = index
if not self._drop_results:
if index < len(compared) - 1:
result["status"] = ("Scanning %s." %
compared[index + 1]["title_long_lc"])
self.postback(result)
result = {"output": "", "chats": [], "index": index,
"params": params, "type": "diff_left"}
if not self._drop_results:
result["done"] = True
self.postback(result)
def flush(self):
try:
self.flush_count += 1
self.log_count += 1
packets_per_second = self.metrics_aggregator.packets_per_second(
self.interval)
packet_count = self.metrics_aggregator.total_count
metrics = self.metrics_aggregator.flush()
count = len(metrics)
if self.flush_count % FLUSH_LOGGING_PERIOD == 0:
self.log_count = 0
if count:
self.submit(metrics)
events = self.metrics_aggregator.flush_events()
event_count = len(events)
if event_count:
self.submit_events(events)
service_checks = self.metrics_aggregator.flush_service_checks()
service_check_count = len(service_checks)
if service_check_count:
self.submit_service_checks(service_checks)
should_log = self.flush_count <= FLUSH_LOGGING_INITIAL or self.log_count <= FLUSH_LOGGING_COUNT
log_func = log.info
if not should_log:
log_func = log.debug
log_func(
"Flush #%s: flushed %s metric%s, %s event%s, and %s service check run%s"
% (self.flush_count, count, plural(count), event_count,
plural(event_count), service_check_count,
plural(service_check_count)))
if self.flush_count == FLUSH_LOGGING_INITIAL:
log.info(
"First flushes done, %s flushes will be logged every %s flushes."
% (FLUSH_LOGGING_COUNT, FLUSH_LOGGING_PERIOD))
# Persist a status message.
packet_count = self.metrics_aggregator.total_count
MonitorstatsdStatus(
flush_count=self.flush_count,
packet_count=packet_count,
packets_per_second=packets_per_second,
metric_count=count,
event_count=event_count,
service_check_count=service_check_count,
).persist()
except Exception:
if self.finished.isSet():
log.debug(
"Couldn't flush metrics, but that's expected as we're stopping"
)
else:
log.exception("Error flushing metrics")
def check_status_lines(cs):
check_lines = [" " + cs.name, " " + "-" * len(cs.name)]
if cs.init_failed_error:
check_lines.append(
" - initialize check class [%s]: %s" % (style(STATUS_ERROR, "red"), repr(cs.init_failed_error))
)
if cs.init_failed_traceback:
check_lines.extend(" " + line for line in cs.init_failed_traceback.split("\n"))
else:
for s in cs.instance_statuses:
c = "green"
if s.has_warnings():
c = "yellow"
if s.has_error():
c = "red"
line = " - instance #%s [%s]" % (s.instance_id, style(s.status, c))
if s.has_error():
line += u": %s" % s.error
if s.metric_count is not None:
line += " collected %s metrics" % s.metric_count
if s.instance_check_stats is not None:
line += " Last run duration: %s" % s.instance_check_stats.get("run_time")
check_lines.append(line)
if s.has_warnings():
for warning in s.warnings:
warn = warning.split("\n")
if not len(warn):
continue
check_lines.append(u" %s: %s" % (style("Warning", "yellow"), warn[0]))
check_lines.extend(u" %s" % l for l in warn[1:])
if s.traceback is not None:
check_lines.extend(" " + line for line in s.traceback.split("\n"))
check_lines += [
" - Collected %s metric%s, %s event%s & %s service check%s"
% (
cs.metric_count,
plural(cs.metric_count),
cs.event_count,
plural(cs.event_count),
cs.service_check_count,
plural(cs.service_check_count),
)
]
if cs.check_stats is not None:
check_lines += [" - Stats: %s" % pretty_statistics(cs.check_stats)]
if cs.library_versions is not None:
check_lines += [" - Dependencies:"]
for library, version in cs.library_versions.iteritems():
check_lines += [" - %s: %s" % (library, version)]
check_lines += [""]
return check_lines
def flush(self):
if self._trs_to_flush is not None:
log.debug("A flush is already in progress, not doing anything")
return
to_flush = []
# Do we have something to do ?
now = datetime.utcnow()
for tr in self._transactions:
if tr.time_to_flush(now):
to_flush.append(tr)
count = len(to_flush)
should_log = self._flush_count + 1 <= FLUSH_LOGGING_INITIAL or (
self._flush_count + 1) % FLUSH_LOGGING_PERIOD == 0
if count > 0:
if should_log:
log.info("Flushing %s transaction%s during flush #%s" %
(count, plural(count), str(self._flush_count + 1)))
else:
log.debug("Flushing %s transaction%s during flush #%s" %
(count, plural(count), str(self._flush_count + 1)))
self._endpoints_errors = {}
self._finished_flushes = 0
# We sort LIFO-style, taking into account errors
self._trs_to_flush = sorted(to_flush,
key=lambda tr:
(-tr._error_count, tr._id))
self._flush_time = datetime.utcnow()
self.flush_next()
else:
if should_log:
log.info("No transaction to flush during flush #%s" %
str(self._flush_count + 1))
else:
log.debug("No transaction to flush during flush #%s" %
str(self._flush_count + 1))
if self._flush_count + 1 == FLUSH_LOGGING_INITIAL:
log.info(
"First flushes done, next flushes will be logged every %s flushes."
% FLUSH_LOGGING_PERIOD)
self._flush_count += 1
ForwarderStatus(
queue_length=self._total_count,
queue_size=self._total_size,
flush_count=self._flush_count,
transactions_received=self._transactions_received,
transactions_flushed=self._transactions_flushed,
transactions_rejected=self._transactions_rejected).persist()
def work_merge(self, params):
"""
Worker branch that merges differences given in params, posting progress
back to application.
"""
error, e = None, None
db1, db2, info = params["db1"], params["db2"], params["info"]
chats, contacts = params["chats"], params["contacts"]
source, contactgroups = params["source"], params["contactgroups"]
count_messages = 0
count_participants = 0
try:
if contacts:
content = util.plural("contact", contacts)
self.postback({"type": "merge", "gauge": 0,
"message": "Merging %s." % content})
db2.insert_contacts(contacts, db1)
self.postback({"type": "merge", "gauge": 100,
"message": "Merged %s." % content})
if contactgroups:
content = util.plural("contact group", contactgroups)
self.postback({"type": "merge", "gauge": 0,
"message": "Merging %s." % content})
db2.replace_contactgroups(contactgroups, db1)
self.postback({"type": "merge", "gauge": 100,
"message": "Merged %s." % content})
for index, chat_data in enumerate(chats):
if self._stop_work:
break # break for i, chat_data in enumerate(chats)
chat1 = chat_data["chat"]["c2" if source else "c1"]
chat2 = chat_data["chat"]["c1" if source else "c2"]
step = -1 if source else 1
messages1, messages2 = chat_data["diff"]["messages"][::step]
participants, participants2 = \
chat_data["diff"]["participants"][::step]
if not chat2:
chat2 = chat1.copy()
chat_data["chat"]["c1" if source else "c2"] = chat2
chat2["id"] = db2.insert_chat(chat2, db1)
if participants:
db2.insert_participants(chat2, participants, db1)
count_participants += len(participants)
if messages1:
db2.insert_messages(chat2, messages1, db1, chat1,
self.yield_ui, self.REFRESH_COUNT)
count_messages += len(messages1)
self.postback({"type": "merge", "index": index,
"params": params})
except Exception, e:
error = traceback.format_exc()
def run_diff(filename1, filename2):
"""Compares the first database for changes with the second."""
if os.path.realpath(filename1) == os.path.realpath(filename2):
output("Error: cannot compare %s with itself." % filename1)
return
db1, db2 = map(skypedata.SkypeDatabase, [filename1, filename2])
counts = collections.defaultdict(lambda: collections.defaultdict(int))
postbacks = Queue.Queue()
bar_text = "%.*s.." % (50, " Scanning %s vs %s" % (db1, db2))
bar = ProgressBar(afterword=bar_text)
bar.start()
chats1, chats2 = db1.get_conversations(), db2.get_conversations()
db1.get_conversations_stats(chats1), db2.get_conversations_stats(chats2)
args = {"db1": db1, "db2": db2, "chats": chats1, "type": "diff_left"}
worker = workers.MergeThread(postbacks.put)
try:
worker.work(args)
while True:
result = postbacks.get()
if "error" in result:
output("Error scanning %s and %s:\n\n%s" %
(db1, db2, result["error"]))
break # break while True
if "done" in result:
break # break while True
if "chats" in result and result["chats"]:
counts[db1]["chats"] += 1
msgs = len(result["chats"][0]["diff"]["messages"])
msgs_text = util.plural("new message", msgs)
contacts_text = util.plural(
"new participant",
result["chats"][0]["diff"]["participants"])
text = ", ".join(filter(None, [msgs_text, contacts_text]))
bar.afterword = (" %s, %s." %
(result["chats"][0]["chat"]["title"], text))
counts[db1]["msgs"] += msgs
if "index" in result:
bar.max = result["count"]
bar.update(result["index"])
if result.get("output"):
log(result["output"])
finally:
worker and (worker.stop(), worker.join())
bar.stop()
bar.afterword = " Scanned %s and %s." % (db1, db2)
bar.update(bar.max)
output()
def run_diff(filename1, filename2):
"""Compares the first database for changes with the second."""
if os.path.realpath(filename1) == os.path.realpath(filename2):
output("Error: cannot compare %s with itself." % filename1)
return
db1, db2 = map(skypedata.SkypeDatabase, [filename1, filename2])
counts = collections.defaultdict(lambda: collections.defaultdict(int))
postbacks = Queue.Queue()
bar_text = "%.*s.." % (50, " Scanning %s vs %s" % (db1, db2))
bar = ProgressBar(afterword=bar_text)
bar.start()
chats1, chats2 = db1.get_conversations(), db2.get_conversations()
db1.get_conversations_stats(chats1), db2.get_conversations_stats(chats2)
args = {"db1": db1, "db2": db2, "chats": chats1, "type": "diff_left"}
worker = workers.MergeThread(postbacks.put)
try:
worker.work(args)
while True:
result = postbacks.get()
if "error" in result:
output("Error scanning %s and %s:\n\n%s" %
(db1, db2, result["error"]))
break # break while True
if "done" in result:
break # break while True
if "chats" in result and result["chats"]:
counts[db1]["chats"] += 1
msgs = len(result["chats"][0]["diff"]["messages"])
msgs_text = util.plural("new message", msgs)
contacts_text = util.plural("new participant",
result["chats"][0]["diff"]["participants"])
text = ", ".join(filter(None, [msgs_text, contacts_text]))
bar.afterword = (" %s, %s." % (result["chats"][0]["chat"]["title"],
text))
counts[db1]["msgs"] += msgs
if "index" in result:
bar.max = result["count"]
bar.update(result["index"])
if result.get("output"):
log(result["output"])
finally:
worker and (worker.stop(), worker.join())
bar.stop()
bar.afterword = " Scanned %s and %s." % (db1, db2)
bar.update(bar.max)
output()
def run_hooks_for(trigger):
from sys import exit
from os.path import sep
from subprocess import call
global _triggers
if trigger not in _triggers:
raise ValueError("unknown trigger: '" + str(trigger) + "'")
hooks = list(set(_hooks[trigger]) - set(_hooks_done[trigger]))
num_done = 0
if len(hooks) > 0:
util.info("running hooks for trigger '" + str(trigger) + "'")
for fname in hooks:
rv = call(config.hooks_dir + sep + fname, env=_create_env())
_hooks_done[trigger].append(fname)
num_done += 1
if rv != 0:
util.error("hook '" + str(fname) + "' exited abnormally")
util.exit(util.ERR_ABNORMAL_HOOK_EXIT)
util.info("successfully ran " + str(num_done) + " " + \
util.plural('hook', num_done))
def tr_error(self, tr):
tr.inc_error_count()
tr.compute_next_flush(self._MAX_WAIT_FOR_REPLAY)
log.warn(
"Transaction %d in error (%s error%s), it will be replayed after %s"
% (tr.get_id(), tr.get_error_count(), plural(
tr.get_error_count()), tr.get_next_flush()))
def tr_error(self, tr):
self._running_flushes -= 1
self._finished_flushes += 1
tr.inc_error_count()
tr.compute_next_flush(self._MAX_WAIT_FOR_REPLAY)
log.warn("Transaction %d in error (%s error%s), it will be replayed after %s",
tr.get_id(),
tr.get_error_count(),
plural(tr.get_error_count()),
tr.get_next_flush())
self._endpoints_errors[tr._endpoint] = self._endpoints_errors.get(tr._endpoint, 0) + 1
# Endpoint failed too many times, it's probably an enpoint issue
# Let's avoid blocking on it
if self._endpoints_errors[tr._endpoint] == self._MAX_ENDPOINT_ERRORS:
new_trs_to_flush = []
for transaction in self._trs_to_flush:
if transaction._endpoint != tr._endpoint:
new_trs_to_flush.append(transaction)
else:
transaction.compute_next_flush(self._MAX_WAIT_FOR_REPLAY)
log.debug('Endpoint %s seems down, removed %s transaction from current flush',
tr._endpoint,
len(self._trs_to_flush) - len(new_trs_to_flush))
self._trs_to_flush = new_trs_to_flush
def tr_error(self, tr):
tr.inc_error_count()
tr.compute_next_flush(self._MAX_WAIT_FOR_REPLAY)
log.warn(
"Transaction %d in error (%s error%s), it will be replayed after %s"
% (tr.get_id(), tr.get_error_count(), plural(tr.get_error_count()), tr.get_next_flush())
)
def tr_error(self, tr):
self._running_flushes -= 1
self._finished_flushes += 1
tr.inc_error_count()
tr.compute_next_flush(self._MAX_WAIT_FOR_REPLAY)
log.warn(
"Transaction %d in error (%s error%s), it will be replayed after %s",
tr.get_id(), tr.get_error_count(), plural(tr.get_error_count()),
tr.get_next_flush())
self._endpoints_errors[tr._endpoint] = self._endpoints_errors.get(
tr._endpoint, 0) + 1
# Endpoint failed too many times, it's probably an enpoint issue
# Let's avoid blocking on it
if self._endpoints_errors[tr._endpoint] == self._MAX_ENDPOINT_ERRORS:
new_trs_to_flush = []
for transaction in self._trs_to_flush:
if transaction._endpoint != tr._endpoint:
new_trs_to_flush.append(transaction)
else:
transaction.compute_next_flush(self._MAX_WAIT_FOR_REPLAY)
log.debug(
'Endpoint %s seems down, removed %s transaction from current flush',
tr._endpoint,
len(self._trs_to_flush) - len(new_trs_to_flush))
self._trs_to_flush = new_trs_to_flush
def flush(self):
if self._trs_to_flush is not None:
log.debug("A flush is already in progress, not doing anything")
return
to_flush = []
# Do we have something to do ?
now = datetime.now()
for tr in self._transactions:
if tr.time_to_flush(now):
to_flush.append(tr)
count = len(to_flush)
should_log = (
self._flush_count + 1 <= FLUSH_LOGGING_INITIAL or (self._flush_count + 1) % FLUSH_LOGGING_PERIOD == 0
)
if count > 0:
if should_log:
log.info(
"Flushing %s transaction%s during flush #%s" % (count, plural(count), str(self._flush_count + 1))
)
else:
log.debug(
"Flushing %s transaction%s during flush #%s" % (count, plural(count), str(self._flush_count + 1))
)
self._trs_to_flush = to_flush
self.flush_next()
else:
if should_log:
log.info("No transaction to flush during flush #%s" % str(self._flush_count + 1))
else:
log.debug("No transaction to flush during flush #%s" % str(self._flush_count + 1))
if self._flush_count + 1 == FLUSH_LOGGING_INITIAL:
log.info("First flushes done, next flushes will be logged every %s flushes." % FLUSH_LOGGING_PERIOD)
self._flush_count += 1
ForwarderStatus(
queue_length=self._total_count,
queue_size=self._total_size,
flush_count=self._flush_count,
transactions_received=self._transactions_received,
transactions_flushed=self._transactions_flushed,
).persist()
def export_chats(chats, path, format, db, messages=None, skip=True, progress=None):
"""
Exports the specified chats from the database under path.
@param chats list of chat dicts, as returned from SkypeDatabase
@param path full path of directory where to save
@param format export format (html|txt|xlsx|csv|filename.ext).
If format is filename.ext, a single file is created:
for single chat exports and multi chat XLSX exports
(for multi-file exports, filenames are named by chats).
@param db SkypeDatabase instance
@param messages list messages to export if a single chat
@param skip whether to skip chats with no messages
@param progress function called before exporting each chat, with the
number of messages exported so far
@return (list of exported filenames, number of chats exported)
"""
files, count = [], 0
def make_filename(chat):
if len(format) > 4: # Filename already given in format
filename = os.path.join(path, format)
else:
args = collections.defaultdict(str); args.update(chat)
filename = "%s.%s" % (conf.ExportChatTemplate % args, format)
filename = os.path.join(path, util.safe_filename(filename))
filename = util.unique_path(filename)
return filename
main.logstatus("Exporting %s from %s %sto %s.",
util.plural("chat", chats), db.filename,
"" if len(format) > 4 else "as %s " % format.upper(),
format if len(format) > 4 else path)
if format.lower().endswith(".xlsx"):
filename = make_filename(chats[0])
count = export_chats_xlsx(chats, filename, db, messages, skip, progress)
files.append(filename)
else:
if not os.path.exists(path):
os.makedirs(path)
export_func = (export_chats_xlsx if format.lower().endswith("xlsx")
else export_chat_csv if format.lower().endswith("csv")
else export_chat_template)
message_count = 0
for chat in chats:
if skip and not messages and not chat["message_count"]:
main.log("Skipping exporting %s: no messages.",
chat["title_long_lc"])
if progress: progress(message_count)
continue # continue for chat in chats
main.status("Exporting %s.", chat["title_long_lc"])
if progress: progress(message_count)
filename = make_filename(chat)
msgs = messages or db.get_messages(chat)
chatarg = [chat] if "xlsx" == format.lower() else chat
export_func(chatarg, filename, db, msgs)
message_count += chat["message_count"]
files.append(filename)
count = len(files)
return (files, count)
def run_diff(filename1, filename2):
"""Compares the first database for changes with the second."""
if os.path.realpath(filename1) == os.path.realpath(filename2):
print("Error: cannot compare %s with itself." % filename1)
return
db1, db2 = map(skypedata.SkypeDatabase, [filename1, filename2])
counts = collections.defaultdict(lambda: collections.defaultdict(int))
postbacks = Queue.Queue()
postfunc = lambda r: postbacks.put(r)
worker = workers.MergeThread(postfunc)
chats1, chats2 = db1.get_conversations(), db2.get_conversations()
db1.get_conversations_stats(chats1), db2.get_conversations_stats(chats2)
for db in [db1, db2]:
db.get_conversations_stats(db.get_conversations())
bar_total = sum(c["message_count"] for c in chats1)
bar_text = "%.*s.." % (50, " Scanning %s vs %s" % (db1, db2))
bar = ProgressBar(max=bar_total, afterword=bar_text)
bar.start()
args = {"db1": db1, "db2": db2, "chats": chats1, "type": "diff_left"}
worker.work(args)
while True:
result = postbacks.get()
if "error" in result:
print("Error scanning %s and %s:\n\n%s" %
(db1, db2, result["error"]))
worker = None # Signal for global break
break # break while True
if "done" in result:
break # break while True
if "chats" in result and result["chats"]:
counts[db1]["chats"] += 1
msgs = len(result["chats"][0]["diff"]["messages"])
msgs_text = util.plural("new message", msgs)
contacts_text = util.plural("new participant",
result["chats"][0]["diff"]["participants"])
text = ", ".join(filter(None, [msgs_text, contacts_text]))
print("%s, %s." % (result["chats"][0]["chat"]["title_long"], text))
counts[db1]["msgs"] += msgs
bar.update(bar.value + result["chats"][0]["chat"]["message_count"])
if result["output"]:
log(result["output"])
bar.stop()
bar.afterword = " Scanned %s and %s." % (db1, db2)
bar.update(bar_total)
print
def flush(self):
if self._trs_to_flush is not None:
log.debug("A flush is already in progress, not doing anything")
return
to_flush = []
# Do we have something to do ?
now = datetime.utcnow()
for tr in self._transactions:
if tr.time_to_flush(now):
to_flush.append(tr)
count = len(to_flush)
should_log = self._flush_count + 1 <= FLUSH_LOGGING_INITIAL or (self._flush_count + 1) % FLUSH_LOGGING_PERIOD == 0
if count > 0:
if should_log:
log.info("Flushing %s transaction%s during flush #%s" % (count,plural(count), str(self._flush_count + 1)))
else:
log.debug("Flushing %s transaction%s during flush #%s" % (count,plural(count), str(self._flush_count + 1)))
self._endpoints_errors = {}
self._finished_flushes = 0
# We sort LIFO-style, taking into account errors
self._trs_to_flush = sorted(to_flush, key=lambda tr: (- tr._error_count, tr._id))
self._flush_time = datetime.utcnow()
self.flush_next()
else:
if should_log:
log.info("No transaction to flush during flush #%s" % str(self._flush_count + 1))
else:
log.debug("No transaction to flush during flush #%s" % str(self._flush_count + 1))
if self._flush_count + 1 == FLUSH_LOGGING_INITIAL:
log.info("First flushes done, next flushes will be logged every %s flushes." % FLUSH_LOGGING_PERIOD)
self._flush_count += 1
ForwarderStatus(
queue_length=self._total_count,
queue_size=self._total_size,
flush_count=self._flush_count,
transactions_received=self._transactions_received,
transactions_flushed=self._transactions_flushed,
transactions_rejected=self._transactions_rejected).persist()
def chooseTracks(self, number, exclude, completion, traceCallback=None,
tags=None):
self._logger.debug("Selecting " + str(number) + " track" +
util.plural(number) + ".")
mycompletion = (lambda thisCallback, trackIDs, completion=completion:
self._chooseTracksCompletion(trackIDs, completion,
thisCallback))
self._chooseTrackIDs(
number, exclude, mycompletion, traceCallback, tags)
def flush(self):
try:
self.flush_count += 1
self.log_count += 1
packets_per_second = self.metrics_aggregator.packets_per_second(
self.interval)
packet_count = self.metrics_aggregator.total_count
metrics = self.metrics_aggregator.flush()
count = len(metrics)
if self.flush_count % FLUSH_LOGGING_PERIOD == 0:
self.log_count = 0
if count:
self.submit(metrics)
events = self.metrics_aggregator.flush_events()
event_count = len(events)
if event_count:
self.submit_events(events)
should_log = self.flush_count <= FLUSH_LOGGING_INITIAL or self.log_count <= FLUSH_LOGGING_COUNT
log_func = log.info
if not should_log:
log_func = log.debug
log_func("Flush #%s: flushed %s metric%s and %s event%s" %
(self.flush_count, count, plural(count), event_count,
plural(event_count)))
if self.flush_count == FLUSH_LOGGING_INITIAL:
log.info(
"First flushes done, %s flushes will be logged every %s flushes."
% (FLUSH_LOGGING_COUNT, FLUSH_LOGGING_PERIOD))
# Persist a status message.
packet_count = self.metrics_aggregator.total_count
DogstatsdStatus(
flush_count=self.flush_count,
packet_count=packet_count,
packets_per_second=packets_per_second,
metric_count=count,
event_count=event_count,
).persist()
except Exception, e:
log.exception("Error flushing metrics")
def flush(self):
if self._trs_to_flush is not None:
log.debug("A flush is already in progress, not doing anything")
return
to_flush = []
now = datetime.utcnow()
for tr in self._transactions:
if tr.time_to_flush(now):
to_flush.append(tr)
count = len(to_flush)
should_log = self._flush_count + 1 <= FLUSH_LOGGING_INITIAL or (
self._flush_count + 1) % FLUSH_LOGGING_PERIOD == 0
if count > 0:
if should_log:
log.info(
"Flushing %s transaction%s during flush #%s" % (count, plural(count), str(self._flush_count + 1)))
else:
log.debug(
"Flushing %s transaction%s during flush #%s" % (count, plural(count), str(self._flush_count + 1)))
self._trs_to_flush = to_flush
self.flush_next()
else:
if should_log:
log.info("No transaction to flush during flush #%s" % str(self._flush_count + 1))
else:
log.debug("No transaction to flush during flush #%s" % str(self._flush_count + 1))
if self._flush_count + 1 == FLUSH_LOGGING_INITIAL:
log.info("First flushes done, next flushes will be logged every %s flushes." % FLUSH_LOGGING_PERIOD)
self._flush_count += 1
ForwarderStatus(
queue_length=self._total_count,
queue_size=self._total_size,
flush_count=self._flush_count,
transactions_received=self._transactions_received,
transactions_flushed=self._transactions_flushed).persist()
def download_events(self):
event_list = self.camera.list_events()
if event_list and len(event_list) > 0:
self.camera.prepare_events(event_list)
total = len(event_list)
logger.info("{} event{} on device".format(total, plural(total),
len(event_list)))
self.prepare_recordings(event_list)
self.add_paths(event_list, "event_filename")
self.camera.download_files(event_list, "filename",
"event_filename")
def to_templated_secular_display(self,fp,group,type= 'lecture'):
gt = GroupTemplate(None)
gt.add_patterns(fp)
gt.calculate_reduction(True)
out = []
for (_,_,(pattern,mult)) in gt.get_patterns_raw(False):
if pattern is None:
# just multiplier, should be just "x lectures"
row = "%d %s, %s" % (mult,util.plural(type),gt.template) # XXX proper plurals
elif mult is None:
# traditional pattern, expand codes sensibly
row = "%s %s" % (util.plural(type),self.atoms_to_secular(fp.patterns(),True))
else:
# start date and multiplier
pp = copy.deepcopy(gt.template)
pp.setAllYear()
row = "%s %s, starting on %s, %s" % (mult,util.plural(type),self.atoms_to_secular([pattern],False),pp)
out.append(row)
prefix = ''
if group is not None:
prefix = "%s term, " % term_names[group.term]
return prefix+", ".join(out)
def _relation_repr(cls, rel):
target = rel.argument
if target and inspect.isfunction(target):
target = target()
if isinstance(target, Mapper):
target = target.class_
target = target.__name__
primaryjoin = ''
lookup = mtl()
foo = rel.key
if rel.primaryjoin is not None and hasattr(
rel.primaryjoin, 'right'):
right_lookup = lookup.get(
rel.primaryjoin.right.table.name,
'%s.c' % rel.primaryjoin.right.table.name)
left_lookup = lookup.get(
rel.primaryjoin.left.table.name,
'%s.c' % rel.primaryjoin.left.table.name)
primaryjoin = ", primaryjoin='%s.%s==%s.%s'" % (
left_lookup, rel.primaryjoin.left.name, right_lookup,
rel.primaryjoin.right.name)
elif hasattr(rel, '_as_string'):
primaryjoin = ', primaryjoin="%s"' % rel._as_string
secondary = ''
secondaryjoin = ''
if rel.secondary is not None:
"""
**HACK**: If there is a secondary relationship like between Venue, Event, and Event_Type, then I'm only
going show a primary relationship.
"Events = relationship('Event', primaryjoin='Venue.id==Event.venue_id')" and not
"Event_Types = relation('EventType', primaryjoin='Venue.id==Event.venue_id', secondary=Event, secondaryjoin='Event.event_type_id==EventType.id')"
"""
if rel.secondary.name in self.table_model_dict:
target = self.table_model_dict[rel.secondary.name]
else:
target = self.find_new_name(
singular(name2label(rel.secondary.name)),
[]) # **HACK**
secondary = ''
secondaryjoin = ''
foo = plural(rel.secondary.name)
backref = ''
# if rel.backref:
# backref=", backref='%s'"%rel.backref.key
return "%s = relationship('%s'%s%s%s%s)" % (
foo, target, primaryjoin, secondary, secondaryjoin,
backref)
def list_events(self):
logger.info("Querying camera for list of events..")
(response, duration,
ts) = self.request(self.get_api_url("APP_EventListReq"),
method='POST',
data="{}")
if response:
data = self.json(response)
if data:
logger.info("{} event{} found in {:.2f}s".format(
data['num'], plural(data['num']),
duration.total_seconds()))
return data.get('event')
return None
def download_files(self, list, key, local_key):
count = 1
skipped = 0
for file in list:
filename = file[key]
localfile = file[local_key]
if os.path.isfile(localfile) and os.stat(localfile).st_size > 0:
skipped += 1
logger.info("{} file{} already downloaded. {} file{} remaining".format(
skipped, plural(skipped),
len(list) - skipped, plural(len(list) - skipped)))
for file in list:
filename = file[key]
localfile = file[local_key]
directory = os.path.dirname(localfile)
if not os.path.exists(directory):
logger.debug("Making dir {}".format(directory))
os.makedirs(directory)
if not os.path.isfile(localfile) or os.stat(
localfile).st_size == 0:
(response, duration,
ts) = self.request(self.get_download_url(filename))
if response:
logger.debug("{}/{}: Downloaded {} in {:.2f}s".format(
count,
len(list) - skipped, filename,
duration.total_seconds()))
with open(localfile, 'wb') as f:
f.write(response.content)
else:
logger.error("Failed to download: {}".format(filename))
count += 1
return list
def run_export(filenames, format):
"""Exports the specified databases in specified format."""
dbs = [skypedata.SkypeDatabase(f) for f in filenames]
is_xlsx_single = ("xlsx_single" == format)
for db in dbs:
formatargs = collections.defaultdict(str)
formatargs["skypename"] = os.path.basename(db.filename)
formatargs.update(db.account or {})
basename = util.safe_filename(conf.ExportDbTemplate % formatargs)
dbstr = "from %s " % db if len(dbs) != 1 else ""
if is_xlsx_single:
export_dir = os.getcwd()
filename = util.unique_path("%s.xlsx" % basename)
else:
export_dir = util.unique_path(os.path.join(os.getcwd(), basename))
filename = format
target = filename if is_xlsx_single else export_dir
try:
print("Exporting as %s %sto %s." %
(format[:4].upper(), dbstr, target))
chats = sorted(db.get_conversations(),
key=lambda x: x["title"].lower())
db.get_conversations_stats(chats)
bar_total = sum(c["message_count"] for c in chats)
bartext = " Exporting %.*s.." % (30, db.filename) # Enforce width
bar = ProgressBar(max=bar_total, afterword=bartext)
bar.start()
result = export.export_chats(chats,
export_dir,
filename,
db,
progress=bar.update)
files, count = result
bar.stop()
if count:
bar.afterword = " Exported %s to %s. " % (db, target)
bar.update(bar_total)
print()
log("Exported %s %sto %s as %s.", util.plural("chat", count),
dbstr, target, format)
else:
print("\nNo messages to export%s." %
("" if len(dbs) == 1 else " from %s" % db))
os.unlink(filename) if is_xlsx_single else os.rmdir(export_dir)
except Exception as e:
print("Error exporting chats: %s\n\n%s" %
(e, traceback.format_exc()))
def run_export(filenames, format):
"""Exports the specified databases in specified format."""
dbs = [skypedata.SkypeDatabase(f) for f in filenames]
is_xlsx_single = ("xlsx_single" == format)
for db in dbs:
formatargs = collections.defaultdict(str)
formatargs["skypename"] = os.path.basename(db.filename)
formatargs.update(db.account or {})
basename = util.safe_filename(conf.ExportDbTemplate % formatargs)
dbstr = "from %s " % db if len(dbs) != 1 else ""
if is_xlsx_single:
export_dir = os.getcwd()
filename = util.unique_path("%s.xlsx" % basename)
else:
export_dir = util.unique_path(os.path.join(os.getcwd(), basename))
filename = format
target = filename if is_xlsx_single else export_dir
try:
print("Exporting as %s %sto %s." %
(format[:4].upper(), dbstr, target))
chats = sorted(db.get_conversations(),
key=lambda x: x["title"].lower())
db.get_conversations_stats(chats)
bar_total = sum(c["message_count"] for c in chats)
bartext = " Exporting %.*s.." % (30, db.filename) # Enforce width
bar = ProgressBar(max=bar_total, afterword=bartext)
bar.start()
result = export.export_chats(chats, export_dir, filename, db,
progress=bar.update)
files, count = result
bar.stop()
if count:
bar.afterword = " Exported %s to %s. " % (db, target)
bar.update(bar_total)
print()
log("Exported %s %sto %s as %s.", util.plural("chat", count),
dbstr, target, format)
else:
print("\nNo messages to export%s." %
("" if len(dbs) == 1 else " from %s" % db))
os.unlink(filename) if is_xlsx_single else os.rmdir(export_dir)
except Exception as e:
print("Error exporting chats: %s\n\n%s" %
(e, traceback.format_exc()))
def tr_error(self, tr):
self._running_flushes -= 1
self._finished_flushes += 1
tr.inc_error_count()
if tr.get_error_count() > self._MAX_ENDPOINT_ERRORS:
log.warn("Transaction %d failed too many (%d) times, removing",
tr.get_id(), tr.get_error_count())
self._remove(tr)
self._transactions_flushed += 1
self.print_queue_stats()
self._transactions_rejected += 1
ForwarderStatus(
queue_length=self._total_count,
queue_size=self._total_size,
flush_count=self._flush_count,
transactions_received=self._transactions_received,
transactions_flushed=self._transactions_flushed,
transactions_rejected=self._transactions_rejected).persist()
else:
tr.compute_next_flush(self._MAX_WAIT_FOR_REPLAY)
log.warn(
"Transaction %d in error (%s error%s), it will be replayed after %s",
tr.get_id(), tr.get_error_count(),
plural(tr.get_error_count()), tr.get_next_flush())
self._endpoints_errors[tr._endpoint] = self._endpoints_errors.get(
tr._endpoint, 0) + 1
# Endpoint failed too many times, it's probably an enpoint issue
# Let's avoid blocking on it
if self._endpoints_errors[
tr._endpoint] == self._MAX_ENDPOINT_ERRORS:
new_trs_to_flush = []
for transaction in self._trs_to_flush:
if transaction._endpoint != tr._endpoint:
new_trs_to_flush.append(transaction)
else:
transaction.compute_next_flush(
self._MAX_WAIT_FOR_REPLAY)
log.debug(
'Endpoint %s seems down, removed %s transaction from current flush',
tr._endpoint,
len(self._trs_to_flush) - len(new_trs_to_flush))
self._trs_to_flush = new_trs_to_flush
def run(self, path, print_file=True):
from functools import reduce
import prompt
if print_file and self.print_file:
_print_file(path)
util.info("description: " + self.description)
if type(self.deduction) is int:
choices = ["do not take this deduction",
"take this deduction (-{} points)".format(
self.deduction)]
got = prompt.prompt(choices, '1')
if got == [1]:
return {'deduction': self.deduction,
'description': self.description}
else:
util.info("taking no points")
return None
elif type(self.deduction) is list:
choices = ["{} (-{} {})".format(y, x, util.plural("point", x)) for
x, y in self.deduction]
got = prompt.prompt(choices, self.deduction_mode)
total_deducted = sum(map(lambda x: self.deduction[x][0], got))
if total_deducted > 0:
deductions = []
for s in got:
if self.deduction[s][0] > 0:
d = {'deduction': self.deduction[s][0],
'description': self.deduction[s][1]}
deductions.append(d)
return {'description': self.description + ':',
'subresults': deductions}
else:
util.info("taking no points")
return None
def __should_fail(self, result):
"""Called before an eval test is about to be failed, but the criteria
specifies that a human should confirm the failure before taking
any points. If this method returns True, the points are ultimately
taken.
"""
import sys
from grader import write_results
from prompt import prompt
util.warning("about to fail a test")
write_results(sys.stdout, [result])
points = result['deduction']
s = util.plural("point", points)
fail_msg = "fail this test (-{} {})".format(points, s)
choices = [fail_msg, "do not fail this test"]
return prompt(choices, '1') == [0]
def run(self, path):
import os
from functools import reduce
import prompt
_print_file(path)
sprint("reviewing '{}' (in directory '{}')".format(
path, os.getcwd()))
sprint("description: " + self.description)
if type(self.deduction) is int:
choices = ["do not take this deduction",
"take this deduction (-{} points)".format(
self.deduction)]
got = prompt.prompt(choices, '1')
if got == [1]:
return {'deduction': self.deduction,
'description': self.description}
else:
sprint("taking no points")
return None
elif type(self.deduction) is list:
choices = ["{} (-{} {})".format(y, x, plural("point", x)) for
x, y in self.deduction]
got = prompt.prompt(choices, self.deduction_mode)
if sum(map(lambda x: self.deduction[x][0], got)) > 0:
deductions = []
for s in got:
if self.deduction[s][0] > 0:
d = {'deduction': self.deduction[s][0],
'description': self.deduction[s][1]}
deductions.append(d)
return deductions
else:
sprint("taking no points")
return None
def _relation_repr(cls, rel):
target = rel.argument
if target and inspect.isfunction(target):
target = target()
if isinstance(target, Mapper):
target = target.class_
target = target.__name__
primaryjoin = ''
lookup = mtl()
foo = rel.key
if rel.primaryjoin is not None and hasattr(rel.primaryjoin, 'right'):
right_lookup = lookup.get(rel.primaryjoin.right.table.name, '%s.c' % rel.primaryjoin.right.table.name)
left_lookup = lookup.get(rel.primaryjoin.left.table.name, '%s.c' % rel.primaryjoin.left.table.name)
primaryjoin = ", primaryjoin='%s.%s==%s.%s'" % (left_lookup,
rel.primaryjoin.left.name,
right_lookup,
rel.primaryjoin.right.name)
elif hasattr(rel, '_as_string'):
primaryjoin = ', primaryjoin="%s"' % rel._as_string
secondary = ''
secondaryjoin = ''
if rel.secondary is not None:
"""
**HACK**: If there is a secondary relationship like between Venue, Event, and Event_Type, then I'm only
going show a primary relationship.
"Events = relationship('Event', primaryjoin='Venue.id==Event.venue_id')" and not
"Event_Types = relation('EventType', primaryjoin='Venue.id==Event.venue_id', secondary=Event, secondaryjoin='Event.event_type_id==EventType.id')"
"""
if rel.secondary.name in self.table_model_dict:
target = self.table_model_dict[rel.secondary.name]
else:
target = self.find_new_name(singular(name2label(rel.secondary.name)), []) # **HACK**
secondary = ''
secondaryjoin = ''
foo = plural(rel.secondary.name)
backref = ''
# if rel.backref:
# backref=", backref='%s'"%rel.backref.key
return "%s = relationship('%s'%s%s%s%s)" % (foo, target, primaryjoin, secondary, secondaryjoin, backref)
def identify_recordings(self):
all_recordings = self.camera.list_recordings()
if all_recordings and len(all_recordings) > 0:
self.camera.prepare_recordings(all_recordings)
filtered_recordings = self.filter_processed(all_recordings)
total = len(filtered_recordings)
self.prepare_recordings(filtered_recordings)
logger.info(
"{} file{} on device of which {} are to be processed".format(
total, plural(total), len(filtered_recordings)))
if len(filtered_recordings) > 0:
logger.info("The oldest recording on the device is: {}".format(
filtered_recordings[0]['start_timestamp']))
requested_sequences = []
if self.config.get('process_manual_requests'):
logger.info("Processing manual requests..")
manual_requests = self.find_manual_requests()
requested_sequences.extend(manual_requests)
if self.config.get('process_motion_detection'):
logger.info("Processing motion detection..")
self.add_paths(filtered_recordings, "thumbnail_filename")
download_list = self.camera.download_files(
filtered_recordings, "thumbnail", "thumbnail_filename")
self.motion.calculate_differences(download_list,
"thumbnail_filename")
requested_sequences.extend(
self.motion.identify_requests(download_list))
self.match_recordings(requested_sequences, all_recordings)
self.remove_empty_requests(requested_sequences)
return (filtered_recordings, requested_sequences)
def run(self):
self._is_running = True
# For identifying "chat:xxx" and "from:xxx" keywords
template_chat = step.Template(templates.SEARCH_ROW_CHAT_HTML)
template_contact = step.Template(templates.SEARCH_ROW_CONTACT_HTML)
template_message = step.Template(templates.SEARCH_ROW_MESSAGE_HTML)
template_row = step.Template(templates.SEARCH_ROW_TABLE_HTML)
template_table = step.Template(templates.SEARCH_ROW_TABLE_HEADER_HTML)
query_parser = searchparser.SearchQueryParser()
wrap_b = lambda x: "<b>%s</b>" % x.group(0)
result = None
while self._is_running:
try:
search = self._queue.get()
if not search:
continue # continue while self._is_running
self._stop_work = False
self._drop_results = False
parser = skypedata.MessageParser(search["db"])
# {"html": html with results, "map": link data map}
# map data: {"contact:666": {"contact": {contact data}}, }
result_type, result_count, count = None, 0, 0
result = {"html": "", "map": {},
"search": search, "count": 0}
sql, params, match_words = query_parser.Parse(search["text"])
# Turn wildcard characters * into regex-compatible .*
match_words_re = [".*".join(map(re.escape, w.split("*")))
for w in match_words]
patt = "(%s)" % "|".join(match_words_re)
# For replacing matching words with <b>words</b>
pattern_replace = re.compile(patt, re.IGNORECASE)
# Find chats with a matching title or matching participants
chats = search["db"].get_conversations()
chats.sort(key=lambda x: x["title"])
chat_map = {} # {chat id: {chat data}}
for chat in chats:
chat_map[chat["id"]] = chat
if "conversations" == search["table"] and match_words:
title_matches = False
matching_authors = []
if self.match_all(chat["title"], match_words):
title_matches = True
for participant in chat["participants"]:
contact = participant["contact"]
if contact:
for n in filter(None, [contact["fullname"],
contact["displayname"], contact["identity"]]):
if self.match_all(n, match_words) \
and contact not in matching_authors:
matching_authors.append(contact)
if title_matches or matching_authors:
count += 1
result_count += 1
result["html"] += template_chat.expand(locals())
key = "chat:%s" % chat["id"]
result["map"][key] = {"chat": chat["id"]}
if not count % conf.SearchResultsChunk \
and not self._drop_results:
result["count"] = result_count
self.postback(result)
result = {"html": "", "map": {},
"search": search, "count": 0}
if self._stop_work:
break # break for chat in chats
if result["html"] and not self._drop_results:
result["count"] = result_count
self.postback(result)
result = {"html": "", "map": {}, "search": search,
"count": 0}
# Find contacts with a matching name
if not self._stop_work and "contacts" == search["table"] \
and match_words:
count = 0
contacts = search["db"].get_contacts()
# Possibly more: country (ISO code, need map), birthday
# (base has YYYYMMDD in integer field).
match_fields = [
"displayname", "skypename", "province", "city",
"pstnnumber", "phone_home", "phone_office",
"phone_mobile", "homepage", "emails", "about",
"mood_text",
]
for contact in contacts:
match = False
fields_filled = {}
for field in match_fields:
if contact[field]:
val = contact[field]
if self.match_all(val, match_words):
match = True
val = pattern_replace.sub(wrap_b, val)
fields_filled[field] = val
if match:
count += 1
result_count += 1
result["html"] += template_contact.expand(locals())
if not (self._drop_results
or count % conf.SearchResultsChunk):
result["count"] = result_count
self.postback(result)
result = {"html": "", "map": {},
"search": search, "count": 0}
if self._stop_work:
break # break for contact in contacts
if result["html"] and not self._drop_results:
result["count"] = result_count
self.postback(result)
result = {"html": "", "map": {},
"search": search, "count": 0}
# Find messages with a matching body
if not self._stop_work and "messages" == search["table"]:
count, result_type = 0, "messages"
chat_messages = {} # {chat id: [message, ]}
chat_order = [] # [chat id, ]
messages = search["db"].get_messages(
additional_sql=sql, additional_params=params,
ascending=False, use_cache=False)
for m in messages:
chat = chat_map.get(m["convo_id"])
chat_title = chat["title_long"]
body = parser.parse(m,
pattern_replace if match_words else None,
html={"w": search["window"].Size.width * 5/9})
count += 1
result_count += 1
result["html"] += template_message.expand(locals())
key = "message:%s" % m["id"]
result["map"][key] = {"chat": chat["id"],
"message": m["id"]}
if not count % conf.SearchResultsChunk \
and not self._drop_results:
result["count"] = result_count
self.postback(result)
result = {"html": "", "map": {},
"search": search, "count": 0}
if self._stop_work or count >= conf.SearchMessagesMax:
break # break for m in messages
infotext = search["table"]
if not self._stop_work and "all tables" == search["table"]:
infotext, result_type = "", "table row"
# Search over all fields of all tables.
for table in search["db"].tables_list:
sql, params, words = \
query_parser.Parse(search["text"], table)
if not sql:
continue # continue for table in search["db"]..
infotext += (", " if infotext else "") + table["name"]
rows = search["db"].execute(sql, params)
row = rows.fetchone()
if not row:
continue # continue for table in search["db"]..
result["html"] = template_table.expand(locals())
count = 0
while row:
count += 1
result_count += 1
result["html"] += template_row.expand(locals())
key = "table:%s:%s" % (table["name"], count)
result["map"][key] = {"table": table["name"],
"row": row}
if not count % conf.SearchResultsChunk \
and not self._drop_results:
result["count"] = result_count
self.postback(result)
result = {"html": "", "map": {},
"search": search, "count": 0}
if self._stop_work \
or result_count >= conf.SearchTableRowsMax:
break # break while row
row = rows.fetchone()
if not self._drop_results:
result["html"] += "</table>"
result["count"] = result_count
self.postback(result)
result = {"html": "", "map": {},
"search": search, "count": 0}
infotext += " (%s)" % util.plural("result", count)
if self._stop_work \
or result_count >= conf.SearchTableRowsMax:
break # break for table in search["db"]..
single_table = ("," not in infotext)
infotext = "table%s: %s" % \
("" if single_table else "s", infotext)
if not single_table:
infotext += "; %s in total" % \
util.plural("result", result_count)
final_text = "No matches found."
if self._drop_results:
result["html"] = ""
if result_count:
final_text = "Finished searching %s." % infotext
if self._stop_work:
final_text += " Stopped by user."
elif "messages" == result_type \
and count >= conf.SearchMessagesMax:
final_text += " Stopped at %s limit %s." % \
(result_type, conf.SearchMessagesMax)
elif "table row" == result_type \
and count >= conf.SearchTableRowsMax:
final_text += " Stopped at %s limit %s." % \
(result_type, conf.SearchTableRowsMax)
result["html"] += "</table><br /><br />%s</font>" % final_text
result["done"] = True
result["count"] = result_count
self.postback(result)
except Exception, e:
if not result:
result = {}
result["done"], result["error"] = True, traceback.format_exc()
result["error_short"] = "%s: %s" % (type(e).__name__, e.message)
self.postback(result)
def export_chat(chat, messages, filename, db):
"""
Exports the chat messages to file.
@param chat chat data dict, as returned from SkypeDatabase
@param messages list of message data dicts
@param filename full path and filename of resulting file, file extension
.html|.txt determines file format
@param db SkypeDatabase instance
"""
result = False
f = None
try:
is_html = filename.lower().endswith(".html")
is_csv = filename.lower().endswith(".csv")
is_txt = filename.lower().endswith(".txt")
if is_txt:
f = codecs.open(filename, "w", "utf-8")
else:
f = open(filename, "w")
# @todo add stats?
parser = skypedata.MessageParser(db)
chat_title = chat["title_long_lc"]
main_data = {
"title": chat_title,
"date1": messages[0]["datetime"].strftime("%d.%m.%Y") \
if len(messages) else "",
"date2": messages[-1]["datetime"].strftime("%d.%m.%Y") \
if len(messages) else "",
"messages_total": util.plural("message", chat["message_count"]),
"chat_created": chat["created_datetime"].strftime("%d.%m.%Y") \
if chat["created_datetime"] else "",
"app": conf.Title,
"now": datetime.datetime.now() \
.strftime("%d.%m.%Y %H:%M"),
"db": db.filename,
"count": str(len(messages)),
"chat_info": "Showing %s" \
% util.plural("message", len(messages)),
}
if is_html:
# Write HTML header and table header
header_data = dict([(k, escape(v)) for k, v in main_data.items()])
if header_data["date1"] and header_data["date2"]:
header_data["chat_info"] += \
" from <b>%(date1)s</b> to <b>%(date2)s</b>" % header_data
header_data["chat_info"] += ".<br />"
if header_data["chat_created"]:
header_data["chat_info"] += \
"Chat created on <b>%(chat_created)s</b>" % header_data
if header_data["messages_total"]:
header_data["chat_info"] += \
("," if header_data["chat_created"] else "Chat has") +\
" <b>%(messages_total)s</b> in total" % header_data
if header_data["chat_created"] or header_data["messages_total"]:
header_data["chat_info"] += ".<br />"
header_data.update({
"title":
"History of Skype " + header_data["title"],
"css_avatars":
"",
"css_chat_picture":
"",
"header_left":
"",
"header_right":
"",
"header_link":
""
})
if chat["meta_picture"]:
raw = chat["meta_picture"].encode("latin1")
if raw.startswith("\0"):
# For some reason, Skype image blobs
# can start with a null byte.
raw = raw[1:]
if raw.startswith("\0"):
raw = "\xFF" + raw[1:]
header_data["header_left"] = htmltag("span", {
"class": "chat_picture",
"title": chat["title"]
})
img = wx.ImageFromStream(cStringIO.StringIO(raw))
header_data["css_chat_picture"] = cssrule("span.chat_picture",{
"background":
"url(data:image/jpg;base64,%s) " \
"center center no-repeat" % base64.b64encode(raw),
"margin": "0px 10px 0px 10px",
"display": "block",
"width": "%spx" % img.Width,
"height": "%spx" % img.Height,
})
if chat["participants"]:
for p in chat["participants"]:
avatar_class = "avatar__default"
if "avatar_image" in p["contact"] \
and p["contact"]["avatar_image"]:
raw = p["contact"]["avatar_image"].encode("latin1")
if raw.startswith("\0"):
# For some reason, Skype avatar image blobs
# can start with a null byte.
raw = raw[1:]
if raw.startswith("\0"):
#raw = raw[1:]
raw = "\xFF" + raw[1:]
# Replace dots and commas, as they are
# not valid CSS identifier characters
avatar_class = "avatar_" \
+ p["identity"].replace(".", "___") \
.replace(",", "---")
header_data["css_avatars"] += cssrule(
"span.%s" % avatar_class, {
"background":
"url(data:image/jpg;base64,%s) center " \
"center no-repeat" % base64.b64encode(raw)
})
if skypedata.CHATS_TYPE_SINGLE == chat["type"]:
title = p["contact"]["name"]
name = escape(p["contact"]["name"])
if p["contact"]["name"] != p["identity"]:
title += " (%s)" % p["identity"]
name += "<br />(%s)" % escape(p["identity"])
side = "right" if (p["identity"] == db.id) else "left"
header_data["header_%s" % side] = "<div>%s" \
"<br />%s</div>" % (
htmltag("span", {"title": title,
"class": "avatar header %s" % avatar_class}
), name
)
if skypedata.CHATS_TYPE_SINGLE != chat["type"]:
header_data["header_link"] = PARTICIPANTS_LINK
for k, v in header_data.items():
header_data[k] = str(v)
f.write(CHAT_HTML_HEADER % header_data)
if skypedata.CHATS_TYPE_SINGLE != chat["type"]:
for p in sorted(chat["participants"],
key=lambda a: a["contact"]["name"].lower()):
img_attr = {"class": "avatar avatar__default"}
img_attr["title"] = p["contact"]["name"]
if p["contact"]["name"] != p["identity"]:
img_attr["title"] += " (%s)" % p["identity"]
if "avatar_image" in p["contact"] \
and p["contact"]["avatar_image"]:
# Replace dots and commas, as they are not valid
# CSS identifier characters
img_attr["class"] = "avatar avatar_%s" \
% p["identity"].replace(".", "___") \
.replace(",", "---")
name = escape(p["contact"]["name"])
if p["contact"]["name"] != p["identity"]:
name += "<br />(%s)" % escape(p["identity"])
f.write(" <span>%(img)s%(name)s</span>\n" % {
"name": name,
"img": htmltag("span", img_attr),
})
f.write(" </div>\r\n </td>\r\n</tr>\r\n</table>\r\n" \
"</td></tr><tr><td><table class='content_table'>\r\n")
elif is_txt:
main_data["hr"] = "-" * 79
f.write("History of Skype %(title)s.\r\n" \
"Showing %(count)s messages" % main_data)
if main_data["date1"] and main_data["date2"]:
f.write(" from %(date1)s to %(date2)s" % main_data)
f.write(".\r\n")
if main_data["chat_created"]:
f.write("Chat created on %(chat_created)s" % main_data)
else:
f.write("Chat has")
if main_data["messages_total"]:
f.write(("," if main_data["chat_created"] else "") +
" %(messages_total)s in total" % main_data)
f.write(".\r\n")
f.write(
"Source: %(db)s.\r\n" \
"Exported with %(app)s on %(now)s." \
"\r\n%(hr)s\r\n" % main_data
)
elif is_csv:
# Initialize CSV writer and write header row
dialect = csv.excel
# Default is "," which is actually not Excel
dialect.delimiter = ";"
# Default is "\r\n", which causes another "\r" to be written
dialect.lineterminator = "\r"
csv_writer = csv.writer(f, dialect)
csv_writer.writerow(["Time", "Author", "Message"])
colourmap = collections.defaultdict(lambda: "remote")
colourmap[db.id] = "local"
previous_day = datetime.date.fromtimestamp(0)
for m in messages:
if m["datetime"].date() != previous_day:
# Day has changed: insert a date header
previous_day = m["datetime"].date()
weekday = previous_day.strftime("%A").capitalize()
date = previous_day.strftime("%d. %B %Y")
if locale.getpreferredencoding():
weekday = weekday.decode(locale.getpreferredencoding())
date = date.decode(locale.getpreferredencoding())
if is_html:
f.write(
"<tr>\r\n\t<td class='t1'></td>\r\n\t" \
"<td class='day t2'></td>\r\n\t" \
"<td class='day t3'></td>\r\n\t" \
"<td class='day' colspan='2'><span class='weekday'>" \
"%(weekday)s</span>, %(date)s</td>\r\n</tr>\r\n" % {
"weekday": escape(weekday),
"date": escape(date)
})
elif is_txt:
f.write("\r\n%(weekday)s, %(date)s\r\n%(hr)s\r\n\r\n" % {
"weekday": weekday,
"date": date,
"hr": "-" * 40
})
if is_html:
body = parser.parse(m, html={"w": -1, "export": True})
f.write("<tr>\r\n\t" \
"<td class='author %(authorclass)s' colspan='2'>" \
"%(name)s</td>\r\n\t" \
"<td class='t3'></td>\r\n\t<td>%(text)s</td>\r\n\t" \
"<td class='timestamp'>%(time)s</td>\r\n</tr>\r\n" % {
"authorclass": colourmap[m["author"]],
"time": m["datetime"].strftime("%H:%S"),
"name": escape(m["from_dispname"]),
"text": body
})
else:
parsed_text = parser.parse(m, text=True)
try:
parsed_text = parsed_text.decode("utf-8")
except Exception, e:
pass
if is_txt:
f.write(
"%(datetime)s %(name)s:\r\n%(text)s\r\n\r\n" % {
"datetime": m["datetime"].strftime("%H:%S"),
"name": m["from_dispname"],
"text": parsed_text
})
elif is_csv:
try:
parsed_text = parser.parse(m, text=True)
parsed_text = parsed_text.decode("utf-8")
except Exception, e:
pass
values = [
m["datetime"].strftime("%Y-%m-%d %H:%M:%S"),
m["from_dispname"].encode("utf-8"),
parsed_text.encode("utf-8")
]
csv_writer.writerow(values)
def flush(self):
try:
self.flush_count += 1
self.log_count += 1
packets_per_second = self.metrics_aggregator.packets_per_second(self.interval)
packet_count = self.metrics_aggregator.total_count
metrics = self.metrics_aggregator.flush()
count = len(metrics)
if self.flush_count % FLUSH_LOGGING_PERIOD == 0:
self.log_count = 0
if count:
self.submit(metrics)
events = self.metrics_aggregator.flush_events()
event_count = len(events)
if event_count:
self.submit_events(events)
service_checks = self.metrics_aggregator.flush_service_checks()
check_count = len(service_checks)
if check_count:
self.submit_service_checks(service_checks)
should_log = self.flush_count <= FLUSH_LOGGING_INITIAL or self.log_count <= FLUSH_LOGGING_COUNT
log_func = log.info
if not should_log:
log_func = log.debug
log_func("Flush #%s: flushed %s metric%s, %s event%s, and %s service check run%s" % (self.flush_count, count, plural(count), event_count, plural(event_count), check_count, plural(check_count)))
if self.flush_count == FLUSH_LOGGING_INITIAL:
log.info("First flushes done, %s flushes will be logged every %s flushes." % (FLUSH_LOGGING_COUNT, FLUSH_LOGGING_PERIOD))
# Persist a status message.
packet_count = self.metrics_aggregator.total_count
DogstatsdStatus(
flush_count=self.flush_count,
packet_count=packet_count,
packets_per_second=packets_per_second,
metric_count=count,
event_count=event_count,
).persist()
except Exception:
if self.finished.isSet():
log.debug("Couldn't flush metrics, but that's expected as we're stopping")
else:
log.exception("Error flushing metrics")
def work_merge_left(self, params):
"""
Worker branch that merges differences given in params, posting progress
back to application.
"""
error, e = None, None
db1, db2 = params["db1"], params["db2"]
chats = params["chats"]
chats1_count = len(db1.get_conversations())
count_messages = 0
count_participants = 0
try:
for index, chat_data in enumerate(chats):
if self._stop_work:
break # break for i, chat_data in enumerate(chats)
chat1 = chat_data["chat"]["c1"]
chat2 = chat_data["chat"]["c2"]
messages = chat_data["diff"]["messages"]
participants = chat_data["diff"]["participants"]
html = "Merged %s" % chat1["title_long_lc"]
if not chat2:
html += " - new chat"
if messages:
newstr = "" if not chat2 else "new "
html += ", %s" % util.plural("%smessage" % newstr,
messages)
else:
html += ", no messages"
html += "."
if not chat2:
chat2 = chat1.copy()
chat_data["chat"]["c2"] = chat2
chat2["id"] = db2.insert_chat(chat2, db1)
if participants:
db2.insert_participants(chat2, participants, db1)
count_participants += len(participants)
if messages:
db2.insert_messages(chat2, messages, db1, chat1,
self.yield_ui, self.REFRESH_COUNT)
count_messages += len(messages)
if not self._drop_results:
result = {
"type": "merge_left",
"index": index,
"output": html,
"count": chats1_count,
"params": params,
"chats": [chat_data["chat"]]
}
if index < len(chats) - 1:
result["status"] = (
"Merging %s." %
chats[index + 1]["chat"]["title_long_lc"])
self.postback(result)
except Exception as e:
error = traceback.format_exc()
finally:
if chats:
html = "Merged %s" % util.plural("new message", count_messages)
if count_participants:
html += " and %s" % util.plural("new participant",
count_participants)
html += " \n\nto %s." % db2
if not self._drop_results:
result = {
"type": "merge_left",
"done": True,
"output": html,
"params": params
}
if error:
result["error"] = error
if e:
result["error_short"] = "%s: %s" % (type(e).__name__,
e.message)
self.postback(result)
def flush(self):
try:
self.flush_count += 1
self.log_count += 1
packets_per_second = self.metrics_aggregator.packets_per_second(self.interval)
packet_count = self.metrics_aggregator.total_count
metrics = self.metrics_aggregator.flush()
count = len(metrics)
if self.flush_count % FLUSH_LOGGING_PERIOD == 0:
self.log_count = 0
if count:
self.submit(metrics)
events = self.metrics_aggregator.flush_events()
event_count = len(events)
if event_count:
self.submit_events(events)
should_log = self.flush_count <= FLUSH_LOGGING_INITIAL or self.log_count <= FLUSH_LOGGING_COUNT
log_func = log.info
if not should_log:
log_func = log.debug
log_func("Flush #%s: flushed %s metric%s and %s event%s" % (self.flush_count, count, plural(count), event_count, plural(event_count)))
if self.flush_count == FLUSH_LOGGING_INITIAL:
log.info("First flushes done, %s flushes will be logged every %s flushes." % (FLUSH_LOGGING_COUNT, FLUSH_LOGGING_PERIOD))
# Persist a status message.
packet_count = self.metrics_aggregator.total_count
DogstatsdStatus(
flush_count=self.flush_count,
packet_count=packet_count,
packets_per_second=packets_per_second,
metric_count=count,
event_count=event_count,
).persist()
except Exception, e:
log.exception("Error flushing metrics")
def check_status_lines(cs):
check_lines = [
' ' + cs.name + ' ({})'.format(cs.check_version),
' ' + '-' * (len(cs.name) + 3 + len(cs.check_version))
]
if cs.init_failed_error:
check_lines.append(" - initialize check class [%s]: %s" %
(style(STATUS_ERROR, 'red'),
repr(cs.init_failed_error)))
if cs.init_failed_traceback:
check_lines.extend(' ' + line for line in
cs.init_failed_traceback.split('\n'))
else:
for s in cs.instance_statuses:
c = 'green'
if s.has_warnings():
c = 'yellow'
if s.has_error():
c = 'red'
line = " - instance #%s [%s]" % (
s.instance_id, style(s.status, c))
if s.has_error():
line += u": %s" % s.error
if s.metric_count is not None:
line += " collected %s metrics" % s.metric_count
if s.instance_check_stats is not None:
line += " Last run duration: %s" % s.instance_check_stats.get('run_time')
check_lines.append(line)
if s.has_warnings():
for warning in s.warnings:
warn = warning.split('\n')
if not len(warn):
continue
check_lines.append(u" %s: %s" %
(style("Warning", 'yellow'), warn[0]))
check_lines.extend(u" %s" % l for l in
warn[1:])
if s.traceback is not None:
check_lines.extend(' ' + line for line in
s.traceback.split('\n'))
check_lines += [
" - Collected %s metric%s, %s event%s & %s service check%s" % (
cs.metric_count, plural(cs.metric_count),
cs.event_count, plural(cs.event_count),
cs.service_check_count, plural(cs.service_check_count)),
]
if cs.check_stats is not None:
check_lines += [
" - Stats: %s" % pretty_statistics(cs.check_stats)
]
if cs.library_versions is not None:
check_lines += [
" - Dependencies:"]
for library, version in cs.library_versions.iteritems():
check_lines += [" - %s: %s" % (library, version)]
check_lines += [""]
return check_lines
def work_diff_merge_left(self, params):
"""
Worker branch that compares all chats on the left side for differences,
copies them over to the right, posting progress back to application.
"""
result = {
"output": "",
"index": 0,
"params": params,
"chats": [],
"count": 0,
"type": "diff_merge_left",
}
error, e = None, None
compared = []
db1, db2 = params["db1"], params["db2"]
try:
chats1 = params.get("chats") or db1.get_conversations()
chats2 = db2.get_conversations()
c1map = dict((c["identity"], c) for c in chats1)
c2map = dict((c["identity"], c) for c in chats2)
for c1 in chats1:
c2 = c2map.get(c1["identity"])
c = c1.copy()
c["messages1"] = c1["message_count"] or 0
c["messages2"] = c2["message_count"] or 0 if c2 else 0
c["c1"], c["c2"] = c1, c2
compared.append(c)
compared.sort(key=lambda x: x["title"].lower())
result["count"] = len(compared)
count_messages = 0
count_participants = 0
for index, chat in enumerate(compared):
diff = self.get_chat_diff_left(chat, db1, db2)
if self._stop_work:
break # break for index, chat in enumerate(compared)
if diff["messages"] \
or (chat["message_count"] and diff["participants"]):
chat1 = chat["c1"]
chat2 = chat["c2"]
new_chat = not chat2
if new_chat:
chat2 = chat1.copy()
chat["c2"] = chat2
chat2["id"] = db2.insert_chat(chat2, db1)
if diff["participants"]:
db2.insert_participants(chat2, diff["participants"],
db1)
count_participants += len(diff["participants"])
if diff["messages"]:
db2.insert_messages(chat2, diff["messages"], db1,
chat1, self.yield_ui,
self.REFRESH_COUNT)
count_messages += len(diff["messages"])
newstr = "" if new_chat else "new "
info = "Merged %s" % chat["title_long_lc"]
if new_chat:
info += " - new chat"
if diff["messages"]:
info += ", %s" % util.plural("%smessage" % newstr,
diff["messages"])
else:
info += ", no messages"
result["output"] = info + "."
result["diff"] = diff
result["index"] = index
result["chats"].append(chat)
if not self._drop_results:
if index < len(compared) - 1:
result["status"] = (
"Scanning %s." %
compared[index + 1]["title_long_lc"])
self.postback(result)
result = {
"output": "",
"index": index,
"params": params,
"count": len(compared),
"type": "diff_merge_left",
"chats": [],
}
except Exception as e:
error = traceback.format_exc()
finally:
if not self._drop_results:
if compared:
info = "Merged %s" % util.plural("new message",
count_messages)
if count_participants:
info += " and %s" % util.plural(
"new participant", count_participants)
info += " \n\nto %s." % db2
else:
info = "Nothing new to merge from %s to %s." % (db1, db2)
result = {
"type": "diff_merge_left",
"done": True,
"output": info,
"params": params,
"chats": []
}
if error:
result["error"] = error
if e:
result["error_short"] = "%s: %s" % (type(e).__name__,
e.message)
self.postback(result)
def work_diff(self, params):
"""
Worker branch that compares all chats for differences, posting results
back to application.
"""
# {"htmls": [html result for db1, db2],
# "chats": [differing chats in db1, db2]}
result = {"htmls": ["", ""], "chats": [[], []],
"params": params, "index": 0, "type": "diff"}
db1, db2 = params["db1"], params["db2"]
chats1 = db1.get_conversations()
chats2 = db2.get_conversations()
skypenames1 = [i["identity"] for i in db1.get_contacts()]
skypenames2 = [i["identity"] for i in db2.get_contacts()]
skypenames1.append(db1.id)
skypenames2.append(db2.id)
c1map = dict((c["identity"], c) for c in chats1)
c2map = dict((c["identity"], c) for c in chats2)
compared = []
for c1 in chats1:
c2 = c2map.get(c1["identity"])
c = c1.copy()
c["messages1"] = c1["message_count"] or 0
c["messages2"] = c2["message_count"] or 0 if c2 else 0
c["c1"], c["c2"] = c1, c2
compared.append(c)
for c2 in chats2:
c1 = c1map.get(c2["identity"])
if not c1:
c = c2.copy()
c["messages1"], c["messages2"] = 0, c["message_count"] or 0
c["c1"], c["c2"] = c1, c2
compared.append(c)
compared.sort(key=lambda x: x["title"].lower())
for index, chat in enumerate(compared):
diff = self.get_chat_diff(chat, db1, db2)
if self._stop_work:
break # break for index, chat in enumerate(compared)
for i in range(2):
new_chat = not chat["c1" if i else "c2"]
newstr = "" if new_chat else "new "
info = util.htmltag("a", {"href": chat["identity"]},
chat["title_long"], utf=False)
if new_chat:
info += " - new chat"
if diff["messages"][i]:
info += ", %s" % util.plural("%smessage" % newstr,
diff["messages"][i])
if diff["participants"][i] and newstr:
info += ", %s" % (
util.plural("%sparticipant" % newstr,
diff["participants"][i]))
if diff["messages"][i] or diff["participants"][i]:
info += ".<br />"
result["htmls"][i] += info
result["chats"][i].append({"chat": chat, "diff": diff})
if not self._drop_results:
self.postback(result)
result = {"htmls": ["", ""], "chats": [[], []],
"params": params, "index": index, "type": "diff"}
if not self._drop_results:
result["done"] = True
self.postback(result)
class ModelFactory(object):
def __init__(self, config):
self.config = config
self.used_model_names = []
self.used_table_names = []
schema = getattr(self.config, 'schema', None)
self._metadata = MetaData(bind=config.engine)
self._foreign_keys = {}
kw = {}
self.schemas = None
if schema:
if isinstance(schema, (list, tuple)):
self.schemas = schema
else:
self.schemas = (schema, )
for schema in self.schemas:
log.info('Reflecting database... schema:%s' % schema)
self._metadata.reflect(schema=schema)
else:
log.info('Reflecting database...')
self._metadata.reflect()
self.DeclarativeBase = declarative_base(metadata=self._metadata)
def _table_repr(self, table):
s = "Table(u'%s', metadata,\n" % (table.name)
for column in table.c:
s += " %s,\n" % column_repr(column)
if table.schema:
s += " schema='%s'\n" % table.schema
s += ")"
return s
def __repr__(self):
tables = self.get_many_to_many_tables()
tables.extend(self.get_tables_with_no_pks())
models = self.models
s = StringIO()
engine = self.config.engine
if not isinstance(engine, basestring):
engine = str(engine.url)
s.write(constants.HEADER_DECL % engine)
if 'postgres' in engine:
s.write(constants.PG_IMPORT)
self.used_table_names = []
self.used_model_names = []
for table in tables:
if table not in self.tables:
continue
table_name = self.find_new_name(table.name, self.used_table_names)
self.used_table_names.append(table_name)
s.write('%s = %s\n\n' % (table_name, self._table_repr(table)))
for model in models:
s.write(model.__repr__())
s.write("\n\n")
if self.config.example or self.config.interactive:
s.write(constants.EXAMPLE_DECL %
(models[0].__name__, models[0].__name__))
if self.config.interactive:
s.write(constants.INTERACTIVE %
([model.__name__ for model in models], models[0].__name__))
return s.getvalue()
@property
def tables(self):
if self.config.options.tables:
tables = set(self.config.options.tables)
return [
self._metadata.tables[t]
for t in set(self._metadata.tables.keys()).intersection(tables)
]
return self._metadata.tables.values()
@property
def table_names(self):
return [t.name for t in self.tables]
@property
def models(self):
if hasattr(self, '_models'):
return self._models
self.used_model_names = []
self.used_table_names = []
self._models = []
for table in self.get_non_many_to_many_tables():
try:
self._models.append(self.create_model(table))
except exc.ArgumentError:
log.warning(
"Table with name %s ha no primary key. No ORM class created"
% table.name)
self._models.sort(by__name__)
return self._models
def get_tables_with_no_pks(self):
r = []
for table in self.get_non_many_to_many_tables():
if not [c for c in table.columns if c.primary_key]:
r.append(table)
return r
def model_table_lookup(self):
if hasattr(self, '_model_table_lookup'):
return self._model_table_lookup
self._model_table_lookup = dict(
((m.__table__.name, m.__name__) for m in self.models))
return self._model_table_lookup
def find_new_name(self, prefix, used, i=0):
if i != 0:
prefix = "%s%d" % (prefix, i)
if prefix in used:
prefix = prefix
return self.find_new_name(prefix, used, i + 1)
return prefix
def create_model(self, table):
#partially borrowed from Jorge Vargas' code
#http://dpaste.org/V6YS/
log.debug('Creating Model from table: %s' % table.name)
model_name = self.find_new_name(singular(name2label(table.name)),
self.used_model_names)
self.used_model_names.append(model_name)
is_many_to_many_table = self.is_many_to_many_table(table)
table_name = self.find_new_name(table.name, self.used_table_names)
self.used_table_names.append(table_name)
mtl = self.model_table_lookup
class Temporal(self.DeclarativeBase):
__table__ = table
@classmethod
def _relation_repr(cls, rel):
target = rel.argument
if target and inspect.isfunction(target):
target = target()
if isinstance(target, Mapper):
target = target.class_
target = target.__name__
primaryjoin = ''
lookup = mtl()
if rel.primaryjoin is not None and hasattr(
rel.primaryjoin, 'right'):
right_lookup = lookup.get(
rel.primaryjoin.right.table.name,
'%s.c' % rel.primaryjoin.right.table.name)
left_lookup = lookup.get(
rel.primaryjoin.left.table.name,
'%s.c' % rel.primaryjoin.left.table.name)
primaryjoin = ", primaryjoin='%s.%s==%s.%s'" % (
left_lookup, rel.primaryjoin.left.name, right_lookup,
rel.primaryjoin.right.name)
elif hasattr(rel, '_as_string'):
primaryjoin = ', primaryjoin="%s"' % rel._as_string
secondary = ''
secondaryjoin = ''
if rel.secondary is not None:
secondary = ", secondary=%s" % rel.secondary.name
right_lookup = lookup.get(
rel.secondaryjoin.right.table.name,
'%s.c' % rel.secondaryjoin.right.table.name)
left_lookup = lookup.get(
rel.secondaryjoin.left.table.name,
'%s.c' % rel.secondaryjoin.left.table.name)
secondaryjoin = ", secondaryjoin='%s.%s==%s.%s'" % (
left_lookup, rel.secondaryjoin.left.name, right_lookup,
rel.secondaryjoin.right.name)
backref = ''
# if rel.backref:
# backref=", backref='%s'"%rel.backref.key
return "%s = relation('%s'%s%s%s%s)" % (rel.key, target,
primaryjoin, secondary,
secondaryjoin, backref)
@classmethod
def __repr__(cls):
log.debug('repring class with name %s' % cls.__name__)
try:
mapper = None
try:
mapper = class_mapper(cls)
except exc.InvalidRequestError:
log.warn(
"A proper mapper could not be generated for the class %s, no relations will be created"
% model_name)
s = ""
s += "class " + model_name + '(DeclarativeBase):\n'
if is_many_to_many_table:
s += " __table__ = %s\n\n" % table_name
else:
s += " __tablename__ = '%s'\n\n" % table_name
if hasattr(cls, '__table_args__'):
#if cls.__table_args__[0]:
#for fkc in cls.__table_args__[0]:
# fkc.__class__.__repr__ = foreignkeyconstraint_repr
# break
s += " __table_args__ = %s\n\n" % cls.__table_args__
s += " #column definitions\n"
for column in sorted(cls.__table__.c, by_name):
s += " %s = %s\n" % (column.name,
column_repr(column))
s += "\n #relation definitions\n"
ess = s
# this is only required in SA 0.5
if mapper and RelationProperty:
for prop in mapper.iterate_properties:
if isinstance(prop, RelationshipProperty):
s += ' %s\n' % cls._relation_repr(prop)
return s
except Exception, e:
log.error("Could not generate class for: %s" %
cls.__name__)
from traceback import format_exc
log.error(format_exc())
return ''
#hack the class to have the right classname
Temporal.__name__ = model_name
#set up some blank table args
Temporal.__table_args__ = {}
#add in the schema
if self.config.schema:
Temporal.__table_args__[1]['schema'] = table.schema
#trick sa's model registry to think the model is the correct name
if model_name != 'Temporal':
Temporal._decl_class_registry[
model_name] = Temporal._decl_class_registry['Temporal']
del Temporal._decl_class_registry['Temporal']
#add in single relations
fks = self.get_single_foreign_keys_by_column(table)
for column, fk in fks.iteritems():
related_table = fk.column.table
if related_table not in self.tables:
continue
log.info(' Adding <primary> foreign key for:%s' %
related_table.name)
backref_name = plural(table_name)
rel = relation(
singular(name2label(related_table.name, related_table.schema)),
primaryjoin=column == fk.column) #, backref=backref_name)
setattr(Temporal, related_table.name,
_deferred_relationship(Temporal, rel))
#add in the relations for the composites
for constraint in table.constraints:
if isinstance(constraint, ForeignKeyConstraint):
if len(constraint.elements) > 1:
related_table = constraint.elements[0].column.table
related_classname = singular(
name2label(related_table.name, related_table.schema))
primary_join = "and_(%s)" % ', '.join([
"%s.%s==%s.%s" % (model_name, k.parent.name,
related_classname, k.column.name)
for k in constraint.elements
])
rel = relation(
related_classname,
primaryjoin=primary_join
# foreign_keys=[k.parent for k in constraint.elements]
)
rel._as_string = primary_join
setattr(Temporal, related_table.name,
rel) # _deferred_relationship(Temporal, rel))
#add in many-to-many relations
for join_table in self.get_related_many_to_many_tables(table.name):
if join_table not in self.tables:
continue
primary_column = [
c for c in join_table.columns if c.foreign_keys
and list(c.foreign_keys)[0].column.table == table
][0]
for column in join_table.columns:
if column.foreign_keys:
key = list(column.foreign_keys)[0]
if key.column.table is not table:
related_column = related_table = list(
column.foreign_keys)[0].column
related_table = related_column.table
if related_table not in self.tables:
continue
log.info(
' Adding <secondary> foreign key(%s) for:%s' %
(key, related_table.name))
setattr(
Temporal, plural(related_table.name),
_deferred_relationship(
Temporal,
relation(
singular(
name2label(related_table.name,
related_table.schema)),
secondary=join_table,
primaryjoin=list(
primary_column.foreign_keys)[0].column
== primary_column,
secondaryjoin=column == related_column)))
break
return Temporal
def run_merge(filenames, output_filename=None):
"""Merges all Skype databases to a new database."""
dbs = [skypedata.SkypeDatabase(f) for f in filenames]
db_base = dbs.pop()
counts = collections.defaultdict(lambda: collections.defaultdict(int))
postbacks = Queue.Queue()
name, ext = os.path.splitext(os.path.split(db_base.filename)[-1])
now = datetime.datetime.now().strftime("%Y%m%d")
if not output_filename:
output_filename = util.unique_path("%s.merged.%s%s" % (name, now, ext))
output("Creating %s, using %s as base." % (output_filename, db_base))
bar = ProgressBar()
bar.start()
shutil.copyfile(db_base.filename, output_filename)
db2 = skypedata.SkypeDatabase(output_filename)
chats2 = db2.get_conversations()
db2.get_conversations_stats(chats2)
args = {"db2": db2, "type": "diff_merge_left"}
worker = workers.MergeThread(postbacks.put)
try:
for db1 in dbs:
chats = db1.get_conversations()
db1.get_conversations_stats(chats)
bar.afterword = " Processing %.*s.." % (30, db1)
worker.work(dict(args, db1=db1, chats=chats))
while True:
result = postbacks.get()
if "error" in result:
output("Error merging %s:\n\n%s" % (db1, result["error"]))
db1 = None # Signal for global break
break # break while True
if "done" in result:
break # break while True
if "diff" in result:
counts[db1]["chats"] += 1
counts[db1]["msgs"] += len(result["diff"]["messages"])
if "index" in result:
bar.max = result["count"]
bar.update(result["index"])
if result.get("output"):
log(result["output"])
if not db1:
break # break for db1 in dbs
bar.stop()
bar.afterword = " Processed %s." % db1
bar.update(bar.max)
output()
finally:
worker and (worker.stop(), worker.join())
if not counts:
output("Nothing new to merge.")
db2.close()
os.unlink(output_filename)
else:
for db1 in dbs:
output("Merged %s in %s from %s." %
(util.plural("message", counts[db1]["msgs"]),
util.plural("chat", counts[db1]["chats"]), db1))
output("Merge into %s complete." % db2)
def run(self):
self._is_running = True
# For identifying "chat:xxx" and "from:xxx" keywords
query_parser = searchparser.SearchQueryParser()
result = None
while self._is_running:
try:
search = self._queue.get()
if not search:
continue # continue while self._is_running
is_text_output = ("text" == search.get("output"))
wrap_html = None # MessageParser wrap function, for HTML output
if is_text_output:
TEMPLATES = {
"chat": templates.SEARCH_ROW_CHAT_TXT,
"contact": templates.SEARCH_ROW_CONTACT_TXT,
"message": templates.SEARCH_ROW_MESSAGE_TXT,
"table": templates.SEARCH_ROW_TABLE_HEADER_TXT,
"row": templates.SEARCH_ROW_TABLE_TXT,
}
wrap_b = lambda x: "**%s**" % x.group(0)
output = {"format": "text"}
else:
TEMPLATES = {
"chat": templates.SEARCH_ROW_CHAT_HTML,
"contact": templates.SEARCH_ROW_CONTACT_HTML,
"message": templates.SEARCH_ROW_MESSAGE_HTML,
"table": templates.SEARCH_ROW_TABLE_HEADER_HTML,
"row": templates.SEARCH_ROW_TABLE_HTML,
}
wrap_b = lambda x: "<b>%s</b>" % x.group(0)
output = {"format": "html"}
width = search.get("width", -1)
if width > 0:
dc = wx.MemoryDC()
dc.SetFont(
wx.Font(8,
wx.SWISS,
wx.NORMAL,
wx.NORMAL,
face=conf.HistoryFontName))
wrap_html = lambda x: wx.lib.wordwrap.wordwrap(
x, width, dc)
output["wrap"] = True
main.log("Searching for \"%(text)s\" in %(table)s (%(db)s)." %
search)
self._stop_work = False
self._drop_results = False
parser = skypedata.MessageParser(search["db"],
wrapfunc=wrap_html)
# {"output": text with results, "map": link data map}
# map data: {"contact:666": {"contact": {contact data}}, }
result_type, result_count, count = None, 0, 0
result = {
"output": "",
"map": {},
"search": search,
"count": 0
}
sql, params, match_words = query_parser.Parse(search["text"])
# Turn wildcard characters * into regex-compatible .*
match_words_re = [
".*".join(map(re.escape, w.split("*")))
for w in match_words
]
patt = "(%s)" % "|".join(match_words_re)
# For replacing matching words with <b>words</b>
pattern_replace = re.compile(patt, re.IGNORECASE)
# Find chats with a matching title or matching participants
chats = search["db"].get_conversations()
chats.sort(key=lambda x: x["title"])
chat_map = {} # {chat id: {chat data}}
template_chat = step.Template(TEMPLATES["chat"])
for chat in chats:
chat_map[chat["id"]] = chat
if "conversations" == search["table"] and match_words:
title_matches = False
matching_authors = []
if self.match_all(chat["title"], match_words):
title_matches = True
for participant in chat["participants"]:
contact = participant["contact"]
if contact:
for n in filter(None, [
contact["fullname"],
contact["displayname"],
contact["identity"]
]):
if self.match_all(n, match_words) \
and contact not in matching_authors:
matching_authors.append(contact)
if title_matches or matching_authors:
count += 1
result_count += 1
result["output"] += template_chat.expand(locals())
key = "chat:%s" % chat["id"]
result["map"][key] = {"chat": chat["id"]}
if not count % conf.SearchResultsChunk \
and not self._drop_results:
result["count"] = result_count
self.postback(result)
result = {
"output": "",
"map": {},
"search": search,
"count": 0
}
if self._stop_work:
break # break for chat in chats
if result["output"] and not self._drop_results:
result["count"] = result_count
self.postback(result)
result = {
"output": "",
"map": {},
"search": search,
"count": 0
}
# Find contacts with a matching name
if not self._stop_work and "contacts" == search["table"] \
and match_words:
count = 0
contacts = search["db"].get_contacts()
# Possibly more: country (ISO code, need map), birthday
# (base has YYYYMMDD in integer field).
match_fields = [
"displayname",
"skypename",
"province",
"city",
"pstnnumber",
"phone_home",
"phone_office",
"phone_mobile",
"homepage",
"emails",
"about",
"mood_text",
]
template_contact = step.Template(TEMPLATES["contact"])
for contact in contacts:
match = False
fields_filled = {}
for field in match_fields:
if contact[field]:
val = contact[field]
if self.match_all(val, match_words):
match = True
val = pattern_replace.sub(wrap_b, val)
fields_filled[field] = val
if match:
count += 1
result_count += 1
result["output"] += template_contact.expand(
locals())
if not (self._drop_results
or count % conf.SearchResultsChunk):
result["count"] = result_count
self.postback(result)
result = {
"output": "",
"map": {},
"search": search,
"count": 0
}
if self._stop_work:
break # break for contact in contacts
if result["output"] and not self._drop_results:
result["count"] = result_count
self.postback(result)
result = {
"output": "",
"map": {},
"search": search,
"count": 0
}
# Find messages with a matching body
if not self._stop_work and "messages" == search["table"]:
template_message = step.Template(TEMPLATES["message"])
count, result_type = 0, "messages"
chat_messages = {} # {chat id: [message, ]}
chat_order = [] # [chat id, ]
messages = search["db"].get_messages(
additional_sql=sql,
additional_params=params,
ascending=False,
use_cache=False)
for m in messages:
chat = chat_map.get(m["convo_id"])
body = parser.parse(
m, pattern_replace if match_words else None,
output)
count += 1
result_count += 1
result["output"] += template_message.expand(locals())
key = "message:%s" % m["id"]
result["map"][key] = {
"chat": chat["id"],
"message": m["id"]
}
if is_text_output or (
not self._drop_results
and not count % conf.SearchResultsChunk):
result["count"] = result_count
self.postback(result)
result = {
"output": "",
"map": {},
"search": search,
"count": 0
}
if self._stop_work or (not is_text_output and count >=
conf.SearchMessagesMax):
break # break for m in messages
infotext = search["table"]
if not self._stop_work and "all tables" == search["table"]:
infotext, result_type = "", "table row"
# Search over all fields of all tables.
template_table = step.Template(TEMPLATES["table"])
template_row = step.Template(TEMPLATES["row"])
for table in search["db"].get_tables():
table["columns"] = search["db"].get_table_columns(
table["name"])
sql, params, words = \
query_parser.Parse(search["text"], table)
if not sql:
continue # continue for table in search["db"]..
infotext += (", " if infotext else "") + table["name"]
rows = search["db"].execute(sql, params)
row = rows.fetchone()
if not row:
continue # continue for table in search["db"]..
result["output"] = template_table.expand(locals())
count = 0
while row:
count += 1
result_count += 1
result["output"] += template_row.expand(locals())
key = "table:%s:%s" % (table["name"], count)
result["map"][key] = {
"table": table["name"],
"row": row
}
if not count % conf.SearchResultsChunk \
and not self._drop_results:
result["count"] = result_count
self.postback(result)
result = {
"output": "",
"map": {},
"search": search,
"count": 0
}
if self._stop_work or (not is_text_output
and result_count >=
conf.SearchTableRowsMax):
break # break while row
row = rows.fetchone()
if not self._drop_results:
if not is_text_output:
result["output"] += "</table>"
result["count"] = result_count
self.postback(result)
result = {
"output": "",
"map": {},
"search": search,
"count": 0
}
infotext += " (%s)" % util.plural("result", count)
if self._stop_work or (
not is_text_output
and result_count >= conf.SearchTableRowsMax):
break # break for table in search["db"]..
single_table = ("," not in infotext)
infotext = "table%s: %s" % \
("" if single_table else "s", infotext)
if not single_table:
infotext += "; %s in total" % \
util.plural("result", result_count)
final_text = "No matches found."
if self._drop_results:
result["output"] = ""
if result_count:
final_text = "Finished searching %s." % infotext
if self._stop_work:
final_text += " Stopped by user."
elif "messages" == result_type and not is_text_output \
and count >= conf.SearchMessagesMax:
final_text += " Stopped at %s limit %s." % \
(result_type, conf.SearchMessagesMax)
elif "table row" == result_type and not is_text_output \
and count >= conf.SearchTableRowsMax:
final_text += " Stopped at %s limit %s." % \
(result_type, conf.SearchTableRowsMax)
result[
"output"] += "</table><br /><br />%s</font>" % final_text
if is_text_output: result["output"] = ""
result["done"] = True
result["count"] = result_count
self.postback(result)
main.log("Search found %(count)s results." % result)
except Exception as e:
if not result:
result = {}
result["done"], result["error"] = True, traceback.format_exc()
result["error_short"] = "%s: %s" % (type(e).__name__,
e.message)
self.postback(result)
def flush(self):
try:
self.flush_count += 1
packets_per_second = self.metrics_aggregator.packets_per_second(self.interval)
packet_count = self.metrics_aggregator.total_count
metrics = self.metrics_aggregator.flush()
count = len(metrics)
should_log = self.flush_count <= FLUSH_LOGGING_INITIAL or self.flush_count % FLUSH_LOGGING_PERIOD == 0
if not count:
if should_log:
log.info("Flush #%s: No metrics to flush." % self.flush_count)
else:
log.debug("Flush #%s: No metrics to flush." % self.flush_count)
else:
if should_log:
log.info("Flush #%s: flushing %s metric%s" % (self.flush_count, count, plural(count)))
else:
log.debug("Flush #%s: flushing %s metric%s" % (self.flush_count, count, plural(count)))
self.submit(metrics)
events = self.metrics_aggregator.flush_events()
event_count = len(events)
if not event_count:
if should_log:
log.info("Flush #%s: No events to flush." % self.flush_count)
else:
log.debug("Flush #%s: No events to flush." % self.flush_count)
else:
if should_log:
log.info("Flush #%s: flushing %s event%s" % (self.flush_count, len(events), plural(len(events))))
else:
log.debug("Flush #%s: flushing %s event%s" % (self.flush_count, len(events), plural(len(events))))
self.submit_events(events)
if self.flush_count == FLUSH_LOGGING_INITIAL:
log.info("First flushes done, next flushes will be logged every %s flushes." % FLUSH_LOGGING_PERIOD)
# Persist a status message.
packet_count = self.metrics_aggregator.total_count
DogstatsdStatus(
flush_count=self.flush_count,
packet_count=packet_count,
packets_per_second=packets_per_second,
metric_count=count,
event_count=event_count,
).persist()
except Exception, e:
log.exception("Error flushing metrics")
Temporal.__table_args__ = {"schema": table.schema}
# trick sa's model registry to think the model is the correct name
if model_name != "Temporal":
Temporal._decl_class_registry[model_name] = Temporal._decl_class_registry["Temporal"]
del Temporal._decl_class_registry["Temporal"]
# add in single relations
fks = self.get_foreign_keys(table)
for related_table in sorted(fks.keys(), by_name):
columns = fks[related_table]
if len(columns) > 1:
continue
column = columns[0]
log.info(" Adding <primary> foreign key for:%s" % related_table.name)
backref_name = plural(table_name)
# import ipdb; ipdb.set_trace()
rel = relation(
singular(name2label(related_table.name, related_table.schema)),
primaryjoin=column == list(column.foreign_keys)[0].column,
) # , backref=backref_name)
setattr(Temporal, related_table.name, _deferred_relationship(Temporal, rel))
# add in many-to-many relations
for join_table in self.get_related_many_to_many_tables(table.name):
primary_column = [
c for c in join_table.columns if c.foreign_keys and list(c.foreign_keys)[0].column.table == table
][0]
# import ipdb; ipdb.set_trace();
def work_diff_left(self, params):
"""
Worker branch that compares all chats on the left side for differences,
posting results back to application.
"""
# {"output": "html result for db1, db2",
# "index": currently processed chat index,
# "chats": [differing chats in db1]}
result = {
"output": "",
"chats": [],
"params": params,
"index": 0,
"type": "diff_left"
}
db1, db2 = params["db1"], params["db2"]
chats1 = params.get("chats") or db1.get_conversations()
chats2 = db2.get_conversations()
c1map = dict((c["identity"], c) for c in chats1)
c2map = dict((c["identity"], c) for c in chats2)
compared = []
for c1 in chats1:
c2 = c2map.get(c1["identity"])
c = c1.copy()
c["messages1"] = c1["message_count"] or 0
c["messages2"] = c2["message_count"] or 0 if c2 else 0
c["c1"], c["c2"] = c1, c2
compared.append(c)
compared.sort(key=lambda x: x["title"].lower())
info_template = step.Template(templates.DIFF_RESULT_ITEM)
for index, chat in enumerate(compared):
diff = self.get_chat_diff_left(chat, db1, db2)
if self._stop_work:
break # break for index, chat in enumerate(compared)
if diff["messages"] \
or (chat["message_count"] and diff["participants"]):
new_chat = not chat["c2"]
newstr = "" if new_chat else "new "
info = info_template.expand(chat=chat)
if new_chat:
info += " - new chat"
if diff["messages"]:
info += ", %s" % util.plural("%smessage" % newstr,
diff["messages"])
else:
info += ", no messages"
if diff["participants"] and not new_chat:
info += ", %s" % (util.plural("%sparticipant" % newstr,
diff["participants"]))
info += ".<br />"
result["output"] += info
result["chats"].append({"chat": chat, "diff": diff})
result["index"] = index
if not self._drop_results:
if index < len(compared) - 1:
result["status"] = ("Scanning %s." %
compared[index + 1]["title_long_lc"])
self.postback(result)
result = {
"output": "",
"chats": [],
"index": index,
"params": params,
"type": "diff_left"
}
if not self._drop_results:
result["done"] = True
self.postback(result)
def create_model(self, table):
# partially borrowed from Jorge Vargas' code
# http://dpaste.org/V6YS/
log.debug('Creating Model from table: %s' % table.name)
model_name = self.find_new_name(singular(name2label(table.name)),
self.used_model_names)
self.used_model_names.append(model_name)
is_many_to_many_table = self.is_many_to_many_table(table)
table_name = self.find_new_name(table.name, self.used_table_names)
self.used_table_names.append(table_name)
self.table_model_dict[table_name] = model_name # Kamil Edit
mtl = self.model_table_lookup
class Temporal(self.DeclarativeBase):
__table__ = table
@classmethod
def _relation_repr(cls, rel):
target = rel.argument
if target and inspect.isfunction(target):
target = target()
if isinstance(target, Mapper):
target = target.class_
target = target.__name__
primaryjoin = ''
lookup = mtl()
foo = rel.key
if rel.primaryjoin is not None and hasattr(
rel.primaryjoin, 'right'):
right_lookup = lookup.get(
rel.primaryjoin.right.table.name,
'%s.c' % rel.primaryjoin.right.table.name)
left_lookup = lookup.get(
rel.primaryjoin.left.table.name,
'%s.c' % rel.primaryjoin.left.table.name)
primaryjoin = ", primaryjoin='%s.%s==%s.%s'" % (
left_lookup, rel.primaryjoin.left.name, right_lookup,
rel.primaryjoin.right.name)
elif hasattr(rel, '_as_string'):
primaryjoin = ', primaryjoin="%s"' % rel._as_string
secondary = ''
secondaryjoin = ''
if rel.secondary is not None:
"""
**HACK**: If there is a secondary relationship like between Venue, Event, and Event_Type, then I'm only
going show a primary relationship.
"Events = relationship('Event', primaryjoin='Venue.id==Event.venue_id')" and not
"Event_Types = relation('EventType', primaryjoin='Venue.id==Event.venue_id', secondary=Event, secondaryjoin='Event.event_type_id==EventType.id')"
"""
if rel.secondary.name in self.table_model_dict:
target = self.table_model_dict[rel.secondary.name]
else:
target = self.find_new_name(
singular(name2label(rel.secondary.name)),
[]) # **HACK**
secondary = ''
secondaryjoin = ''
foo = plural(rel.secondary.name)
backref = ''
# if rel.backref:
# backref=", backref='%s'"%rel.backref.key
return "%s = relationship('%s'%s%s%s%s)" % (
foo, target, primaryjoin, secondary, secondaryjoin,
backref)
@classmethod
def __repr__(cls):
log.debug('repring class with name %s' % cls.__name__)
try:
mapper = None
try:
mapper = class_mapper(cls)
except exc.InvalidRequestError:
log.warn(
"A proper mapper could not be generated for the class %s, no relations will be created"
% model_name)
s = ""
s += "class " + model_name + '(DeclarativeBase):\n'
if is_many_to_many_table:
s += " __table__ = %s\n\n" % table_name
else:
s += " __tablename__ = '%s'\n\n" % table_name
if hasattr(cls, '__table_args__'):
# if cls.__table_args__[0]:
# for fkc in cls.__table_args__[0]:
# fkc.__class__.__repr__ = foreignkeyconstraint_repr
# break
s += " __table_args__ = %s\n\n" % cls.__table_args__
s += " #column definitions\n"
for column in sorted(cls.__table__.c, by_name):
s += " %s = %s\n" % (column.name,
column_repr(column))
s += "\n #relation definitions\n"
# this is only required in SA 0.5
if mapper and RelationProperty:
for prop in mapper.iterate_properties:
if isinstance(prop, RelationshipProperty):
s += ' %s\n' % cls._relation_repr(prop)
return s
except Exception:
log.error("Could not generate class for: %s" %
cls.__name__)
from traceback import format_exc
log.error(format_exc())
return ''
# hack the class to have the right classname
Temporal.__name__ = model_name
# set up some blank table args
Temporal.__table_args__ = {}
# add in the schema
if self.config.schema:
Temporal.__table_args__[1]['schema'] = table.schema
# trick sa's model registry to think the model is the correct name
if model_name != 'Temporal':
Temporal._decl_class_registry[
model_name] = Temporal._decl_class_registry['Temporal']
del Temporal._decl_class_registry['Temporal']
# add in single relations
fks = self.get_single_foreign_keys_by_column(table)
for column, fk in fks.iteritems():
related_table = fk.column.table
if related_table not in self.tables:
continue
log.info(' Adding <primary> foreign key for:%s' %
related_table.name)
rel = relation(
singular(name2label(related_table.name, related_table.schema)),
primaryjoin=column == fk.column) # , backref=backref_name)
setattr(Temporal, related_table.name,
_deferred_relationship(Temporal, rel))
# add in the relations for the composites
for constraint in table.constraints:
if isinstance(constraint, ForeignKeyConstraint):
if len(constraint.elements) > 1:
related_table = constraint.elements[0].column.table
related_classname = singular(
name2label(related_table.name, related_table.schema))
primary_join = "and_(%s)" % ', '.join([
"%s.%s==%s.%s" % (model_name, k.parent.name,
related_classname, k.column.name)
for k in constraint.elements
])
rel = relation(
related_classname,
primaryjoin=primary_join
# foreign_keys=[k.parent for k in constraint.elements]
)
rel._as_string = primary_join
setattr(Temporal, related_table.name,
rel) # _deferred_relationship(Temporal, rel))
# add in many-to-many relations
for join_table in self.get_related_many_to_many_tables(table.name):
if join_table not in self.tables:
continue
primary_column = [
c for c in join_table.columns if c.foreign_keys
and list(c.foreign_keys)[0].column.table == table
][0]
for column in join_table.columns:
if column.foreign_keys:
key = list(column.foreign_keys)[0]
if key.column.table is not table:
related_column = related_table = list(
column.foreign_keys)[0].column
related_table = related_column.table
if related_table not in self.tables:
continue
log.info(
' Adding <secondary> foreign key(%s) for:%s' %
(key, related_table.name))
setattr(
Temporal, plural(related_table.name),
_deferred_relationship(
Temporal,
relation(
singular(
name2label(related_table.name,
related_table.schema)),
secondary=join_table,
primaryjoin=list(
primary_column.foreign_keys)[0].column
== primary_column,
secondaryjoin=column == related_column)))
break
return Temporal
def cropPlaylist(self, number):
self._sendDebug(
"Cropping playlist by " + str(number) + " track"
+ util.plural(number) + ".")
for _ in range(number):
self.deleteTrack(0)
def body_lines(self):
# Metadata whitelist
metadata_whitelist = [
'hostname',
'fqdn',
'ipv4',
'instance-id'
]
lines = [
'Clocks',
'======',
''
]
try:
ntp_offset, ntp_styles = get_ntp_info()
lines.append(' ' + style('NTP offset', *ntp_styles) + ': ' + style('%s s' % round(ntp_offset, 4), *ntp_styles))
except Exception as e:
lines.append(' NTP offset: Unknown (%s)' % str(e))
lines.append(' System UTC time: ' + datetime.datetime.utcnow().__str__())
lines.append('')
# Paths to checks.d/conf.d
lines += [
'Paths',
'=====',
''
]
osname = config.get_os()
try:
confd_path = config.get_confd_path(osname)
except config.PathNotFound:
confd_path = 'Not found'
try:
checksd_path = config.get_checksd_path(osname)
except config.PathNotFound:
checksd_path = 'Not found'
lines.append(' conf.d: ' + confd_path)
lines.append(' checks.d: ' + checksd_path)
lines.append('')
# Hostnames
lines += [
'Hostnames',
'=========',
''
]
if not self.host_metadata:
lines.append(" No host information available yet.")
else:
for key, host in self.host_metadata.iteritems():
for whitelist_item in metadata_whitelist:
if whitelist_item in key:
lines.append(" " + key + ": " + host)
break
lines.append('')
# Checks.d Status
lines += [
'Checks',
'======',
''
]
check_statuses = self.check_statuses + get_jmx_status()
if not check_statuses:
lines.append(" No checks have run yet.")
else:
for cs in check_statuses:
check_lines = [
' ' + cs.name + ' ({})'.format(cs.check_version),
' ' + '-' * (len(cs.name) + 3 + len(cs.check_version))
]
if cs.init_failed_error:
check_lines.append(" - initialize check class [%s]: %s" %
(style(STATUS_ERROR, 'red'),
repr(cs.init_failed_error)))
if self.verbose and cs.init_failed_traceback:
check_lines.extend(' ' + line for line in
cs.init_failed_traceback.split('\n'))
else:
for s in cs.instance_statuses:
c = 'green'
if s.has_warnings():
c = 'yellow'
if s.has_error():
c = 'red'
line = " - instance #%s [%s]" % (
s.instance_id, style(s.status, c))
if s.has_error():
line += u": %s" % s.error
if s.metric_count is not None:
line += " collected %s metrics" % s.metric_count
if s.instance_check_stats is not None:
line += " Last run duration: %s" % s.instance_check_stats.get('run_time')
check_lines.append(line)
if s.has_warnings():
for warning in s.warnings:
warn = warning.split('\n')
if not len(warn):
continue
check_lines.append(u" %s: %s" %
(style("Warning", 'yellow'), warn[0]))
check_lines.extend(u" %s" % l for l in
warn[1:])
if self.verbose and s.traceback is not None:
check_lines.extend(' ' + line for line in
s.traceback.split('\n'))
check_lines += [
" - Collected %s metric%s, %s event%s & %s service check%s" % (
cs.metric_count, plural(cs.metric_count),
cs.event_count, plural(cs.event_count),
cs.service_check_count, plural(cs.service_check_count)),
]
if cs.check_stats is not None:
check_lines += [
" - Stats: %s" % pretty_statistics(cs.check_stats)
]
if cs.library_versions is not None:
check_lines += [
" - Dependencies:"]
for library, version in cs.library_versions.iteritems():
check_lines += [
" - %s: %s" % (library, version)]
check_lines += [""]
lines += check_lines
# Metadata status
metadata_enabled = 1#_is_affirmative(get_config().get('display_service_metadata', False))
if metadata_enabled:
lines += [
"",
"Service metadata",
"================",
""
]
if not check_statuses:
lines.append(" No checks have run yet.")
else:
meta_lines = []
for cs in check_statuses:
# Check title
check_line = [
' ' + cs.name,
' ' + '-' * len(cs.name)
]
instance_lines = []
for i, meta in enumerate(cs.service_metadata):
if not meta:
continue
instance_lines += [" - instance #%s:" % i]
for k, v in meta.iteritems():
instance_lines += [" - %s: %s" % (k, v)]
if instance_lines:
check_line += instance_lines
meta_lines += check_line
if meta_lines:
lines += meta_lines
else:
lines.append(" No metadata were collected.")
# Emitter status
lines += [
"",
"Emitters",
"========",
""
]
if not self.emitter_statuses:
lines.append(" No emitters have run yet.")
else:
for es in self.emitter_statuses:
c = 'green'
if es.has_error():
c = 'red'
line = " - %s [%s]" % (es.name, style(es.status, c))
if es.status != STATUS_OK:
line += ": %s" % es.error
lines.append(line)
return lines
def prompt(choices, mode='*'):
if mode not in PROMPT_MODES:
raise ValueError("mode '{}' is invalid".format(mode))
if len(choices) > 26:
raise ValueError("too many choices")
if mode == '*':
header = "select zero or more:"
max, min = float('inf'), 0
elif mode == '+':
header = "select one or more:"
max, min = float('inf'), 1
elif mode in [1, '1']:
header = "select one:"
max, min = 1, 1
elif mode == '?':
header = "select zero or one:"
max, min = 1, 0
letters = list(map(lambda x: chr(ord('a') + x), range(len(choices))))
num_selections = 0
selections = [] # unique indices into choices list
while num_selections < min or num_selections < max:
util.print(util.green(header))
for i in range(len(choices)):
if i in selections:
choice = " × "
else:
choice = " "
choice += str(letters[i]) + '. ' + str(choices[i])
if i in selections:
choice = util.yellow(choice)
util.print(choice)
try:
sel = input(util.green("make a selection (or ! to commit): "))
except KeyboardInterrupt:
util.exit(util.ERR_INTERRUPTED)
if sel == '!':
if num_selections < min:
util.error("can't stop now; you must make "
"{} {}".format(min,
util.plural("selection", min)))
continue
else:
break
try:
if letters.index(sel) in selections:
selections.remove(letters.index(sel))
continue
selections.append(letters.index(sel))
num_selections += 1
except ValueError:
if sel == '':
util.print("make a selection (or ! to commit)")
else:
util.error("invalid selection: not in list")
continue
return selections
def export_chats(chats,
path,
format,
db,
messages=None,
skip=True,
progress=None):
"""
Exports the specified chats from the database under path.
@param chats list of chat dicts, as returned from SkypeDatabase
@param path full path of directory where to save
@param format export format (html|txt|xlsx|csv|filename.ext).
If format is filename.ext, a single file is created:
for single chat exports and multi chat XLSX exports
(for multi-file exports, filenames are named by chats).
@param db SkypeDatabase instance
@param messages list messages to export if a single chat
@param skip whether to skip chats with no messages
@param progress function called before exporting each chat, with the
number of messages exported so far
@return (list of exported filenames, number of chats exported)
"""
files, count = [], 0
def make_filename(chat):
if len(format) > 4: # Filename already given in format
filename = os.path.join(path, format)
else:
args = collections.defaultdict(str)
args.update(chat)
filename = "%s.%s" % (conf.ExportChatTemplate % args, format)
filename = os.path.join(path, util.safe_filename(filename))
filename = util.unique_path(filename)
return filename
main.logstatus("Exporting %s from %s %sto %s.", util.plural("chat", chats),
db.filename,
"" if len(format) > 4 else "as %s " % format.upper(),
format if len(format) > 4 else path)
if format.lower().endswith(".xlsx"):
filename = make_filename(chats[0])
count = export_chats_xlsx(chats, filename, db, messages, skip,
progress)
files.append(filename)
else:
if not os.path.exists(path):
os.makedirs(path)
export_func = (export_chats_xlsx if format.lower().endswith("xlsx")
else export_chat_csv if format.lower().endswith("csv")
else export_chat_template)
message_count = 0
for chat in chats:
if skip and not messages and not chat["message_count"]:
main.log("Skipping exporting %s: no messages.",
chat["title_long_lc"])
if progress: progress(message_count)
continue # continue for chat in chats
main.status("Exporting %s.", chat["title_long_lc"])
if progress: progress(message_count)
filename = make_filename(chat)
msgs = messages or db.get_messages(chat)
chatarg = [chat] if "xlsx" == format.lower() else chat
export_func(chatarg, filename, db, msgs)
message_count += chat["message_count"]
files.append(filename)
count = len(files)
return (files, count)
def body_lines(self):
# Metadata whitelist
metadata_whitelist = [
'hostname',
'fqdn',
'ipv4',
'instance-id'
]
lines = [
'Clocks',
'======',
''
]
try:
ntp_offset, ntp_styles = get_ntp_info()
lines.append(' ' + style('NTP offset', *ntp_styles) + ': ' + style('%s s' % round(ntp_offset, 4), *ntp_styles))
except Exception as e:
lines.append(' NTP offset: Unknown (%s)' % str(e))
lines.append(' System UTC time: ' + datetime.datetime.utcnow().__str__())
lines.append('')
# Paths to checks.d/conf.d
lines += [
'Paths',
'=====',
''
]
osname = config.get_os()
try:
confd_path = config.get_confd_path(osname)
except config.PathNotFound:
confd_path = 'Not found'
try:
checksd_path = config.get_checksd_path(osname)
except config.PathNotFound:
checksd_path = 'Not found'
lines.append(' conf.d: ' + confd_path)
lines.append(' checks.d: ' + checksd_path)
lines.append('')
# Hostnames
lines += [
'Hostnames',
'=========',
''
]
if not self.host_metadata:
lines.append(" No host information available yet.")
else:
for key, host in self.host_metadata.iteritems():
for whitelist_item in metadata_whitelist:
if whitelist_item in key:
lines.append(" " + key + ": " + host)
break
lines.append('')
# Checks.d Status
lines += [
'Checks',
'======',
''
]
check_statuses = self.check_statuses + get_jmx_status()
if not check_statuses:
lines.append(" No checks have run yet.")
else:
for cs in check_statuses:
check_lines = [
' ' + cs.name + ' ({})'.format(cs.check_version),
' ' + '-' * (len(cs.name) + 3 + len(cs.check_version))
]
if cs.init_failed_error:
check_lines.append(" - initialize check class [%s]: %s" %
(style(STATUS_ERROR, 'red'),
repr(cs.init_failed_error)))
if self.verbose and cs.init_failed_traceback:
check_lines.extend(' ' + line for line in
cs.init_failed_traceback.split('\n'))
else:
for s in cs.instance_statuses:
c = 'green'
if s.has_warnings():
c = 'yellow'
if s.has_error():
c = 'red'
line = " - instance #%s [%s]" % (
s.instance_id, style(s.status, c))
if s.has_error():
line += u": %s" % s.error
if s.metric_count is not None:
line += " collected %s metrics" % s.metric_count
if s.instance_check_stats is not None:
line += " Last run duration: %s" % s.instance_check_stats.get('run_time')
check_lines.append(line)
if s.has_warnings():
for warning in s.warnings:
warn = warning.split('\n')
if not len(warn):
continue
check_lines.append(u" %s: %s" %
(style("Warning", 'yellow'), warn[0]))
check_lines.extend(u" %s" % l for l in
warn[1:])
if self.verbose and s.traceback is not None:
check_lines.extend(' ' + line for line in
s.traceback.split('\n'))
check_lines += [
" - Collected %s metric%s, %s event%s & %s service check%s" % (
cs.metric_count, plural(cs.metric_count),
cs.event_count, plural(cs.event_count),
cs.service_check_count, plural(cs.service_check_count)),
]
if cs.check_stats is not None:
check_lines += [
" - Stats: %s" % pretty_statistics(cs.check_stats)
]
if cs.library_versions is not None:
check_lines += [
" - Dependencies:"]
for library, version in cs.library_versions.iteritems():
check_lines += [
" - %s: %s" % (library, version)]
check_lines += [""]
lines += check_lines
# Metadata status
metadata_enabled = _is_affirmative(get_config().get('display_service_metadata', False))
if metadata_enabled:
lines += [
"",
"Service metadata",
"================",
""
]
if not check_statuses:
lines.append(" No checks have run yet.")
else:
meta_lines = []
for cs in check_statuses:
# Check title
check_line = [
' ' + cs.name,
' ' + '-' * len(cs.name)
]
instance_lines = []
for i, meta in enumerate(cs.service_metadata):
if not meta:
continue
instance_lines += [" - instance #%s:" % i]
for k, v in meta.iteritems():
instance_lines += [" - %s: %s" % (k, v)]
if instance_lines:
check_line += instance_lines
meta_lines += check_line
if meta_lines:
lines += meta_lines
else:
lines.append(" No metadata were collected.")
# Emitter status
lines += [
"",
"Emitters",
"========",
""
]
if not self.emitter_statuses:
lines.append(" No emitters have run yet.")
else:
for es in self.emitter_statuses:
c = 'green'
if es.has_error():
c = 'red'
line = " - %s [%s]" % (es.name, style(es.status, c))
if es.status != STATUS_OK:
line += ": %s" % es.error
lines.append(line)
return lines
class CollectorStatus(AgentStatus):
NAME = 'Collector'
def __init__(self,
check_statuses=None,
emitter_statuses=None,
metadata=None):
AgentStatus.__init__(self)
self.check_statuses = check_statuses or []
self.emitter_statuses = emitter_statuses or []
self.metadata = metadata or []
@property
def status(self):
for check_status in self.check_statuses:
if check_status.status == STATUS_ERROR:
return STATUS_ERROR
return STATUS_OK
def has_error(self):
return self.status != STATUS_OK
def body_lines(self):
# Metadata whitelist
metadata_whitelist = ['hostname', 'fqdn', 'ipv4', 'instance-id']
lines = ['Clocks', '======', '']
try:
ntp_offset, ntp_styles = get_ntp_info()
lines.append(' ' + style('NTP offset', *ntp_styles) + ': ' +
style('%s s' % round(ntp_offset, 4), *ntp_styles))
except Exception, e:
lines.append(' NTP offset: Unknown (%s)' % str(e))
lines.append(' System UTC time: ' +
datetime.datetime.utcnow().__str__())
lines.append('')
# Paths to checks.d/conf.d
lines += ['Paths', '=====', '']
osname = config.get_os()
try:
confd_path = config.get_confd_path(osname)
except config.PathNotFound:
confd_path = 'Not found'
try:
checksd_path = config.get_checksd_path(osname)
except config.PathNotFound:
checksd_path = 'Not found'
lines.append(' conf.d: ' + confd_path)
lines.append(' checks.d: ' + checksd_path)
lines.append('')
# Hostnames
lines += ['Hostnames', '=========', '']
if not self.metadata:
lines.append(" No host information available yet.")
else:
for key, host in self.metadata.items():
for whitelist_item in metadata_whitelist:
if whitelist_item in key:
lines.append(" " + key + ": " + host)
break
lines.append('')
# Checks.d Status
lines += ['Checks', '======', '']
check_statuses = self.check_statuses + get_jmx_status()
if not check_statuses:
lines.append(" No checks have run yet.")
else:
for cs in check_statuses:
check_lines = [' ' + cs.name, ' ' + '-' * len(cs.name)]
if cs.init_failed_error:
check_lines.append(
" - initialize check class [%s]: %s" % (style(
STATUS_ERROR, 'red'), repr(cs.init_failed_error)))
if self.verbose and cs.init_failed_traceback:
check_lines.extend(
' ' + line
for line in cs.init_failed_traceback.split('\n'))
else:
for s in cs.instance_statuses:
c = 'green'
if s.has_warnings():
c = 'yellow'
if s.has_error():
c = 'red'
line = " - instance #%s [%s]" % (s.instance_id,
style(s.status, c))
if s.has_error():
line += u": %s" % s.error
if s.metric_count is not None:
line += " collected %s metrics" % s.metric_count
check_lines.append(line)
if s.has_warnings():
for warning in s.warnings:
warn = warning.split('\n')
if not len(warn): continue
check_lines.append(
u" %s: %s" %
(style("Warning", 'yellow'), warn[0]))
check_lines.extend(u" %s" % l
for l in warn[1:])
if self.verbose and s.traceback is not None:
check_lines.extend(
' ' + line
for line in s.traceback.split('\n'))
check_lines += [
" - Collected %s metric%s, %s event%s & %s service check%s"
% (cs.metric_count, plural(
cs.metric_count), cs.event_count,
plural(cs.event_count), cs.service_check_count,
plural(cs.service_check_count)),
]
if cs.library_versions is not None:
check_lines += [" - Dependencies:"]
for library, version in cs.library_versions.iteritems(
):
check_lines += [
" - %s: %s" % (library, version)
]
check_lines += [""]
lines += check_lines
# Emitter status
lines += ["", "Emitters", "========", ""]
if not self.emitter_statuses:
lines.append(" No emitters have run yet.")
else:
for es in self.emitter_statuses:
c = 'green'
if es.has_error():
c = 'red'
line = " - %s [%s]" % (es.name, style(es.status, c))
if es.status != STATUS_OK:
line += ": %s" % es.error
lines.append(line)
return lines
warn = warning.split('\n')
if not len(warn):
continue
check_lines.append(
u" %s: %s" %
(style("Warning", 'yellow'), warn[0]))
check_lines.extend(u" %s" % l
for l in warn[1:])
if self.verbose and s.traceback is not None:
check_lines.extend(
' ' + line
for line in s.traceback.split('\n'))
check_lines += [
" - Collected %s metric%s, %s event%s & %s service check%s"
% (cs.metric_count, plural(
cs.metric_count), cs.event_count,
plural(cs.event_count), cs.service_check_count,
plural(cs.service_check_count)),
]
if cs.check_stats is not None:
check_lines += [
" - Stats: %s" %
pretty_statistics(cs.check_stats)
]
if cs.library_versions is not None:
check_lines += [" - Dependencies:"]
for library, version in cs.library_versions.iteritems(
):
check_lines += [
def run_export(filenames, format, chatnames, authornames, ask_password):
"""Exports the specified databases in specified format."""
dbs = [skypedata.SkypeDatabase(f) for f in filenames]
is_xlsx_single = ("xlsx_single" == format)
for db in dbs:
if (ask_password and db.id and conf.SharedImageAutoDownload
and format.lower().endswith("html")):
prompt = "Enter Skype password for '%s': " % db.id
while not skypedata.SharedImageDownload.has_login(db.id):
with warnings.catch_warnings():
warnings.simplefilter("ignore") # possible GetPassWarning
output(prompt, end="") # getpass output can raise errors
pw = getpass.getpass("", io.BytesIO())
if not pw: continue # while
try:
skypedata.SharedImageDownload.login(db.id, pw)
except Exception as e:
log("Error signing in %s on Skype web.\n\n%s",
db.id, util.format_exc(e))
prompt = "%s\nEnter Skype password for '%s': " % (e, db.id)
formatargs = collections.defaultdict(str)
formatargs["skypename"] = os.path.basename(db.filename)
formatargs.update(db.account or {})
basename = util.safe_filename(conf.ExportDbTemplate % formatargs)
dbstr = "from %s " % db if len(dbs) != 1 else ""
if is_xlsx_single:
export_dir = os.getcwd()
filename = util.unique_path("%s.xlsx" % basename)
else:
export_dir = util.unique_path(os.path.join(os.getcwd(), basename))
filename = format
target = filename if is_xlsx_single else export_dir
try:
extras = [("", chatnames)] if chatnames else []
extras += [(" with authors", authornames)] if authornames else []
output("Exporting%s%s as %s %sto %s." %
(" chats" if extras else "",
",".join("%s like %s" % (x, y) for x, y in extras),
format[:4].upper(), dbstr, target))
chats = sorted(db.get_conversations(chatnames, authornames),
key=lambda x: x["title"].lower())
db.get_conversations_stats(chats)
bar_total = sum(c["message_count"] for c in chats)
bartext = " Exporting %.*s.." % (30, db.filename) # Enforce width
bar = ProgressBar(max=bar_total, afterword=bartext)
bar.start()
result = export.export_chats(chats, export_dir, filename, db,
progress=bar.update)
files, count = result
bar.stop()
if count:
bar.afterword = " Exported %s to %s. " % (db, target)
bar.update(bar_total)
output()
log("Exported %s %sto %s as %s.", util.plural("chat", count),
dbstr, target, format)
else:
output("\nNo messages to export%s." %
("" if len(dbs) == 1 else " from %s" % db))
os.unlink(filename) if is_xlsx_single else os.rmdir(export_dir)
except Exception as e:
output("Error exporting chats: %s\n\n%s" %
(e, traceback.format_exc()))
def run(self):
self._is_running = True
while self._is_running:
params = self._queue.get()
self._stop_work = False
self._drop_results = False
if params:
# {"htmls": [html result for db1, db2],
# "chats": [differing chats in db1, db2]}
result = {
"htmls": ["", ""],
"chats": [[], []],
"params": params
}
db1, db2 = params["db1"], params["db2"]
chats1 = db1.get_conversations()
chats2 = db2.get_conversations()
skypenames1 = [i["identity"] for i in db1.get_contacts()]
skypenames2 = [i["identity"] for i in db2.get_contacts()]
skypenames1.append(db1.id)
skypenames2.append(db2.id)
c1map = dict((c["identity"], c) for c in chats1)
c2map = dict((c["identity"], c) for c in chats2)
compared = []
for c1 in chats1:
c2 = c2map.get(c1["identity"], None)
c = c1.copy()
c["messages1"] = c1["message_count"] or 0
c["messages2"] = c2["message_count"] or 0 if c2 else 0
c["c1"] = c1
c["c2"] = c2
compared.append(c)
for c2 in chats2:
c1 = c1map.get(c2["identity"], None)
if not c1:
c = c2.copy()
c["messages2"] = c["message_count"] or 0
c["messages1"] = 0
c["c1"] = c1
c["c2"] = c2
compared.append(c)
compared.sort(key=lambda x: x["title"])
for chat in compared:
diff = self.get_chat_diff(chat, db1, db2)
if self._stop_work:
break # break for chat in compared
for i in range(2):
new_chat = not chat["c1" if i else "c2"]
newstr = "" if new_chat else "new "
info = export.htmltag("a", {"href": chat["identity"]},
chat["title_long"],
utf=False)
if new_chat:
info += " - new chat"
skypenames_other = [skypenames1, skypenames2][1 - i]
if diff["messages"][i]:
info += ", %s" % (util.plural(
"%smessage" % newstr, len(
diff["messages"][i])))
if diff["participants"][i] and newstr:
info += ", %s" % (util.plural(
"%sparticipant" % newstr,
len(diff["participants"][i])))
if diff["messages"][i] or diff["participants"][i]:
info += ".<br />"
result["htmls"][i] += info
result["chats"][i].append({
"chat": chat,
"diff": diff
})
if not self._drop_results \
and not len(result["chats"][i]) \
% conf.DiffResultsChunk:
self._postback(result)
result = {
"htmls": ["", ""],
"chats": [[], []],
"params": params
}
if not self._drop_results:
result["done"] = True
self._postback(result)
