def to_templated_secular_display(self, fp, group, type='lecture'): gt = GroupTemplate(None) gt.add_patterns(fp) gt.calculate_reduction(True) out = [] for (_, _, (pattern, mult)) in gt.get_patterns_raw(False): if pattern is None: # just multiplier, should be just "x lectures" row = "%d %s, %s" % (mult, util.plural(type), gt.template ) # XXX proper plurals elif mult is None: # traditional pattern, expand codes sensibly row = "%s %s" % (util.plural(type), self.atoms_to_secular(fp.patterns(), True)) else: # start date and multiplier pp = copy.deepcopy(gt.template) pp.setAllYear() row = "%s %s, starting on %s, %s" % (mult, util.plural(type), self.atoms_to_secular( [pattern], False), pp) out.append(row) prefix = '' if group is not None: prefix = "%s term, " % term_names[group.term] return prefix + ", ".join(out)
def run_merge(filenames): """Merges all Skype databases to a new database.""" dbs = [skypedata.SkypeDatabase(f) for f in filenames] db_base = dbs.pop() counts = collections.defaultdict(lambda: collections.defaultdict(int)) postbacks = Queue.Queue() postfunc = lambda r: postbacks.put(r) worker = workers.MergeThread(postfunc) name, ext = os.path.splitext(os.path.split(db_base.filename)[-1]) now = datetime.datetime.now().strftime("%Y%m%d") filename_final = util.unique_path("%s.merged.%s%s" % (name, now, ext)) print("Creating %s, using %s as base." % (filename_final, db_base)) shutil.copyfile(db_base.filename, filename_final) db2 = skypedata.SkypeDatabase(filename_final) chats2 = db2.get_conversations() db2.get_conversations_stats(chats2) for db1 in dbs: chats = db1.get_conversations() db1.get_conversations_stats(chats) bar_total = sum(c["message_count"] for c in chats) bar_text = " Processing %.*s.." % (30, db1) bar = ProgressBar(max=bar_total, afterword=bar_text) bar.start() args = {"db1": db1, "db2": db2, "chats": chats, "type": "diff_merge_left"} worker.work(args) while True: result = postbacks.get() if "error" in result: print("Error merging %s:\n\n%s" % (db1, result["error"])) worker = None # Signal for global break break # break while True if "done" in result: break # break while True if "diff" in result: counts[db1]["chats"] += 1 counts[db1]["msgs"] += len(result["diff"]["messages"]) msgcounts = sum(c["message_count"] for c in result["chats"]) bar.update(bar.value + msgcounts) if result["output"]: log(result["output"]) if not worker: break # break for db1 in dbs bar.stop() bar.afterword = " Processed %s." % db1 bar.update(bar_total) print if not counts: print("Nothing new to merge.") db2.close() os.unlink(filename_final) else: for db1 in dbs: print("Merged %s in %s from %s." % (util.plural("message", counts[db1]["msgs"]), util.plural("chat", counts[db1]["chats"]), db1)) print("Merge into %s complete." % db2)
def work_diff_left(self, params): """ Worker branch that compares all chats on the left side for differences, posting results back to application. """ # {"output": "html result for db1, db2", # "index": currently processed chat index, # "chats": [differing chats in db1]} result = {"output": "", "chats": [], "params": params, "index": 0, "type": "diff_left"} db1, db2 = params["db1"], params["db2"] chats1 = params.get("chats") or db1.get_conversations() chats2 = db2.get_conversations() c1map = dict((c["identity"], c) for c in chats1) c2map = dict((c["identity"], c) for c in chats2) compared = [] for c1 in chats1: c2 = c2map.get(c1["identity"]) c = c1.copy() c["messages1"] = c1["message_count"] or 0 c["messages2"] = c2["message_count"] or 0 if c2 else 0 c["c1"], c["c2"] = c1, c2 compared.append(c) compared.sort(key=lambda x: x["title"].lower()) info_template = step.Template(templates.DIFF_RESULT_ITEM) for index, chat in enumerate(compared): diff = self.get_chat_diff_left(chat, db1, db2) if self._stop_work: break # break for index, chat in enumerate(compared) if diff["messages"] \ or (chat["message_count"] and diff["participants"]): new_chat = not chat["c2"] newstr = "" if new_chat else "new " info = info_template.expand(chat=chat) if new_chat: info += " - new chat" if diff["messages"]: info += ", %s" % util.plural("%smessage" % newstr, diff["messages"]) else: info += ", no messages" if diff["participants"] and not new_chat: info += ", %s" % ( util.plural("%sparticipant" % newstr, diff["participants"])) info += ".<br />" result["output"] += info result["chats"].append({"chat": chat, "diff": diff}) result["index"] = index if not self._drop_results: if index < len(compared) - 1: result["status"] = ("Scanning %s." % compared[index + 1]["title_long_lc"]) self.postback(result) result = {"output": "", "chats": [], "index": index, "params": params, "type": "diff_left"} if not self._drop_results: result["done"] = True self.postback(result)
def flush(self): try: self.flush_count += 1 self.log_count += 1 packets_per_second = self.metrics_aggregator.packets_per_second( self.interval) packet_count = self.metrics_aggregator.total_count metrics = self.metrics_aggregator.flush() count = len(metrics) if self.flush_count % FLUSH_LOGGING_PERIOD == 0: self.log_count = 0 if count: self.submit(metrics) events = self.metrics_aggregator.flush_events() event_count = len(events) if event_count: self.submit_events(events) service_checks = self.metrics_aggregator.flush_service_checks() service_check_count = len(service_checks) if service_check_count: self.submit_service_checks(service_checks) should_log = self.flush_count <= FLUSH_LOGGING_INITIAL or self.log_count <= FLUSH_LOGGING_COUNT log_func = log.info if not should_log: log_func = log.debug log_func( "Flush #%s: flushed %s metric%s, %s event%s, and %s service check run%s" % (self.flush_count, count, plural(count), event_count, plural(event_count), service_check_count, plural(service_check_count))) if self.flush_count == FLUSH_LOGGING_INITIAL: log.info( "First flushes done, %s flushes will be logged every %s flushes." % (FLUSH_LOGGING_COUNT, FLUSH_LOGGING_PERIOD)) # Persist a status message. packet_count = self.metrics_aggregator.total_count MonitorstatsdStatus( flush_count=self.flush_count, packet_count=packet_count, packets_per_second=packets_per_second, metric_count=count, event_count=event_count, service_check_count=service_check_count, ).persist() except Exception: if self.finished.isSet(): log.debug( "Couldn't flush metrics, but that's expected as we're stopping" ) else: log.exception("Error flushing metrics")
def check_status_lines(cs): check_lines = [" " + cs.name, " " + "-" * len(cs.name)] if cs.init_failed_error: check_lines.append( " - initialize check class [%s]: %s" % (style(STATUS_ERROR, "red"), repr(cs.init_failed_error)) ) if cs.init_failed_traceback: check_lines.extend(" " + line for line in cs.init_failed_traceback.split("\n")) else: for s in cs.instance_statuses: c = "green" if s.has_warnings(): c = "yellow" if s.has_error(): c = "red" line = " - instance #%s [%s]" % (s.instance_id, style(s.status, c)) if s.has_error(): line += u": %s" % s.error if s.metric_count is not None: line += " collected %s metrics" % s.metric_count if s.instance_check_stats is not None: line += " Last run duration: %s" % s.instance_check_stats.get("run_time") check_lines.append(line) if s.has_warnings(): for warning in s.warnings: warn = warning.split("\n") if not len(warn): continue check_lines.append(u" %s: %s" % (style("Warning", "yellow"), warn[0])) check_lines.extend(u" %s" % l for l in warn[1:]) if s.traceback is not None: check_lines.extend(" " + line for line in s.traceback.split("\n")) check_lines += [ " - Collected %s metric%s, %s event%s & %s service check%s" % ( cs.metric_count, plural(cs.metric_count), cs.event_count, plural(cs.event_count), cs.service_check_count, plural(cs.service_check_count), ) ] if cs.check_stats is not None: check_lines += [" - Stats: %s" % pretty_statistics(cs.check_stats)] if cs.library_versions is not None: check_lines += [" - Dependencies:"] for library, version in cs.library_versions.iteritems(): check_lines += [" - %s: %s" % (library, version)] check_lines += [""] return check_lines
def flush(self): if self._trs_to_flush is not None: log.debug("A flush is already in progress, not doing anything") return to_flush = [] # Do we have something to do ? now = datetime.utcnow() for tr in self._transactions: if tr.time_to_flush(now): to_flush.append(tr) count = len(to_flush) should_log = self._flush_count + 1 <= FLUSH_LOGGING_INITIAL or ( self._flush_count + 1) % FLUSH_LOGGING_PERIOD == 0 if count > 0: if should_log: log.info("Flushing %s transaction%s during flush #%s" % (count, plural(count), str(self._flush_count + 1))) else: log.debug("Flushing %s transaction%s during flush #%s" % (count, plural(count), str(self._flush_count + 1))) self._endpoints_errors = {} self._finished_flushes = 0 # We sort LIFO-style, taking into account errors self._trs_to_flush = sorted(to_flush, key=lambda tr: (-tr._error_count, tr._id)) self._flush_time = datetime.utcnow() self.flush_next() else: if should_log: log.info("No transaction to flush during flush #%s" % str(self._flush_count + 1)) else: log.debug("No transaction to flush during flush #%s" % str(self._flush_count + 1)) if self._flush_count + 1 == FLUSH_LOGGING_INITIAL: log.info( "First flushes done, next flushes will be logged every %s flushes." % FLUSH_LOGGING_PERIOD) self._flush_count += 1 ForwarderStatus( queue_length=self._total_count, queue_size=self._total_size, flush_count=self._flush_count, transactions_received=self._transactions_received, transactions_flushed=self._transactions_flushed, transactions_rejected=self._transactions_rejected).persist()
def work_merge(self, params): """ Worker branch that merges differences given in params, posting progress back to application. """ error, e = None, None db1, db2, info = params["db1"], params["db2"], params["info"] chats, contacts = params["chats"], params["contacts"] source, contactgroups = params["source"], params["contactgroups"] count_messages = 0 count_participants = 0 try: if contacts: content = util.plural("contact", contacts) self.postback({"type": "merge", "gauge": 0, "message": "Merging %s." % content}) db2.insert_contacts(contacts, db1) self.postback({"type": "merge", "gauge": 100, "message": "Merged %s." % content}) if contactgroups: content = util.plural("contact group", contactgroups) self.postback({"type": "merge", "gauge": 0, "message": "Merging %s." % content}) db2.replace_contactgroups(contactgroups, db1) self.postback({"type": "merge", "gauge": 100, "message": "Merged %s." % content}) for index, chat_data in enumerate(chats): if self._stop_work: break # break for i, chat_data in enumerate(chats) chat1 = chat_data["chat"]["c2" if source else "c1"] chat2 = chat_data["chat"]["c1" if source else "c2"] step = -1 if source else 1 messages1, messages2 = chat_data["diff"]["messages"][::step] participants, participants2 = \ chat_data["diff"]["participants"][::step] if not chat2: chat2 = chat1.copy() chat_data["chat"]["c1" if source else "c2"] = chat2 chat2["id"] = db2.insert_chat(chat2, db1) if participants: db2.insert_participants(chat2, participants, db1) count_participants += len(participants) if messages1: db2.insert_messages(chat2, messages1, db1, chat1, self.yield_ui, self.REFRESH_COUNT) count_messages += len(messages1) self.postback({"type": "merge", "index": index, "params": params}) except Exception, e: error = traceback.format_exc()
def run_diff(filename1, filename2): """Compares the first database for changes with the second.""" if os.path.realpath(filename1) == os.path.realpath(filename2): output("Error: cannot compare %s with itself." % filename1) return db1, db2 = map(skypedata.SkypeDatabase, [filename1, filename2]) counts = collections.defaultdict(lambda: collections.defaultdict(int)) postbacks = Queue.Queue() bar_text = "%.*s.." % (50, " Scanning %s vs %s" % (db1, db2)) bar = ProgressBar(afterword=bar_text) bar.start() chats1, chats2 = db1.get_conversations(), db2.get_conversations() db1.get_conversations_stats(chats1), db2.get_conversations_stats(chats2) args = {"db1": db1, "db2": db2, "chats": chats1, "type": "diff_left"} worker = workers.MergeThread(postbacks.put) try: worker.work(args) while True: result = postbacks.get() if "error" in result: output("Error scanning %s and %s:\n\n%s" % (db1, db2, result["error"])) break # break while True if "done" in result: break # break while True if "chats" in result and result["chats"]: counts[db1]["chats"] += 1 msgs = len(result["chats"][0]["diff"]["messages"]) msgs_text = util.plural("new message", msgs) contacts_text = util.plural( "new participant", result["chats"][0]["diff"]["participants"]) text = ", ".join(filter(None, [msgs_text, contacts_text])) bar.afterword = (" %s, %s." % (result["chats"][0]["chat"]["title"], text)) counts[db1]["msgs"] += msgs if "index" in result: bar.max = result["count"] bar.update(result["index"]) if result.get("output"): log(result["output"]) finally: worker and (worker.stop(), worker.join()) bar.stop() bar.afterword = " Scanned %s and %s." % (db1, db2) bar.update(bar.max) output()
def run_diff(filename1, filename2): """Compares the first database for changes with the second.""" if os.path.realpath(filename1) == os.path.realpath(filename2): output("Error: cannot compare %s with itself." % filename1) return db1, db2 = map(skypedata.SkypeDatabase, [filename1, filename2]) counts = collections.defaultdict(lambda: collections.defaultdict(int)) postbacks = Queue.Queue() bar_text = "%.*s.." % (50, " Scanning %s vs %s" % (db1, db2)) bar = ProgressBar(afterword=bar_text) bar.start() chats1, chats2 = db1.get_conversations(), db2.get_conversations() db1.get_conversations_stats(chats1), db2.get_conversations_stats(chats2) args = {"db1": db1, "db2": db2, "chats": chats1, "type": "diff_left"} worker = workers.MergeThread(postbacks.put) try: worker.work(args) while True: result = postbacks.get() if "error" in result: output("Error scanning %s and %s:\n\n%s" % (db1, db2, result["error"])) break # break while True if "done" in result: break # break while True if "chats" in result and result["chats"]: counts[db1]["chats"] += 1 msgs = len(result["chats"][0]["diff"]["messages"]) msgs_text = util.plural("new message", msgs) contacts_text = util.plural("new participant", result["chats"][0]["diff"]["participants"]) text = ", ".join(filter(None, [msgs_text, contacts_text])) bar.afterword = (" %s, %s." % (result["chats"][0]["chat"]["title"], text)) counts[db1]["msgs"] += msgs if "index" in result: bar.max = result["count"] bar.update(result["index"]) if result.get("output"): log(result["output"]) finally: worker and (worker.stop(), worker.join()) bar.stop() bar.afterword = " Scanned %s and %s." % (db1, db2) bar.update(bar.max) output()
def run_hooks_for(trigger): from sys import exit from os.path import sep from subprocess import call global _triggers if trigger not in _triggers: raise ValueError("unknown trigger: '" + str(trigger) + "'") hooks = list(set(_hooks[trigger]) - set(_hooks_done[trigger])) num_done = 0 if len(hooks) > 0: util.info("running hooks for trigger '" + str(trigger) + "'") for fname in hooks: rv = call(config.hooks_dir + sep + fname, env=_create_env()) _hooks_done[trigger].append(fname) num_done += 1 if rv != 0: util.error("hook '" + str(fname) + "' exited abnormally") util.exit(util.ERR_ABNORMAL_HOOK_EXIT) util.info("successfully ran " + str(num_done) + " " + \ util.plural('hook', num_done))
def tr_error(self, tr): tr.inc_error_count() tr.compute_next_flush(self._MAX_WAIT_FOR_REPLAY) log.warn( "Transaction %d in error (%s error%s), it will be replayed after %s" % (tr.get_id(), tr.get_error_count(), plural( tr.get_error_count()), tr.get_next_flush()))
def tr_error(self, tr): self._running_flushes -= 1 self._finished_flushes += 1 tr.inc_error_count() tr.compute_next_flush(self._MAX_WAIT_FOR_REPLAY) log.warn("Transaction %d in error (%s error%s), it will be replayed after %s", tr.get_id(), tr.get_error_count(), plural(tr.get_error_count()), tr.get_next_flush()) self._endpoints_errors[tr._endpoint] = self._endpoints_errors.get(tr._endpoint, 0) + 1 # Endpoint failed too many times, it's probably an enpoint issue # Let's avoid blocking on it if self._endpoints_errors[tr._endpoint] == self._MAX_ENDPOINT_ERRORS: new_trs_to_flush = [] for transaction in self._trs_to_flush: if transaction._endpoint != tr._endpoint: new_trs_to_flush.append(transaction) else: transaction.compute_next_flush(self._MAX_WAIT_FOR_REPLAY) log.debug('Endpoint %s seems down, removed %s transaction from current flush', tr._endpoint, len(self._trs_to_flush) - len(new_trs_to_flush)) self._trs_to_flush = new_trs_to_flush
def tr_error(self, tr): tr.inc_error_count() tr.compute_next_flush(self._MAX_WAIT_FOR_REPLAY) log.warn( "Transaction %d in error (%s error%s), it will be replayed after %s" % (tr.get_id(), tr.get_error_count(), plural(tr.get_error_count()), tr.get_next_flush()) )
def tr_error(self, tr): self._running_flushes -= 1 self._finished_flushes += 1 tr.inc_error_count() tr.compute_next_flush(self._MAX_WAIT_FOR_REPLAY) log.warn( "Transaction %d in error (%s error%s), it will be replayed after %s", tr.get_id(), tr.get_error_count(), plural(tr.get_error_count()), tr.get_next_flush()) self._endpoints_errors[tr._endpoint] = self._endpoints_errors.get( tr._endpoint, 0) + 1 # Endpoint failed too many times, it's probably an enpoint issue # Let's avoid blocking on it if self._endpoints_errors[tr._endpoint] == self._MAX_ENDPOINT_ERRORS: new_trs_to_flush = [] for transaction in self._trs_to_flush: if transaction._endpoint != tr._endpoint: new_trs_to_flush.append(transaction) else: transaction.compute_next_flush(self._MAX_WAIT_FOR_REPLAY) log.debug( 'Endpoint %s seems down, removed %s transaction from current flush', tr._endpoint, len(self._trs_to_flush) - len(new_trs_to_flush)) self._trs_to_flush = new_trs_to_flush
def flush(self): if self._trs_to_flush is not None: log.debug("A flush is already in progress, not doing anything") return to_flush = [] # Do we have something to do ? now = datetime.now() for tr in self._transactions: if tr.time_to_flush(now): to_flush.append(tr) count = len(to_flush) should_log = ( self._flush_count + 1 <= FLUSH_LOGGING_INITIAL or (self._flush_count + 1) % FLUSH_LOGGING_PERIOD == 0 ) if count > 0: if should_log: log.info( "Flushing %s transaction%s during flush #%s" % (count, plural(count), str(self._flush_count + 1)) ) else: log.debug( "Flushing %s transaction%s during flush #%s" % (count, plural(count), str(self._flush_count + 1)) ) self._trs_to_flush = to_flush self.flush_next() else: if should_log: log.info("No transaction to flush during flush #%s" % str(self._flush_count + 1)) else: log.debug("No transaction to flush during flush #%s" % str(self._flush_count + 1)) if self._flush_count + 1 == FLUSH_LOGGING_INITIAL: log.info("First flushes done, next flushes will be logged every %s flushes." % FLUSH_LOGGING_PERIOD) self._flush_count += 1 ForwarderStatus( queue_length=self._total_count, queue_size=self._total_size, flush_count=self._flush_count, transactions_received=self._transactions_received, transactions_flushed=self._transactions_flushed, ).persist()
def export_chats(chats, path, format, db, messages=None, skip=True, progress=None): """ Exports the specified chats from the database under path. @param chats list of chat dicts, as returned from SkypeDatabase @param path full path of directory where to save @param format export format (html|txt|xlsx|csv|filename.ext). If format is filename.ext, a single file is created: for single chat exports and multi chat XLSX exports (for multi-file exports, filenames are named by chats). @param db SkypeDatabase instance @param messages list messages to export if a single chat @param skip whether to skip chats with no messages @param progress function called before exporting each chat, with the number of messages exported so far @return (list of exported filenames, number of chats exported) """ files, count = [], 0 def make_filename(chat): if len(format) > 4: # Filename already given in format filename = os.path.join(path, format) else: args = collections.defaultdict(str); args.update(chat) filename = "%s.%s" % (conf.ExportChatTemplate % args, format) filename = os.path.join(path, util.safe_filename(filename)) filename = util.unique_path(filename) return filename main.logstatus("Exporting %s from %s %sto %s.", util.plural("chat", chats), db.filename, "" if len(format) > 4 else "as %s " % format.upper(), format if len(format) > 4 else path) if format.lower().endswith(".xlsx"): filename = make_filename(chats[0]) count = export_chats_xlsx(chats, filename, db, messages, skip, progress) files.append(filename) else: if not os.path.exists(path): os.makedirs(path) export_func = (export_chats_xlsx if format.lower().endswith("xlsx") else export_chat_csv if format.lower().endswith("csv") else export_chat_template) message_count = 0 for chat in chats: if skip and not messages and not chat["message_count"]: main.log("Skipping exporting %s: no messages.", chat["title_long_lc"]) if progress: progress(message_count) continue # continue for chat in chats main.status("Exporting %s.", chat["title_long_lc"]) if progress: progress(message_count) filename = make_filename(chat) msgs = messages or db.get_messages(chat) chatarg = [chat] if "xlsx" == format.lower() else chat export_func(chatarg, filename, db, msgs) message_count += chat["message_count"] files.append(filename) count = len(files) return (files, count)
def run_diff(filename1, filename2): """Compares the first database for changes with the second.""" if os.path.realpath(filename1) == os.path.realpath(filename2): print("Error: cannot compare %s with itself." % filename1) return db1, db2 = map(skypedata.SkypeDatabase, [filename1, filename2]) counts = collections.defaultdict(lambda: collections.defaultdict(int)) postbacks = Queue.Queue() postfunc = lambda r: postbacks.put(r) worker = workers.MergeThread(postfunc) chats1, chats2 = db1.get_conversations(), db2.get_conversations() db1.get_conversations_stats(chats1), db2.get_conversations_stats(chats2) for db in [db1, db2]: db.get_conversations_stats(db.get_conversations()) bar_total = sum(c["message_count"] for c in chats1) bar_text = "%.*s.." % (50, " Scanning %s vs %s" % (db1, db2)) bar = ProgressBar(max=bar_total, afterword=bar_text) bar.start() args = {"db1": db1, "db2": db2, "chats": chats1, "type": "diff_left"} worker.work(args) while True: result = postbacks.get() if "error" in result: print("Error scanning %s and %s:\n\n%s" % (db1, db2, result["error"])) worker = None # Signal for global break break # break while True if "done" in result: break # break while True if "chats" in result and result["chats"]: counts[db1]["chats"] += 1 msgs = len(result["chats"][0]["diff"]["messages"]) msgs_text = util.plural("new message", msgs) contacts_text = util.plural("new participant", result["chats"][0]["diff"]["participants"]) text = ", ".join(filter(None, [msgs_text, contacts_text])) print("%s, %s." % (result["chats"][0]["chat"]["title_long"], text)) counts[db1]["msgs"] += msgs bar.update(bar.value + result["chats"][0]["chat"]["message_count"]) if result["output"]: log(result["output"]) bar.stop() bar.afterword = " Scanned %s and %s." % (db1, db2) bar.update(bar_total) print
def flush(self): if self._trs_to_flush is not None: log.debug("A flush is already in progress, not doing anything") return to_flush = [] # Do we have something to do ? now = datetime.utcnow() for tr in self._transactions: if tr.time_to_flush(now): to_flush.append(tr) count = len(to_flush) should_log = self._flush_count + 1 <= FLUSH_LOGGING_INITIAL or (self._flush_count + 1) % FLUSH_LOGGING_PERIOD == 0 if count > 0: if should_log: log.info("Flushing %s transaction%s during flush #%s" % (count,plural(count), str(self._flush_count + 1))) else: log.debug("Flushing %s transaction%s during flush #%s" % (count,plural(count), str(self._flush_count + 1))) self._endpoints_errors = {} self._finished_flushes = 0 # We sort LIFO-style, taking into account errors self._trs_to_flush = sorted(to_flush, key=lambda tr: (- tr._error_count, tr._id)) self._flush_time = datetime.utcnow() self.flush_next() else: if should_log: log.info("No transaction to flush during flush #%s" % str(self._flush_count + 1)) else: log.debug("No transaction to flush during flush #%s" % str(self._flush_count + 1)) if self._flush_count + 1 == FLUSH_LOGGING_INITIAL: log.info("First flushes done, next flushes will be logged every %s flushes." % FLUSH_LOGGING_PERIOD) self._flush_count += 1 ForwarderStatus( queue_length=self._total_count, queue_size=self._total_size, flush_count=self._flush_count, transactions_received=self._transactions_received, transactions_flushed=self._transactions_flushed, transactions_rejected=self._transactions_rejected).persist()
def chooseTracks(self, number, exclude, completion, traceCallback=None, tags=None): self._logger.debug("Selecting " + str(number) + " track" + util.plural(number) + ".") mycompletion = (lambda thisCallback, trackIDs, completion=completion: self._chooseTracksCompletion(trackIDs, completion, thisCallback)) self._chooseTrackIDs( number, exclude, mycompletion, traceCallback, tags)
def flush(self): try: self.flush_count += 1 self.log_count += 1 packets_per_second = self.metrics_aggregator.packets_per_second( self.interval) packet_count = self.metrics_aggregator.total_count metrics = self.metrics_aggregator.flush() count = len(metrics) if self.flush_count % FLUSH_LOGGING_PERIOD == 0: self.log_count = 0 if count: self.submit(metrics) events = self.metrics_aggregator.flush_events() event_count = len(events) if event_count: self.submit_events(events) should_log = self.flush_count <= FLUSH_LOGGING_INITIAL or self.log_count <= FLUSH_LOGGING_COUNT log_func = log.info if not should_log: log_func = log.debug log_func("Flush #%s: flushed %s metric%s and %s event%s" % (self.flush_count, count, plural(count), event_count, plural(event_count))) if self.flush_count == FLUSH_LOGGING_INITIAL: log.info( "First flushes done, %s flushes will be logged every %s flushes." % (FLUSH_LOGGING_COUNT, FLUSH_LOGGING_PERIOD)) # Persist a status message. packet_count = self.metrics_aggregator.total_count DogstatsdStatus( flush_count=self.flush_count, packet_count=packet_count, packets_per_second=packets_per_second, metric_count=count, event_count=event_count, ).persist() except Exception, e: log.exception("Error flushing metrics")
def flush(self): if self._trs_to_flush is not None: log.debug("A flush is already in progress, not doing anything") return to_flush = [] now = datetime.utcnow() for tr in self._transactions: if tr.time_to_flush(now): to_flush.append(tr) count = len(to_flush) should_log = self._flush_count + 1 <= FLUSH_LOGGING_INITIAL or ( self._flush_count + 1) % FLUSH_LOGGING_PERIOD == 0 if count > 0: if should_log: log.info( "Flushing %s transaction%s during flush #%s" % (count, plural(count), str(self._flush_count + 1))) else: log.debug( "Flushing %s transaction%s during flush #%s" % (count, plural(count), str(self._flush_count + 1))) self._trs_to_flush = to_flush self.flush_next() else: if should_log: log.info("No transaction to flush during flush #%s" % str(self._flush_count + 1)) else: log.debug("No transaction to flush during flush #%s" % str(self._flush_count + 1)) if self._flush_count + 1 == FLUSH_LOGGING_INITIAL: log.info("First flushes done, next flushes will be logged every %s flushes." % FLUSH_LOGGING_PERIOD) self._flush_count += 1 ForwarderStatus( queue_length=self._total_count, queue_size=self._total_size, flush_count=self._flush_count, transactions_received=self._transactions_received, transactions_flushed=self._transactions_flushed).persist()
def download_events(self): event_list = self.camera.list_events() if event_list and len(event_list) > 0: self.camera.prepare_events(event_list) total = len(event_list) logger.info("{} event{} on device".format(total, plural(total), len(event_list))) self.prepare_recordings(event_list) self.add_paths(event_list, "event_filename") self.camera.download_files(event_list, "filename", "event_filename")
def to_templated_secular_display(self,fp,group,type= 'lecture'): gt = GroupTemplate(None) gt.add_patterns(fp) gt.calculate_reduction(True) out = [] for (_,_,(pattern,mult)) in gt.get_patterns_raw(False): if pattern is None: # just multiplier, should be just "x lectures" row = "%d %s, %s" % (mult,util.plural(type),gt.template) # XXX proper plurals elif mult is None: # traditional pattern, expand codes sensibly row = "%s %s" % (util.plural(type),self.atoms_to_secular(fp.patterns(),True)) else: # start date and multiplier pp = copy.deepcopy(gt.template) pp.setAllYear() row = "%s %s, starting on %s, %s" % (mult,util.plural(type),self.atoms_to_secular([pattern],False),pp) out.append(row) prefix = '' if group is not None: prefix = "%s term, " % term_names[group.term] return prefix+", ".join(out)
def _relation_repr(cls, rel): target = rel.argument if target and inspect.isfunction(target): target = target() if isinstance(target, Mapper): target = target.class_ target = target.__name__ primaryjoin = '' lookup = mtl() foo = rel.key if rel.primaryjoin is not None and hasattr( rel.primaryjoin, 'right'): right_lookup = lookup.get( rel.primaryjoin.right.table.name, '%s.c' % rel.primaryjoin.right.table.name) left_lookup = lookup.get( rel.primaryjoin.left.table.name, '%s.c' % rel.primaryjoin.left.table.name) primaryjoin = ", primaryjoin='%s.%s==%s.%s'" % ( left_lookup, rel.primaryjoin.left.name, right_lookup, rel.primaryjoin.right.name) elif hasattr(rel, '_as_string'): primaryjoin = ', primaryjoin="%s"' % rel._as_string secondary = '' secondaryjoin = '' if rel.secondary is not None: """ **HACK**: If there is a secondary relationship like between Venue, Event, and Event_Type, then I'm only going show a primary relationship. "Events = relationship('Event', primaryjoin='Venue.id==Event.venue_id')" and not "Event_Types = relation('EventType', primaryjoin='Venue.id==Event.venue_id', secondary=Event, secondaryjoin='Event.event_type_id==EventType.id')" """ if rel.secondary.name in self.table_model_dict: target = self.table_model_dict[rel.secondary.name] else: target = self.find_new_name( singular(name2label(rel.secondary.name)), []) # **HACK** secondary = '' secondaryjoin = '' foo = plural(rel.secondary.name) backref = '' # if rel.backref: # backref=", backref='%s'"%rel.backref.key return "%s = relationship('%s'%s%s%s%s)" % ( foo, target, primaryjoin, secondary, secondaryjoin, backref)
def list_events(self): logger.info("Querying camera for list of events..") (response, duration, ts) = self.request(self.get_api_url("APP_EventListReq"), method='POST', data="{}") if response: data = self.json(response) if data: logger.info("{} event{} found in {:.2f}s".format( data['num'], plural(data['num']), duration.total_seconds())) return data.get('event') return None
def download_files(self, list, key, local_key): count = 1 skipped = 0 for file in list: filename = file[key] localfile = file[local_key] if os.path.isfile(localfile) and os.stat(localfile).st_size > 0: skipped += 1 logger.info("{} file{} already downloaded. {} file{} remaining".format( skipped, plural(skipped), len(list) - skipped, plural(len(list) - skipped))) for file in list: filename = file[key] localfile = file[local_key] directory = os.path.dirname(localfile) if not os.path.exists(directory): logger.debug("Making dir {}".format(directory)) os.makedirs(directory) if not os.path.isfile(localfile) or os.stat( localfile).st_size == 0: (response, duration, ts) = self.request(self.get_download_url(filename)) if response: logger.debug("{}/{}: Downloaded {} in {:.2f}s".format( count, len(list) - skipped, filename, duration.total_seconds())) with open(localfile, 'wb') as f: f.write(response.content) else: logger.error("Failed to download: {}".format(filename)) count += 1 return list
def run_export(filenames, format): """Exports the specified databases in specified format.""" dbs = [skypedata.SkypeDatabase(f) for f in filenames] is_xlsx_single = ("xlsx_single" == format) for db in dbs: formatargs = collections.defaultdict(str) formatargs["skypename"] = os.path.basename(db.filename) formatargs.update(db.account or {}) basename = util.safe_filename(conf.ExportDbTemplate % formatargs) dbstr = "from %s " % db if len(dbs) != 1 else "" if is_xlsx_single: export_dir = os.getcwd() filename = util.unique_path("%s.xlsx" % basename) else: export_dir = util.unique_path(os.path.join(os.getcwd(), basename)) filename = format target = filename if is_xlsx_single else export_dir try: print("Exporting as %s %sto %s." % (format[:4].upper(), dbstr, target)) chats = sorted(db.get_conversations(), key=lambda x: x["title"].lower()) db.get_conversations_stats(chats) bar_total = sum(c["message_count"] for c in chats) bartext = " Exporting %.*s.." % (30, db.filename) # Enforce width bar = ProgressBar(max=bar_total, afterword=bartext) bar.start() result = export.export_chats(chats, export_dir, filename, db, progress=bar.update) files, count = result bar.stop() if count: bar.afterword = " Exported %s to %s. " % (db, target) bar.update(bar_total) print() log("Exported %s %sto %s as %s.", util.plural("chat", count), dbstr, target, format) else: print("\nNo messages to export%s." % ("" if len(dbs) == 1 else " from %s" % db)) os.unlink(filename) if is_xlsx_single else os.rmdir(export_dir) except Exception as e: print("Error exporting chats: %s\n\n%s" % (e, traceback.format_exc()))
def tr_error(self, tr): self._running_flushes -= 1 self._finished_flushes += 1 tr.inc_error_count() if tr.get_error_count() > self._MAX_ENDPOINT_ERRORS: log.warn("Transaction %d failed too many (%d) times, removing", tr.get_id(), tr.get_error_count()) self._remove(tr) self._transactions_flushed += 1 self.print_queue_stats() self._transactions_rejected += 1 ForwarderStatus( queue_length=self._total_count, queue_size=self._total_size, flush_count=self._flush_count, transactions_received=self._transactions_received, transactions_flushed=self._transactions_flushed, transactions_rejected=self._transactions_rejected).persist() else: tr.compute_next_flush(self._MAX_WAIT_FOR_REPLAY) log.warn( "Transaction %d in error (%s error%s), it will be replayed after %s", tr.get_id(), tr.get_error_count(), plural(tr.get_error_count()), tr.get_next_flush()) self._endpoints_errors[tr._endpoint] = self._endpoints_errors.get( tr._endpoint, 0) + 1 # Endpoint failed too many times, it's probably an enpoint issue # Let's avoid blocking on it if self._endpoints_errors[ tr._endpoint] == self._MAX_ENDPOINT_ERRORS: new_trs_to_flush = [] for transaction in self._trs_to_flush: if transaction._endpoint != tr._endpoint: new_trs_to_flush.append(transaction) else: transaction.compute_next_flush( self._MAX_WAIT_FOR_REPLAY) log.debug( 'Endpoint %s seems down, removed %s transaction from current flush', tr._endpoint, len(self._trs_to_flush) - len(new_trs_to_flush)) self._trs_to_flush = new_trs_to_flush
def run(self, path, print_file=True): from functools import reduce import prompt if print_file and self.print_file: _print_file(path) util.info("description: " + self.description) if type(self.deduction) is int: choices = ["do not take this deduction", "take this deduction (-{} points)".format( self.deduction)] got = prompt.prompt(choices, '1') if got == [1]: return {'deduction': self.deduction, 'description': self.description} else: util.info("taking no points") return None elif type(self.deduction) is list: choices = ["{} (-{} {})".format(y, x, util.plural("point", x)) for x, y in self.deduction] got = prompt.prompt(choices, self.deduction_mode) total_deducted = sum(map(lambda x: self.deduction[x][0], got)) if total_deducted > 0: deductions = [] for s in got: if self.deduction[s][0] > 0: d = {'deduction': self.deduction[s][0], 'description': self.deduction[s][1]} deductions.append(d) return {'description': self.description + ':', 'subresults': deductions} else: util.info("taking no points") return None
def __should_fail(self, result): """Called before an eval test is about to be failed, but the criteria specifies that a human should confirm the failure before taking any points. If this method returns True, the points are ultimately taken. """ import sys from grader import write_results from prompt import prompt util.warning("about to fail a test") write_results(sys.stdout, [result]) points = result['deduction'] s = util.plural("point", points) fail_msg = "fail this test (-{} {})".format(points, s) choices = [fail_msg, "do not fail this test"] return prompt(choices, '1') == [0]
def run(self, path): import os from functools import reduce import prompt _print_file(path) sprint("reviewing '{}' (in directory '{}')".format( path, os.getcwd())) sprint("description: " + self.description) if type(self.deduction) is int: choices = ["do not take this deduction", "take this deduction (-{} points)".format( self.deduction)] got = prompt.prompt(choices, '1') if got == [1]: return {'deduction': self.deduction, 'description': self.description} else: sprint("taking no points") return None elif type(self.deduction) is list: choices = ["{} (-{} {})".format(y, x, plural("point", x)) for x, y in self.deduction] got = prompt.prompt(choices, self.deduction_mode) if sum(map(lambda x: self.deduction[x][0], got)) > 0: deductions = [] for s in got: if self.deduction[s][0] > 0: d = {'deduction': self.deduction[s][0], 'description': self.deduction[s][1]} deductions.append(d) return deductions else: sprint("taking no points") return None
def _relation_repr(cls, rel): target = rel.argument if target and inspect.isfunction(target): target = target() if isinstance(target, Mapper): target = target.class_ target = target.__name__ primaryjoin = '' lookup = mtl() foo = rel.key if rel.primaryjoin is not None and hasattr(rel.primaryjoin, 'right'): right_lookup = lookup.get(rel.primaryjoin.right.table.name, '%s.c' % rel.primaryjoin.right.table.name) left_lookup = lookup.get(rel.primaryjoin.left.table.name, '%s.c' % rel.primaryjoin.left.table.name) primaryjoin = ", primaryjoin='%s.%s==%s.%s'" % (left_lookup, rel.primaryjoin.left.name, right_lookup, rel.primaryjoin.right.name) elif hasattr(rel, '_as_string'): primaryjoin = ', primaryjoin="%s"' % rel._as_string secondary = '' secondaryjoin = '' if rel.secondary is not None: """ **HACK**: If there is a secondary relationship like between Venue, Event, and Event_Type, then I'm only going show a primary relationship. "Events = relationship('Event', primaryjoin='Venue.id==Event.venue_id')" and not "Event_Types = relation('EventType', primaryjoin='Venue.id==Event.venue_id', secondary=Event, secondaryjoin='Event.event_type_id==EventType.id')" """ if rel.secondary.name in self.table_model_dict: target = self.table_model_dict[rel.secondary.name] else: target = self.find_new_name(singular(name2label(rel.secondary.name)), []) # **HACK** secondary = '' secondaryjoin = '' foo = plural(rel.secondary.name) backref = '' # if rel.backref: # backref=", backref='%s'"%rel.backref.key return "%s = relationship('%s'%s%s%s%s)" % (foo, target, primaryjoin, secondary, secondaryjoin, backref)
def identify_recordings(self): all_recordings = self.camera.list_recordings() if all_recordings and len(all_recordings) > 0: self.camera.prepare_recordings(all_recordings) filtered_recordings = self.filter_processed(all_recordings) total = len(filtered_recordings) self.prepare_recordings(filtered_recordings) logger.info( "{} file{} on device of which {} are to be processed".format( total, plural(total), len(filtered_recordings))) if len(filtered_recordings) > 0: logger.info("The oldest recording on the device is: {}".format( filtered_recordings[0]['start_timestamp'])) requested_sequences = [] if self.config.get('process_manual_requests'): logger.info("Processing manual requests..") manual_requests = self.find_manual_requests() requested_sequences.extend(manual_requests) if self.config.get('process_motion_detection'): logger.info("Processing motion detection..") self.add_paths(filtered_recordings, "thumbnail_filename") download_list = self.camera.download_files( filtered_recordings, "thumbnail", "thumbnail_filename") self.motion.calculate_differences(download_list, "thumbnail_filename") requested_sequences.extend( self.motion.identify_requests(download_list)) self.match_recordings(requested_sequences, all_recordings) self.remove_empty_requests(requested_sequences) return (filtered_recordings, requested_sequences)
def run(self): self._is_running = True # For identifying "chat:xxx" and "from:xxx" keywords template_chat = step.Template(templates.SEARCH_ROW_CHAT_HTML) template_contact = step.Template(templates.SEARCH_ROW_CONTACT_HTML) template_message = step.Template(templates.SEARCH_ROW_MESSAGE_HTML) template_row = step.Template(templates.SEARCH_ROW_TABLE_HTML) template_table = step.Template(templates.SEARCH_ROW_TABLE_HEADER_HTML) query_parser = searchparser.SearchQueryParser() wrap_b = lambda x: "<b>%s</b>" % x.group(0) result = None while self._is_running: try: search = self._queue.get() if not search: continue # continue while self._is_running self._stop_work = False self._drop_results = False parser = skypedata.MessageParser(search["db"]) # {"html": html with results, "map": link data map} # map data: {"contact:666": {"contact": {contact data}}, } result_type, result_count, count = None, 0, 0 result = {"html": "", "map": {}, "search": search, "count": 0} sql, params, match_words = query_parser.Parse(search["text"]) # Turn wildcard characters * into regex-compatible .* match_words_re = [".*".join(map(re.escape, w.split("*"))) for w in match_words] patt = "(%s)" % "|".join(match_words_re) # For replacing matching words with <b>words</b> pattern_replace = re.compile(patt, re.IGNORECASE) # Find chats with a matching title or matching participants chats = search["db"].get_conversations() chats.sort(key=lambda x: x["title"]) chat_map = {} # {chat id: {chat data}} for chat in chats: chat_map[chat["id"]] = chat if "conversations" == search["table"] and match_words: title_matches = False matching_authors = [] if self.match_all(chat["title"], match_words): title_matches = True for participant in chat["participants"]: contact = participant["contact"] if contact: for n in filter(None, [contact["fullname"], contact["displayname"], contact["identity"]]): if self.match_all(n, match_words) \ and contact not in matching_authors: matching_authors.append(contact) if title_matches or matching_authors: count += 1 result_count += 1 result["html"] += template_chat.expand(locals()) key = "chat:%s" % chat["id"] result["map"][key] = {"chat": chat["id"]} if not count % conf.SearchResultsChunk \ and not self._drop_results: result["count"] = result_count self.postback(result) result = {"html": "", "map": {}, "search": search, "count": 0} if self._stop_work: break # break for chat in chats if result["html"] and not self._drop_results: result["count"] = result_count self.postback(result) result = {"html": "", "map": {}, "search": search, "count": 0} # Find contacts with a matching name if not self._stop_work and "contacts" == search["table"] \ and match_words: count = 0 contacts = search["db"].get_contacts() # Possibly more: country (ISO code, need map), birthday # (base has YYYYMMDD in integer field). match_fields = [ "displayname", "skypename", "province", "city", "pstnnumber", "phone_home", "phone_office", "phone_mobile", "homepage", "emails", "about", "mood_text", ] for contact in contacts: match = False fields_filled = {} for field in match_fields: if contact[field]: val = contact[field] if self.match_all(val, match_words): match = True val = pattern_replace.sub(wrap_b, val) fields_filled[field] = val if match: count += 1 result_count += 1 result["html"] += template_contact.expand(locals()) if not (self._drop_results or count % conf.SearchResultsChunk): result["count"] = result_count self.postback(result) result = {"html": "", "map": {}, "search": search, "count": 0} if self._stop_work: break # break for contact in contacts if result["html"] and not self._drop_results: result["count"] = result_count self.postback(result) result = {"html": "", "map": {}, "search": search, "count": 0} # Find messages with a matching body if not self._stop_work and "messages" == search["table"]: count, result_type = 0, "messages" chat_messages = {} # {chat id: [message, ]} chat_order = [] # [chat id, ] messages = search["db"].get_messages( additional_sql=sql, additional_params=params, ascending=False, use_cache=False) for m in messages: chat = chat_map.get(m["convo_id"]) chat_title = chat["title_long"] body = parser.parse(m, pattern_replace if match_words else None, html={"w": search["window"].Size.width * 5/9}) count += 1 result_count += 1 result["html"] += template_message.expand(locals()) key = "message:%s" % m["id"] result["map"][key] = {"chat": chat["id"], "message": m["id"]} if not count % conf.SearchResultsChunk \ and not self._drop_results: result["count"] = result_count self.postback(result) result = {"html": "", "map": {}, "search": search, "count": 0} if self._stop_work or count >= conf.SearchMessagesMax: break # break for m in messages infotext = search["table"] if not self._stop_work and "all tables" == search["table"]: infotext, result_type = "", "table row" # Search over all fields of all tables. for table in search["db"].tables_list: sql, params, words = \ query_parser.Parse(search["text"], table) if not sql: continue # continue for table in search["db"].. infotext += (", " if infotext else "") + table["name"] rows = search["db"].execute(sql, params) row = rows.fetchone() if not row: continue # continue for table in search["db"].. result["html"] = template_table.expand(locals()) count = 0 while row: count += 1 result_count += 1 result["html"] += template_row.expand(locals()) key = "table:%s:%s" % (table["name"], count) result["map"][key] = {"table": table["name"], "row": row} if not count % conf.SearchResultsChunk \ and not self._drop_results: result["count"] = result_count self.postback(result) result = {"html": "", "map": {}, "search": search, "count": 0} if self._stop_work \ or result_count >= conf.SearchTableRowsMax: break # break while row row = rows.fetchone() if not self._drop_results: result["html"] += "</table>" result["count"] = result_count self.postback(result) result = {"html": "", "map": {}, "search": search, "count": 0} infotext += " (%s)" % util.plural("result", count) if self._stop_work \ or result_count >= conf.SearchTableRowsMax: break # break for table in search["db"].. single_table = ("," not in infotext) infotext = "table%s: %s" % \ ("" if single_table else "s", infotext) if not single_table: infotext += "; %s in total" % \ util.plural("result", result_count) final_text = "No matches found." if self._drop_results: result["html"] = "" if result_count: final_text = "Finished searching %s." % infotext if self._stop_work: final_text += " Stopped by user." elif "messages" == result_type \ and count >= conf.SearchMessagesMax: final_text += " Stopped at %s limit %s." % \ (result_type, conf.SearchMessagesMax) elif "table row" == result_type \ and count >= conf.SearchTableRowsMax: final_text += " Stopped at %s limit %s." % \ (result_type, conf.SearchTableRowsMax) result["html"] += "</table><br /><br />%s</font>" % final_text result["done"] = True result["count"] = result_count self.postback(result) except Exception, e: if not result: result = {} result["done"], result["error"] = True, traceback.format_exc() result["error_short"] = "%s: %s" % (type(e).__name__, e.message) self.postback(result)
def export_chat(chat, messages, filename, db): """ Exports the chat messages to file. @param chat chat data dict, as returned from SkypeDatabase @param messages list of message data dicts @param filename full path and filename of resulting file, file extension .html|.txt determines file format @param db SkypeDatabase instance """ result = False f = None try: is_html = filename.lower().endswith(".html") is_csv = filename.lower().endswith(".csv") is_txt = filename.lower().endswith(".txt") if is_txt: f = codecs.open(filename, "w", "utf-8") else: f = open(filename, "w") # @todo add stats? parser = skypedata.MessageParser(db) chat_title = chat["title_long_lc"] main_data = { "title": chat_title, "date1": messages[0]["datetime"].strftime("%d.%m.%Y") \ if len(messages) else "", "date2": messages[-1]["datetime"].strftime("%d.%m.%Y") \ if len(messages) else "", "messages_total": util.plural("message", chat["message_count"]), "chat_created": chat["created_datetime"].strftime("%d.%m.%Y") \ if chat["created_datetime"] else "", "app": conf.Title, "now": datetime.datetime.now() \ .strftime("%d.%m.%Y %H:%M"), "db": db.filename, "count": str(len(messages)), "chat_info": "Showing %s" \ % util.plural("message", len(messages)), } if is_html: # Write HTML header and table header header_data = dict([(k, escape(v)) for k, v in main_data.items()]) if header_data["date1"] and header_data["date2"]: header_data["chat_info"] += \ " from <b>%(date1)s</b> to <b>%(date2)s</b>" % header_data header_data["chat_info"] += ".<br />" if header_data["chat_created"]: header_data["chat_info"] += \ "Chat created on <b>%(chat_created)s</b>" % header_data if header_data["messages_total"]: header_data["chat_info"] += \ ("," if header_data["chat_created"] else "Chat has") +\ " <b>%(messages_total)s</b> in total" % header_data if header_data["chat_created"] or header_data["messages_total"]: header_data["chat_info"] += ".<br />" header_data.update({ "title": "History of Skype " + header_data["title"], "css_avatars": "", "css_chat_picture": "", "header_left": "", "header_right": "", "header_link": "" }) if chat["meta_picture"]: raw = chat["meta_picture"].encode("latin1") if raw.startswith("\0"): # For some reason, Skype image blobs # can start with a null byte. raw = raw[1:] if raw.startswith("\0"): raw = "\xFF" + raw[1:] header_data["header_left"] = htmltag("span", { "class": "chat_picture", "title": chat["title"] }) img = wx.ImageFromStream(cStringIO.StringIO(raw)) header_data["css_chat_picture"] = cssrule("span.chat_picture",{ "background": "url(data:image/jpg;base64,%s) " \ "center center no-repeat" % base64.b64encode(raw), "margin": "0px 10px 0px 10px", "display": "block", "width": "%spx" % img.Width, "height": "%spx" % img.Height, }) if chat["participants"]: for p in chat["participants"]: avatar_class = "avatar__default" if "avatar_image" in p["contact"] \ and p["contact"]["avatar_image"]: raw = p["contact"]["avatar_image"].encode("latin1") if raw.startswith("\0"): # For some reason, Skype avatar image blobs # can start with a null byte. raw = raw[1:] if raw.startswith("\0"): #raw = raw[1:] raw = "\xFF" + raw[1:] # Replace dots and commas, as they are # not valid CSS identifier characters avatar_class = "avatar_" \ + p["identity"].replace(".", "___") \ .replace(",", "---") header_data["css_avatars"] += cssrule( "span.%s" % avatar_class, { "background": "url(data:image/jpg;base64,%s) center " \ "center no-repeat" % base64.b64encode(raw) }) if skypedata.CHATS_TYPE_SINGLE == chat["type"]: title = p["contact"]["name"] name = escape(p["contact"]["name"]) if p["contact"]["name"] != p["identity"]: title += " (%s)" % p["identity"] name += "<br />(%s)" % escape(p["identity"]) side = "right" if (p["identity"] == db.id) else "left" header_data["header_%s" % side] = "<div>%s" \ "<br />%s</div>" % ( htmltag("span", {"title": title, "class": "avatar header %s" % avatar_class} ), name ) if skypedata.CHATS_TYPE_SINGLE != chat["type"]: header_data["header_link"] = PARTICIPANTS_LINK for k, v in header_data.items(): header_data[k] = str(v) f.write(CHAT_HTML_HEADER % header_data) if skypedata.CHATS_TYPE_SINGLE != chat["type"]: for p in sorted(chat["participants"], key=lambda a: a["contact"]["name"].lower()): img_attr = {"class": "avatar avatar__default"} img_attr["title"] = p["contact"]["name"] if p["contact"]["name"] != p["identity"]: img_attr["title"] += " (%s)" % p["identity"] if "avatar_image" in p["contact"] \ and p["contact"]["avatar_image"]: # Replace dots and commas, as they are not valid # CSS identifier characters img_attr["class"] = "avatar avatar_%s" \ % p["identity"].replace(".", "___") \ .replace(",", "---") name = escape(p["contact"]["name"]) if p["contact"]["name"] != p["identity"]: name += "<br />(%s)" % escape(p["identity"]) f.write(" <span>%(img)s%(name)s</span>\n" % { "name": name, "img": htmltag("span", img_attr), }) f.write(" </div>\r\n </td>\r\n</tr>\r\n</table>\r\n" \ "</td></tr><tr><td><table class='content_table'>\r\n") elif is_txt: main_data["hr"] = "-" * 79 f.write("History of Skype %(title)s.\r\n" \ "Showing %(count)s messages" % main_data) if main_data["date1"] and main_data["date2"]: f.write(" from %(date1)s to %(date2)s" % main_data) f.write(".\r\n") if main_data["chat_created"]: f.write("Chat created on %(chat_created)s" % main_data) else: f.write("Chat has") if main_data["messages_total"]: f.write(("," if main_data["chat_created"] else "") + " %(messages_total)s in total" % main_data) f.write(".\r\n") f.write( "Source: %(db)s.\r\n" \ "Exported with %(app)s on %(now)s." \ "\r\n%(hr)s\r\n" % main_data ) elif is_csv: # Initialize CSV writer and write header row dialect = csv.excel # Default is "," which is actually not Excel dialect.delimiter = ";" # Default is "\r\n", which causes another "\r" to be written dialect.lineterminator = "\r" csv_writer = csv.writer(f, dialect) csv_writer.writerow(["Time", "Author", "Message"]) colourmap = collections.defaultdict(lambda: "remote") colourmap[db.id] = "local" previous_day = datetime.date.fromtimestamp(0) for m in messages: if m["datetime"].date() != previous_day: # Day has changed: insert a date header previous_day = m["datetime"].date() weekday = previous_day.strftime("%A").capitalize() date = previous_day.strftime("%d. %B %Y") if locale.getpreferredencoding(): weekday = weekday.decode(locale.getpreferredencoding()) date = date.decode(locale.getpreferredencoding()) if is_html: f.write( "<tr>\r\n\t<td class='t1'></td>\r\n\t" \ "<td class='day t2'></td>\r\n\t" \ "<td class='day t3'></td>\r\n\t" \ "<td class='day' colspan='2'><span class='weekday'>" \ "%(weekday)s</span>, %(date)s</td>\r\n</tr>\r\n" % { "weekday": escape(weekday), "date": escape(date) }) elif is_txt: f.write("\r\n%(weekday)s, %(date)s\r\n%(hr)s\r\n\r\n" % { "weekday": weekday, "date": date, "hr": "-" * 40 }) if is_html: body = parser.parse(m, html={"w": -1, "export": True}) f.write("<tr>\r\n\t" \ "<td class='author %(authorclass)s' colspan='2'>" \ "%(name)s</td>\r\n\t" \ "<td class='t3'></td>\r\n\t<td>%(text)s</td>\r\n\t" \ "<td class='timestamp'>%(time)s</td>\r\n</tr>\r\n" % { "authorclass": colourmap[m["author"]], "time": m["datetime"].strftime("%H:%S"), "name": escape(m["from_dispname"]), "text": body }) else: parsed_text = parser.parse(m, text=True) try: parsed_text = parsed_text.decode("utf-8") except Exception, e: pass if is_txt: f.write( "%(datetime)s %(name)s:\r\n%(text)s\r\n\r\n" % { "datetime": m["datetime"].strftime("%H:%S"), "name": m["from_dispname"], "text": parsed_text }) elif is_csv: try: parsed_text = parser.parse(m, text=True) parsed_text = parsed_text.decode("utf-8") except Exception, e: pass values = [ m["datetime"].strftime("%Y-%m-%d %H:%M:%S"), m["from_dispname"].encode("utf-8"), parsed_text.encode("utf-8") ] csv_writer.writerow(values)
def flush(self): try: self.flush_count += 1 self.log_count += 1 packets_per_second = self.metrics_aggregator.packets_per_second(self.interval) packet_count = self.metrics_aggregator.total_count metrics = self.metrics_aggregator.flush() count = len(metrics) if self.flush_count % FLUSH_LOGGING_PERIOD == 0: self.log_count = 0 if count: self.submit(metrics) events = self.metrics_aggregator.flush_events() event_count = len(events) if event_count: self.submit_events(events) service_checks = self.metrics_aggregator.flush_service_checks() check_count = len(service_checks) if check_count: self.submit_service_checks(service_checks) should_log = self.flush_count <= FLUSH_LOGGING_INITIAL or self.log_count <= FLUSH_LOGGING_COUNT log_func = log.info if not should_log: log_func = log.debug log_func("Flush #%s: flushed %s metric%s, %s event%s, and %s service check run%s" % (self.flush_count, count, plural(count), event_count, plural(event_count), check_count, plural(check_count))) if self.flush_count == FLUSH_LOGGING_INITIAL: log.info("First flushes done, %s flushes will be logged every %s flushes." % (FLUSH_LOGGING_COUNT, FLUSH_LOGGING_PERIOD)) # Persist a status message. packet_count = self.metrics_aggregator.total_count DogstatsdStatus( flush_count=self.flush_count, packet_count=packet_count, packets_per_second=packets_per_second, metric_count=count, event_count=event_count, ).persist() except Exception: if self.finished.isSet(): log.debug("Couldn't flush metrics, but that's expected as we're stopping") else: log.exception("Error flushing metrics")
def work_merge_left(self, params): """ Worker branch that merges differences given in params, posting progress back to application. """ error, e = None, None db1, db2 = params["db1"], params["db2"] chats = params["chats"] chats1_count = len(db1.get_conversations()) count_messages = 0 count_participants = 0 try: for index, chat_data in enumerate(chats): if self._stop_work: break # break for i, chat_data in enumerate(chats) chat1 = chat_data["chat"]["c1"] chat2 = chat_data["chat"]["c2"] messages = chat_data["diff"]["messages"] participants = chat_data["diff"]["participants"] html = "Merged %s" % chat1["title_long_lc"] if not chat2: html += " - new chat" if messages: newstr = "" if not chat2 else "new " html += ", %s" % util.plural("%smessage" % newstr, messages) else: html += ", no messages" html += "." if not chat2: chat2 = chat1.copy() chat_data["chat"]["c2"] = chat2 chat2["id"] = db2.insert_chat(chat2, db1) if participants: db2.insert_participants(chat2, participants, db1) count_participants += len(participants) if messages: db2.insert_messages(chat2, messages, db1, chat1, self.yield_ui, self.REFRESH_COUNT) count_messages += len(messages) if not self._drop_results: result = { "type": "merge_left", "index": index, "output": html, "count": chats1_count, "params": params, "chats": [chat_data["chat"]] } if index < len(chats) - 1: result["status"] = ( "Merging %s." % chats[index + 1]["chat"]["title_long_lc"]) self.postback(result) except Exception as e: error = traceback.format_exc() finally: if chats: html = "Merged %s" % util.plural("new message", count_messages) if count_participants: html += " and %s" % util.plural("new participant", count_participants) html += " \n\nto %s." % db2 if not self._drop_results: result = { "type": "merge_left", "done": True, "output": html, "params": params } if error: result["error"] = error if e: result["error_short"] = "%s: %s" % (type(e).__name__, e.message) self.postback(result)
def flush(self): try: self.flush_count += 1 self.log_count += 1 packets_per_second = self.metrics_aggregator.packets_per_second(self.interval) packet_count = self.metrics_aggregator.total_count metrics = self.metrics_aggregator.flush() count = len(metrics) if self.flush_count % FLUSH_LOGGING_PERIOD == 0: self.log_count = 0 if count: self.submit(metrics) events = self.metrics_aggregator.flush_events() event_count = len(events) if event_count: self.submit_events(events) should_log = self.flush_count <= FLUSH_LOGGING_INITIAL or self.log_count <= FLUSH_LOGGING_COUNT log_func = log.info if not should_log: log_func = log.debug log_func("Flush #%s: flushed %s metric%s and %s event%s" % (self.flush_count, count, plural(count), event_count, plural(event_count))) if self.flush_count == FLUSH_LOGGING_INITIAL: log.info("First flushes done, %s flushes will be logged every %s flushes." % (FLUSH_LOGGING_COUNT, FLUSH_LOGGING_PERIOD)) # Persist a status message. packet_count = self.metrics_aggregator.total_count DogstatsdStatus( flush_count=self.flush_count, packet_count=packet_count, packets_per_second=packets_per_second, metric_count=count, event_count=event_count, ).persist() except Exception, e: log.exception("Error flushing metrics")
def check_status_lines(cs): check_lines = [ ' ' + cs.name + ' ({})'.format(cs.check_version), ' ' + '-' * (len(cs.name) + 3 + len(cs.check_version)) ] if cs.init_failed_error: check_lines.append(" - initialize check class [%s]: %s" % (style(STATUS_ERROR, 'red'), repr(cs.init_failed_error))) if cs.init_failed_traceback: check_lines.extend(' ' + line for line in cs.init_failed_traceback.split('\n')) else: for s in cs.instance_statuses: c = 'green' if s.has_warnings(): c = 'yellow' if s.has_error(): c = 'red' line = " - instance #%s [%s]" % ( s.instance_id, style(s.status, c)) if s.has_error(): line += u": %s" % s.error if s.metric_count is not None: line += " collected %s metrics" % s.metric_count if s.instance_check_stats is not None: line += " Last run duration: %s" % s.instance_check_stats.get('run_time') check_lines.append(line) if s.has_warnings(): for warning in s.warnings: warn = warning.split('\n') if not len(warn): continue check_lines.append(u" %s: %s" % (style("Warning", 'yellow'), warn[0])) check_lines.extend(u" %s" % l for l in warn[1:]) if s.traceback is not None: check_lines.extend(' ' + line for line in s.traceback.split('\n')) check_lines += [ " - Collected %s metric%s, %s event%s & %s service check%s" % ( cs.metric_count, plural(cs.metric_count), cs.event_count, plural(cs.event_count), cs.service_check_count, plural(cs.service_check_count)), ] if cs.check_stats is not None: check_lines += [ " - Stats: %s" % pretty_statistics(cs.check_stats) ] if cs.library_versions is not None: check_lines += [ " - Dependencies:"] for library, version in cs.library_versions.iteritems(): check_lines += [" - %s: %s" % (library, version)] check_lines += [""] return check_lines
def work_diff_merge_left(self, params): """ Worker branch that compares all chats on the left side for differences, copies them over to the right, posting progress back to application. """ result = { "output": "", "index": 0, "params": params, "chats": [], "count": 0, "type": "diff_merge_left", } error, e = None, None compared = [] db1, db2 = params["db1"], params["db2"] try: chats1 = params.get("chats") or db1.get_conversations() chats2 = db2.get_conversations() c1map = dict((c["identity"], c) for c in chats1) c2map = dict((c["identity"], c) for c in chats2) for c1 in chats1: c2 = c2map.get(c1["identity"]) c = c1.copy() c["messages1"] = c1["message_count"] or 0 c["messages2"] = c2["message_count"] or 0 if c2 else 0 c["c1"], c["c2"] = c1, c2 compared.append(c) compared.sort(key=lambda x: x["title"].lower()) result["count"] = len(compared) count_messages = 0 count_participants = 0 for index, chat in enumerate(compared): diff = self.get_chat_diff_left(chat, db1, db2) if self._stop_work: break # break for index, chat in enumerate(compared) if diff["messages"] \ or (chat["message_count"] and diff["participants"]): chat1 = chat["c1"] chat2 = chat["c2"] new_chat = not chat2 if new_chat: chat2 = chat1.copy() chat["c2"] = chat2 chat2["id"] = db2.insert_chat(chat2, db1) if diff["participants"]: db2.insert_participants(chat2, diff["participants"], db1) count_participants += len(diff["participants"]) if diff["messages"]: db2.insert_messages(chat2, diff["messages"], db1, chat1, self.yield_ui, self.REFRESH_COUNT) count_messages += len(diff["messages"]) newstr = "" if new_chat else "new " info = "Merged %s" % chat["title_long_lc"] if new_chat: info += " - new chat" if diff["messages"]: info += ", %s" % util.plural("%smessage" % newstr, diff["messages"]) else: info += ", no messages" result["output"] = info + "." result["diff"] = diff result["index"] = index result["chats"].append(chat) if not self._drop_results: if index < len(compared) - 1: result["status"] = ( "Scanning %s." % compared[index + 1]["title_long_lc"]) self.postback(result) result = { "output": "", "index": index, "params": params, "count": len(compared), "type": "diff_merge_left", "chats": [], } except Exception as e: error = traceback.format_exc() finally: if not self._drop_results: if compared: info = "Merged %s" % util.plural("new message", count_messages) if count_participants: info += " and %s" % util.plural( "new participant", count_participants) info += " \n\nto %s." % db2 else: info = "Nothing new to merge from %s to %s." % (db1, db2) result = { "type": "diff_merge_left", "done": True, "output": info, "params": params, "chats": [] } if error: result["error"] = error if e: result["error_short"] = "%s: %s" % (type(e).__name__, e.message) self.postback(result)
def work_diff(self, params): """ Worker branch that compares all chats for differences, posting results back to application. """ # {"htmls": [html result for db1, db2], # "chats": [differing chats in db1, db2]} result = {"htmls": ["", ""], "chats": [[], []], "params": params, "index": 0, "type": "diff"} db1, db2 = params["db1"], params["db2"] chats1 = db1.get_conversations() chats2 = db2.get_conversations() skypenames1 = [i["identity"] for i in db1.get_contacts()] skypenames2 = [i["identity"] for i in db2.get_contacts()] skypenames1.append(db1.id) skypenames2.append(db2.id) c1map = dict((c["identity"], c) for c in chats1) c2map = dict((c["identity"], c) for c in chats2) compared = [] for c1 in chats1: c2 = c2map.get(c1["identity"]) c = c1.copy() c["messages1"] = c1["message_count"] or 0 c["messages2"] = c2["message_count"] or 0 if c2 else 0 c["c1"], c["c2"] = c1, c2 compared.append(c) for c2 in chats2: c1 = c1map.get(c2["identity"]) if not c1: c = c2.copy() c["messages1"], c["messages2"] = 0, c["message_count"] or 0 c["c1"], c["c2"] = c1, c2 compared.append(c) compared.sort(key=lambda x: x["title"].lower()) for index, chat in enumerate(compared): diff = self.get_chat_diff(chat, db1, db2) if self._stop_work: break # break for index, chat in enumerate(compared) for i in range(2): new_chat = not chat["c1" if i else "c2"] newstr = "" if new_chat else "new " info = util.htmltag("a", {"href": chat["identity"]}, chat["title_long"], utf=False) if new_chat: info += " - new chat" if diff["messages"][i]: info += ", %s" % util.plural("%smessage" % newstr, diff["messages"][i]) if diff["participants"][i] and newstr: info += ", %s" % ( util.plural("%sparticipant" % newstr, diff["participants"][i])) if diff["messages"][i] or diff["participants"][i]: info += ".<br />" result["htmls"][i] += info result["chats"][i].append({"chat": chat, "diff": diff}) if not self._drop_results: self.postback(result) result = {"htmls": ["", ""], "chats": [[], []], "params": params, "index": index, "type": "diff"} if not self._drop_results: result["done"] = True self.postback(result)
class ModelFactory(object): def __init__(self, config): self.config = config self.used_model_names = [] self.used_table_names = [] schema = getattr(self.config, 'schema', None) self._metadata = MetaData(bind=config.engine) self._foreign_keys = {} kw = {} self.schemas = None if schema: if isinstance(schema, (list, tuple)): self.schemas = schema else: self.schemas = (schema, ) for schema in self.schemas: log.info('Reflecting database... schema:%s' % schema) self._metadata.reflect(schema=schema) else: log.info('Reflecting database...') self._metadata.reflect() self.DeclarativeBase = declarative_base(metadata=self._metadata) def _table_repr(self, table): s = "Table(u'%s', metadata,\n" % (table.name) for column in table.c: s += " %s,\n" % column_repr(column) if table.schema: s += " schema='%s'\n" % table.schema s += ")" return s def __repr__(self): tables = self.get_many_to_many_tables() tables.extend(self.get_tables_with_no_pks()) models = self.models s = StringIO() engine = self.config.engine if not isinstance(engine, basestring): engine = str(engine.url) s.write(constants.HEADER_DECL % engine) if 'postgres' in engine: s.write(constants.PG_IMPORT) self.used_table_names = [] self.used_model_names = [] for table in tables: if table not in self.tables: continue table_name = self.find_new_name(table.name, self.used_table_names) self.used_table_names.append(table_name) s.write('%s = %s\n\n' % (table_name, self._table_repr(table))) for model in models: s.write(model.__repr__()) s.write("\n\n") if self.config.example or self.config.interactive: s.write(constants.EXAMPLE_DECL % (models[0].__name__, models[0].__name__)) if self.config.interactive: s.write(constants.INTERACTIVE % ([model.__name__ for model in models], models[0].__name__)) return s.getvalue() @property def tables(self): if self.config.options.tables: tables = set(self.config.options.tables) return [ self._metadata.tables[t] for t in set(self._metadata.tables.keys()).intersection(tables) ] return self._metadata.tables.values() @property def table_names(self): return [t.name for t in self.tables] @property def models(self): if hasattr(self, '_models'): return self._models self.used_model_names = [] self.used_table_names = [] self._models = [] for table in self.get_non_many_to_many_tables(): try: self._models.append(self.create_model(table)) except exc.ArgumentError: log.warning( "Table with name %s ha no primary key. No ORM class created" % table.name) self._models.sort(by__name__) return self._models def get_tables_with_no_pks(self): r = [] for table in self.get_non_many_to_many_tables(): if not [c for c in table.columns if c.primary_key]: r.append(table) return r def model_table_lookup(self): if hasattr(self, '_model_table_lookup'): return self._model_table_lookup self._model_table_lookup = dict( ((m.__table__.name, m.__name__) for m in self.models)) return self._model_table_lookup def find_new_name(self, prefix, used, i=0): if i != 0: prefix = "%s%d" % (prefix, i) if prefix in used: prefix = prefix return self.find_new_name(prefix, used, i + 1) return prefix def create_model(self, table): #partially borrowed from Jorge Vargas' code #http://dpaste.org/V6YS/ log.debug('Creating Model from table: %s' % table.name) model_name = self.find_new_name(singular(name2label(table.name)), self.used_model_names) self.used_model_names.append(model_name) is_many_to_many_table = self.is_many_to_many_table(table) table_name = self.find_new_name(table.name, self.used_table_names) self.used_table_names.append(table_name) mtl = self.model_table_lookup class Temporal(self.DeclarativeBase): __table__ = table @classmethod def _relation_repr(cls, rel): target = rel.argument if target and inspect.isfunction(target): target = target() if isinstance(target, Mapper): target = target.class_ target = target.__name__ primaryjoin = '' lookup = mtl() if rel.primaryjoin is not None and hasattr( rel.primaryjoin, 'right'): right_lookup = lookup.get( rel.primaryjoin.right.table.name, '%s.c' % rel.primaryjoin.right.table.name) left_lookup = lookup.get( rel.primaryjoin.left.table.name, '%s.c' % rel.primaryjoin.left.table.name) primaryjoin = ", primaryjoin='%s.%s==%s.%s'" % ( left_lookup, rel.primaryjoin.left.name, right_lookup, rel.primaryjoin.right.name) elif hasattr(rel, '_as_string'): primaryjoin = ', primaryjoin="%s"' % rel._as_string secondary = '' secondaryjoin = '' if rel.secondary is not None: secondary = ", secondary=%s" % rel.secondary.name right_lookup = lookup.get( rel.secondaryjoin.right.table.name, '%s.c' % rel.secondaryjoin.right.table.name) left_lookup = lookup.get( rel.secondaryjoin.left.table.name, '%s.c' % rel.secondaryjoin.left.table.name) secondaryjoin = ", secondaryjoin='%s.%s==%s.%s'" % ( left_lookup, rel.secondaryjoin.left.name, right_lookup, rel.secondaryjoin.right.name) backref = '' # if rel.backref: # backref=", backref='%s'"%rel.backref.key return "%s = relation('%s'%s%s%s%s)" % (rel.key, target, primaryjoin, secondary, secondaryjoin, backref) @classmethod def __repr__(cls): log.debug('repring class with name %s' % cls.__name__) try: mapper = None try: mapper = class_mapper(cls) except exc.InvalidRequestError: log.warn( "A proper mapper could not be generated for the class %s, no relations will be created" % model_name) s = "" s += "class " + model_name + '(DeclarativeBase):\n' if is_many_to_many_table: s += " __table__ = %s\n\n" % table_name else: s += " __tablename__ = '%s'\n\n" % table_name if hasattr(cls, '__table_args__'): #if cls.__table_args__[0]: #for fkc in cls.__table_args__[0]: # fkc.__class__.__repr__ = foreignkeyconstraint_repr # break s += " __table_args__ = %s\n\n" % cls.__table_args__ s += " #column definitions\n" for column in sorted(cls.__table__.c, by_name): s += " %s = %s\n" % (column.name, column_repr(column)) s += "\n #relation definitions\n" ess = s # this is only required in SA 0.5 if mapper and RelationProperty: for prop in mapper.iterate_properties: if isinstance(prop, RelationshipProperty): s += ' %s\n' % cls._relation_repr(prop) return s except Exception, e: log.error("Could not generate class for: %s" % cls.__name__) from traceback import format_exc log.error(format_exc()) return '' #hack the class to have the right classname Temporal.__name__ = model_name #set up some blank table args Temporal.__table_args__ = {} #add in the schema if self.config.schema: Temporal.__table_args__[1]['schema'] = table.schema #trick sa's model registry to think the model is the correct name if model_name != 'Temporal': Temporal._decl_class_registry[ model_name] = Temporal._decl_class_registry['Temporal'] del Temporal._decl_class_registry['Temporal'] #add in single relations fks = self.get_single_foreign_keys_by_column(table) for column, fk in fks.iteritems(): related_table = fk.column.table if related_table not in self.tables: continue log.info(' Adding <primary> foreign key for:%s' % related_table.name) backref_name = plural(table_name) rel = relation( singular(name2label(related_table.name, related_table.schema)), primaryjoin=column == fk.column) #, backref=backref_name) setattr(Temporal, related_table.name, _deferred_relationship(Temporal, rel)) #add in the relations for the composites for constraint in table.constraints: if isinstance(constraint, ForeignKeyConstraint): if len(constraint.elements) > 1: related_table = constraint.elements[0].column.table related_classname = singular( name2label(related_table.name, related_table.schema)) primary_join = "and_(%s)" % ', '.join([ "%s.%s==%s.%s" % (model_name, k.parent.name, related_classname, k.column.name) for k in constraint.elements ]) rel = relation( related_classname, primaryjoin=primary_join # foreign_keys=[k.parent for k in constraint.elements] ) rel._as_string = primary_join setattr(Temporal, related_table.name, rel) # _deferred_relationship(Temporal, rel)) #add in many-to-many relations for join_table in self.get_related_many_to_many_tables(table.name): if join_table not in self.tables: continue primary_column = [ c for c in join_table.columns if c.foreign_keys and list(c.foreign_keys)[0].column.table == table ][0] for column in join_table.columns: if column.foreign_keys: key = list(column.foreign_keys)[0] if key.column.table is not table: related_column = related_table = list( column.foreign_keys)[0].column related_table = related_column.table if related_table not in self.tables: continue log.info( ' Adding <secondary> foreign key(%s) for:%s' % (key, related_table.name)) setattr( Temporal, plural(related_table.name), _deferred_relationship( Temporal, relation( singular( name2label(related_table.name, related_table.schema)), secondary=join_table, primaryjoin=list( primary_column.foreign_keys)[0].column == primary_column, secondaryjoin=column == related_column))) break return Temporal
def run_merge(filenames, output_filename=None): """Merges all Skype databases to a new database.""" dbs = [skypedata.SkypeDatabase(f) for f in filenames] db_base = dbs.pop() counts = collections.defaultdict(lambda: collections.defaultdict(int)) postbacks = Queue.Queue() name, ext = os.path.splitext(os.path.split(db_base.filename)[-1]) now = datetime.datetime.now().strftime("%Y%m%d") if not output_filename: output_filename = util.unique_path("%s.merged.%s%s" % (name, now, ext)) output("Creating %s, using %s as base." % (output_filename, db_base)) bar = ProgressBar() bar.start() shutil.copyfile(db_base.filename, output_filename) db2 = skypedata.SkypeDatabase(output_filename) chats2 = db2.get_conversations() db2.get_conversations_stats(chats2) args = {"db2": db2, "type": "diff_merge_left"} worker = workers.MergeThread(postbacks.put) try: for db1 in dbs: chats = db1.get_conversations() db1.get_conversations_stats(chats) bar.afterword = " Processing %.*s.." % (30, db1) worker.work(dict(args, db1=db1, chats=chats)) while True: result = postbacks.get() if "error" in result: output("Error merging %s:\n\n%s" % (db1, result["error"])) db1 = None # Signal for global break break # break while True if "done" in result: break # break while True if "diff" in result: counts[db1]["chats"] += 1 counts[db1]["msgs"] += len(result["diff"]["messages"]) if "index" in result: bar.max = result["count"] bar.update(result["index"]) if result.get("output"): log(result["output"]) if not db1: break # break for db1 in dbs bar.stop() bar.afterword = " Processed %s." % db1 bar.update(bar.max) output() finally: worker and (worker.stop(), worker.join()) if not counts: output("Nothing new to merge.") db2.close() os.unlink(output_filename) else: for db1 in dbs: output("Merged %s in %s from %s." % (util.plural("message", counts[db1]["msgs"]), util.plural("chat", counts[db1]["chats"]), db1)) output("Merge into %s complete." % db2)
def run(self): self._is_running = True # For identifying "chat:xxx" and "from:xxx" keywords query_parser = searchparser.SearchQueryParser() result = None while self._is_running: try: search = self._queue.get() if not search: continue # continue while self._is_running is_text_output = ("text" == search.get("output")) wrap_html = None # MessageParser wrap function, for HTML output if is_text_output: TEMPLATES = { "chat": templates.SEARCH_ROW_CHAT_TXT, "contact": templates.SEARCH_ROW_CONTACT_TXT, "message": templates.SEARCH_ROW_MESSAGE_TXT, "table": templates.SEARCH_ROW_TABLE_HEADER_TXT, "row": templates.SEARCH_ROW_TABLE_TXT, } wrap_b = lambda x: "**%s**" % x.group(0) output = {"format": "text"} else: TEMPLATES = { "chat": templates.SEARCH_ROW_CHAT_HTML, "contact": templates.SEARCH_ROW_CONTACT_HTML, "message": templates.SEARCH_ROW_MESSAGE_HTML, "table": templates.SEARCH_ROW_TABLE_HEADER_HTML, "row": templates.SEARCH_ROW_TABLE_HTML, } wrap_b = lambda x: "<b>%s</b>" % x.group(0) output = {"format": "html"} width = search.get("width", -1) if width > 0: dc = wx.MemoryDC() dc.SetFont( wx.Font(8, wx.SWISS, wx.NORMAL, wx.NORMAL, face=conf.HistoryFontName)) wrap_html = lambda x: wx.lib.wordwrap.wordwrap( x, width, dc) output["wrap"] = True main.log("Searching for \"%(text)s\" in %(table)s (%(db)s)." % search) self._stop_work = False self._drop_results = False parser = skypedata.MessageParser(search["db"], wrapfunc=wrap_html) # {"output": text with results, "map": link data map} # map data: {"contact:666": {"contact": {contact data}}, } result_type, result_count, count = None, 0, 0 result = { "output": "", "map": {}, "search": search, "count": 0 } sql, params, match_words = query_parser.Parse(search["text"]) # Turn wildcard characters * into regex-compatible .* match_words_re = [ ".*".join(map(re.escape, w.split("*"))) for w in match_words ] patt = "(%s)" % "|".join(match_words_re) # For replacing matching words with <b>words</b> pattern_replace = re.compile(patt, re.IGNORECASE) # Find chats with a matching title or matching participants chats = search["db"].get_conversations() chats.sort(key=lambda x: x["title"]) chat_map = {} # {chat id: {chat data}} template_chat = step.Template(TEMPLATES["chat"]) for chat in chats: chat_map[chat["id"]] = chat if "conversations" == search["table"] and match_words: title_matches = False matching_authors = [] if self.match_all(chat["title"], match_words): title_matches = True for participant in chat["participants"]: contact = participant["contact"] if contact: for n in filter(None, [ contact["fullname"], contact["displayname"], contact["identity"] ]): if self.match_all(n, match_words) \ and contact not in matching_authors: matching_authors.append(contact) if title_matches or matching_authors: count += 1 result_count += 1 result["output"] += template_chat.expand(locals()) key = "chat:%s" % chat["id"] result["map"][key] = {"chat": chat["id"]} if not count % conf.SearchResultsChunk \ and not self._drop_results: result["count"] = result_count self.postback(result) result = { "output": "", "map": {}, "search": search, "count": 0 } if self._stop_work: break # break for chat in chats if result["output"] and not self._drop_results: result["count"] = result_count self.postback(result) result = { "output": "", "map": {}, "search": search, "count": 0 } # Find contacts with a matching name if not self._stop_work and "contacts" == search["table"] \ and match_words: count = 0 contacts = search["db"].get_contacts() # Possibly more: country (ISO code, need map), birthday # (base has YYYYMMDD in integer field). match_fields = [ "displayname", "skypename", "province", "city", "pstnnumber", "phone_home", "phone_office", "phone_mobile", "homepage", "emails", "about", "mood_text", ] template_contact = step.Template(TEMPLATES["contact"]) for contact in contacts: match = False fields_filled = {} for field in match_fields: if contact[field]: val = contact[field] if self.match_all(val, match_words): match = True val = pattern_replace.sub(wrap_b, val) fields_filled[field] = val if match: count += 1 result_count += 1 result["output"] += template_contact.expand( locals()) if not (self._drop_results or count % conf.SearchResultsChunk): result["count"] = result_count self.postback(result) result = { "output": "", "map": {}, "search": search, "count": 0 } if self._stop_work: break # break for contact in contacts if result["output"] and not self._drop_results: result["count"] = result_count self.postback(result) result = { "output": "", "map": {}, "search": search, "count": 0 } # Find messages with a matching body if not self._stop_work and "messages" == search["table"]: template_message = step.Template(TEMPLATES["message"]) count, result_type = 0, "messages" chat_messages = {} # {chat id: [message, ]} chat_order = [] # [chat id, ] messages = search["db"].get_messages( additional_sql=sql, additional_params=params, ascending=False, use_cache=False) for m in messages: chat = chat_map.get(m["convo_id"]) body = parser.parse( m, pattern_replace if match_words else None, output) count += 1 result_count += 1 result["output"] += template_message.expand(locals()) key = "message:%s" % m["id"] result["map"][key] = { "chat": chat["id"], "message": m["id"] } if is_text_output or ( not self._drop_results and not count % conf.SearchResultsChunk): result["count"] = result_count self.postback(result) result = { "output": "", "map": {}, "search": search, "count": 0 } if self._stop_work or (not is_text_output and count >= conf.SearchMessagesMax): break # break for m in messages infotext = search["table"] if not self._stop_work and "all tables" == search["table"]: infotext, result_type = "", "table row" # Search over all fields of all tables. template_table = step.Template(TEMPLATES["table"]) template_row = step.Template(TEMPLATES["row"]) for table in search["db"].get_tables(): table["columns"] = search["db"].get_table_columns( table["name"]) sql, params, words = \ query_parser.Parse(search["text"], table) if not sql: continue # continue for table in search["db"].. infotext += (", " if infotext else "") + table["name"] rows = search["db"].execute(sql, params) row = rows.fetchone() if not row: continue # continue for table in search["db"].. result["output"] = template_table.expand(locals()) count = 0 while row: count += 1 result_count += 1 result["output"] += template_row.expand(locals()) key = "table:%s:%s" % (table["name"], count) result["map"][key] = { "table": table["name"], "row": row } if not count % conf.SearchResultsChunk \ and not self._drop_results: result["count"] = result_count self.postback(result) result = { "output": "", "map": {}, "search": search, "count": 0 } if self._stop_work or (not is_text_output and result_count >= conf.SearchTableRowsMax): break # break while row row = rows.fetchone() if not self._drop_results: if not is_text_output: result["output"] += "</table>" result["count"] = result_count self.postback(result) result = { "output": "", "map": {}, "search": search, "count": 0 } infotext += " (%s)" % util.plural("result", count) if self._stop_work or ( not is_text_output and result_count >= conf.SearchTableRowsMax): break # break for table in search["db"].. single_table = ("," not in infotext) infotext = "table%s: %s" % \ ("" if single_table else "s", infotext) if not single_table: infotext += "; %s in total" % \ util.plural("result", result_count) final_text = "No matches found." if self._drop_results: result["output"] = "" if result_count: final_text = "Finished searching %s." % infotext if self._stop_work: final_text += " Stopped by user." elif "messages" == result_type and not is_text_output \ and count >= conf.SearchMessagesMax: final_text += " Stopped at %s limit %s." % \ (result_type, conf.SearchMessagesMax) elif "table row" == result_type and not is_text_output \ and count >= conf.SearchTableRowsMax: final_text += " Stopped at %s limit %s." % \ (result_type, conf.SearchTableRowsMax) result[ "output"] += "</table><br /><br />%s</font>" % final_text if is_text_output: result["output"] = "" result["done"] = True result["count"] = result_count self.postback(result) main.log("Search found %(count)s results." % result) except Exception as e: if not result: result = {} result["done"], result["error"] = True, traceback.format_exc() result["error_short"] = "%s: %s" % (type(e).__name__, e.message) self.postback(result)
def flush(self): try: self.flush_count += 1 packets_per_second = self.metrics_aggregator.packets_per_second(self.interval) packet_count = self.metrics_aggregator.total_count metrics = self.metrics_aggregator.flush() count = len(metrics) should_log = self.flush_count <= FLUSH_LOGGING_INITIAL or self.flush_count % FLUSH_LOGGING_PERIOD == 0 if not count: if should_log: log.info("Flush #%s: No metrics to flush." % self.flush_count) else: log.debug("Flush #%s: No metrics to flush." % self.flush_count) else: if should_log: log.info("Flush #%s: flushing %s metric%s" % (self.flush_count, count, plural(count))) else: log.debug("Flush #%s: flushing %s metric%s" % (self.flush_count, count, plural(count))) self.submit(metrics) events = self.metrics_aggregator.flush_events() event_count = len(events) if not event_count: if should_log: log.info("Flush #%s: No events to flush." % self.flush_count) else: log.debug("Flush #%s: No events to flush." % self.flush_count) else: if should_log: log.info("Flush #%s: flushing %s event%s" % (self.flush_count, len(events), plural(len(events)))) else: log.debug("Flush #%s: flushing %s event%s" % (self.flush_count, len(events), plural(len(events)))) self.submit_events(events) if self.flush_count == FLUSH_LOGGING_INITIAL: log.info("First flushes done, next flushes will be logged every %s flushes." % FLUSH_LOGGING_PERIOD) # Persist a status message. packet_count = self.metrics_aggregator.total_count DogstatsdStatus( flush_count=self.flush_count, packet_count=packet_count, packets_per_second=packets_per_second, metric_count=count, event_count=event_count, ).persist() except Exception, e: log.exception("Error flushing metrics")
Temporal.__table_args__ = {"schema": table.schema} # trick sa's model registry to think the model is the correct name if model_name != "Temporal": Temporal._decl_class_registry[model_name] = Temporal._decl_class_registry["Temporal"] del Temporal._decl_class_registry["Temporal"] # add in single relations fks = self.get_foreign_keys(table) for related_table in sorted(fks.keys(), by_name): columns = fks[related_table] if len(columns) > 1: continue column = columns[0] log.info(" Adding <primary> foreign key for:%s" % related_table.name) backref_name = plural(table_name) # import ipdb; ipdb.set_trace() rel = relation( singular(name2label(related_table.name, related_table.schema)), primaryjoin=column == list(column.foreign_keys)[0].column, ) # , backref=backref_name) setattr(Temporal, related_table.name, _deferred_relationship(Temporal, rel)) # add in many-to-many relations for join_table in self.get_related_many_to_many_tables(table.name): primary_column = [ c for c in join_table.columns if c.foreign_keys and list(c.foreign_keys)[0].column.table == table ][0] # import ipdb; ipdb.set_trace();
def work_diff_left(self, params): """ Worker branch that compares all chats on the left side for differences, posting results back to application. """ # {"output": "html result for db1, db2", # "index": currently processed chat index, # "chats": [differing chats in db1]} result = { "output": "", "chats": [], "params": params, "index": 0, "type": "diff_left" } db1, db2 = params["db1"], params["db2"] chats1 = params.get("chats") or db1.get_conversations() chats2 = db2.get_conversations() c1map = dict((c["identity"], c) for c in chats1) c2map = dict((c["identity"], c) for c in chats2) compared = [] for c1 in chats1: c2 = c2map.get(c1["identity"]) c = c1.copy() c["messages1"] = c1["message_count"] or 0 c["messages2"] = c2["message_count"] or 0 if c2 else 0 c["c1"], c["c2"] = c1, c2 compared.append(c) compared.sort(key=lambda x: x["title"].lower()) info_template = step.Template(templates.DIFF_RESULT_ITEM) for index, chat in enumerate(compared): diff = self.get_chat_diff_left(chat, db1, db2) if self._stop_work: break # break for index, chat in enumerate(compared) if diff["messages"] \ or (chat["message_count"] and diff["participants"]): new_chat = not chat["c2"] newstr = "" if new_chat else "new " info = info_template.expand(chat=chat) if new_chat: info += " - new chat" if diff["messages"]: info += ", %s" % util.plural("%smessage" % newstr, diff["messages"]) else: info += ", no messages" if diff["participants"] and not new_chat: info += ", %s" % (util.plural("%sparticipant" % newstr, diff["participants"])) info += ".<br />" result["output"] += info result["chats"].append({"chat": chat, "diff": diff}) result["index"] = index if not self._drop_results: if index < len(compared) - 1: result["status"] = ("Scanning %s." % compared[index + 1]["title_long_lc"]) self.postback(result) result = { "output": "", "chats": [], "index": index, "params": params, "type": "diff_left" } if not self._drop_results: result["done"] = True self.postback(result)
def create_model(self, table): # partially borrowed from Jorge Vargas' code # http://dpaste.org/V6YS/ log.debug('Creating Model from table: %s' % table.name) model_name = self.find_new_name(singular(name2label(table.name)), self.used_model_names) self.used_model_names.append(model_name) is_many_to_many_table = self.is_many_to_many_table(table) table_name = self.find_new_name(table.name, self.used_table_names) self.used_table_names.append(table_name) self.table_model_dict[table_name] = model_name # Kamil Edit mtl = self.model_table_lookup class Temporal(self.DeclarativeBase): __table__ = table @classmethod def _relation_repr(cls, rel): target = rel.argument if target and inspect.isfunction(target): target = target() if isinstance(target, Mapper): target = target.class_ target = target.__name__ primaryjoin = '' lookup = mtl() foo = rel.key if rel.primaryjoin is not None and hasattr( rel.primaryjoin, 'right'): right_lookup = lookup.get( rel.primaryjoin.right.table.name, '%s.c' % rel.primaryjoin.right.table.name) left_lookup = lookup.get( rel.primaryjoin.left.table.name, '%s.c' % rel.primaryjoin.left.table.name) primaryjoin = ", primaryjoin='%s.%s==%s.%s'" % ( left_lookup, rel.primaryjoin.left.name, right_lookup, rel.primaryjoin.right.name) elif hasattr(rel, '_as_string'): primaryjoin = ', primaryjoin="%s"' % rel._as_string secondary = '' secondaryjoin = '' if rel.secondary is not None: """ **HACK**: If there is a secondary relationship like between Venue, Event, and Event_Type, then I'm only going show a primary relationship. "Events = relationship('Event', primaryjoin='Venue.id==Event.venue_id')" and not "Event_Types = relation('EventType', primaryjoin='Venue.id==Event.venue_id', secondary=Event, secondaryjoin='Event.event_type_id==EventType.id')" """ if rel.secondary.name in self.table_model_dict: target = self.table_model_dict[rel.secondary.name] else: target = self.find_new_name( singular(name2label(rel.secondary.name)), []) # **HACK** secondary = '' secondaryjoin = '' foo = plural(rel.secondary.name) backref = '' # if rel.backref: # backref=", backref='%s'"%rel.backref.key return "%s = relationship('%s'%s%s%s%s)" % ( foo, target, primaryjoin, secondary, secondaryjoin, backref) @classmethod def __repr__(cls): log.debug('repring class with name %s' % cls.__name__) try: mapper = None try: mapper = class_mapper(cls) except exc.InvalidRequestError: log.warn( "A proper mapper could not be generated for the class %s, no relations will be created" % model_name) s = "" s += "class " + model_name + '(DeclarativeBase):\n' if is_many_to_many_table: s += " __table__ = %s\n\n" % table_name else: s += " __tablename__ = '%s'\n\n" % table_name if hasattr(cls, '__table_args__'): # if cls.__table_args__[0]: # for fkc in cls.__table_args__[0]: # fkc.__class__.__repr__ = foreignkeyconstraint_repr # break s += " __table_args__ = %s\n\n" % cls.__table_args__ s += " #column definitions\n" for column in sorted(cls.__table__.c, by_name): s += " %s = %s\n" % (column.name, column_repr(column)) s += "\n #relation definitions\n" # this is only required in SA 0.5 if mapper and RelationProperty: for prop in mapper.iterate_properties: if isinstance(prop, RelationshipProperty): s += ' %s\n' % cls._relation_repr(prop) return s except Exception: log.error("Could not generate class for: %s" % cls.__name__) from traceback import format_exc log.error(format_exc()) return '' # hack the class to have the right classname Temporal.__name__ = model_name # set up some blank table args Temporal.__table_args__ = {} # add in the schema if self.config.schema: Temporal.__table_args__[1]['schema'] = table.schema # trick sa's model registry to think the model is the correct name if model_name != 'Temporal': Temporal._decl_class_registry[ model_name] = Temporal._decl_class_registry['Temporal'] del Temporal._decl_class_registry['Temporal'] # add in single relations fks = self.get_single_foreign_keys_by_column(table) for column, fk in fks.iteritems(): related_table = fk.column.table if related_table not in self.tables: continue log.info(' Adding <primary> foreign key for:%s' % related_table.name) rel = relation( singular(name2label(related_table.name, related_table.schema)), primaryjoin=column == fk.column) # , backref=backref_name) setattr(Temporal, related_table.name, _deferred_relationship(Temporal, rel)) # add in the relations for the composites for constraint in table.constraints: if isinstance(constraint, ForeignKeyConstraint): if len(constraint.elements) > 1: related_table = constraint.elements[0].column.table related_classname = singular( name2label(related_table.name, related_table.schema)) primary_join = "and_(%s)" % ', '.join([ "%s.%s==%s.%s" % (model_name, k.parent.name, related_classname, k.column.name) for k in constraint.elements ]) rel = relation( related_classname, primaryjoin=primary_join # foreign_keys=[k.parent for k in constraint.elements] ) rel._as_string = primary_join setattr(Temporal, related_table.name, rel) # _deferred_relationship(Temporal, rel)) # add in many-to-many relations for join_table in self.get_related_many_to_many_tables(table.name): if join_table not in self.tables: continue primary_column = [ c for c in join_table.columns if c.foreign_keys and list(c.foreign_keys)[0].column.table == table ][0] for column in join_table.columns: if column.foreign_keys: key = list(column.foreign_keys)[0] if key.column.table is not table: related_column = related_table = list( column.foreign_keys)[0].column related_table = related_column.table if related_table not in self.tables: continue log.info( ' Adding <secondary> foreign key(%s) for:%s' % (key, related_table.name)) setattr( Temporal, plural(related_table.name), _deferred_relationship( Temporal, relation( singular( name2label(related_table.name, related_table.schema)), secondary=join_table, primaryjoin=list( primary_column.foreign_keys)[0].column == primary_column, secondaryjoin=column == related_column))) break return Temporal
def cropPlaylist(self, number): self._sendDebug( "Cropping playlist by " + str(number) + " track" + util.plural(number) + ".") for _ in range(number): self.deleteTrack(0)
def body_lines(self): # Metadata whitelist metadata_whitelist = [ 'hostname', 'fqdn', 'ipv4', 'instance-id' ] lines = [ 'Clocks', '======', '' ] try: ntp_offset, ntp_styles = get_ntp_info() lines.append(' ' + style('NTP offset', *ntp_styles) + ': ' + style('%s s' % round(ntp_offset, 4), *ntp_styles)) except Exception as e: lines.append(' NTP offset: Unknown (%s)' % str(e)) lines.append(' System UTC time: ' + datetime.datetime.utcnow().__str__()) lines.append('') # Paths to checks.d/conf.d lines += [ 'Paths', '=====', '' ] osname = config.get_os() try: confd_path = config.get_confd_path(osname) except config.PathNotFound: confd_path = 'Not found' try: checksd_path = config.get_checksd_path(osname) except config.PathNotFound: checksd_path = 'Not found' lines.append(' conf.d: ' + confd_path) lines.append(' checks.d: ' + checksd_path) lines.append('') # Hostnames lines += [ 'Hostnames', '=========', '' ] if not self.host_metadata: lines.append(" No host information available yet.") else: for key, host in self.host_metadata.iteritems(): for whitelist_item in metadata_whitelist: if whitelist_item in key: lines.append(" " + key + ": " + host) break lines.append('') # Checks.d Status lines += [ 'Checks', '======', '' ] check_statuses = self.check_statuses + get_jmx_status() if not check_statuses: lines.append(" No checks have run yet.") else: for cs in check_statuses: check_lines = [ ' ' + cs.name + ' ({})'.format(cs.check_version), ' ' + '-' * (len(cs.name) + 3 + len(cs.check_version)) ] if cs.init_failed_error: check_lines.append(" - initialize check class [%s]: %s" % (style(STATUS_ERROR, 'red'), repr(cs.init_failed_error))) if self.verbose and cs.init_failed_traceback: check_lines.extend(' ' + line for line in cs.init_failed_traceback.split('\n')) else: for s in cs.instance_statuses: c = 'green' if s.has_warnings(): c = 'yellow' if s.has_error(): c = 'red' line = " - instance #%s [%s]" % ( s.instance_id, style(s.status, c)) if s.has_error(): line += u": %s" % s.error if s.metric_count is not None: line += " collected %s metrics" % s.metric_count if s.instance_check_stats is not None: line += " Last run duration: %s" % s.instance_check_stats.get('run_time') check_lines.append(line) if s.has_warnings(): for warning in s.warnings: warn = warning.split('\n') if not len(warn): continue check_lines.append(u" %s: %s" % (style("Warning", 'yellow'), warn[0])) check_lines.extend(u" %s" % l for l in warn[1:]) if self.verbose and s.traceback is not None: check_lines.extend(' ' + line for line in s.traceback.split('\n')) check_lines += [ " - Collected %s metric%s, %s event%s & %s service check%s" % ( cs.metric_count, plural(cs.metric_count), cs.event_count, plural(cs.event_count), cs.service_check_count, plural(cs.service_check_count)), ] if cs.check_stats is not None: check_lines += [ " - Stats: %s" % pretty_statistics(cs.check_stats) ] if cs.library_versions is not None: check_lines += [ " - Dependencies:"] for library, version in cs.library_versions.iteritems(): check_lines += [ " - %s: %s" % (library, version)] check_lines += [""] lines += check_lines # Metadata status metadata_enabled = 1#_is_affirmative(get_config().get('display_service_metadata', False)) if metadata_enabled: lines += [ "", "Service metadata", "================", "" ] if not check_statuses: lines.append(" No checks have run yet.") else: meta_lines = [] for cs in check_statuses: # Check title check_line = [ ' ' + cs.name, ' ' + '-' * len(cs.name) ] instance_lines = [] for i, meta in enumerate(cs.service_metadata): if not meta: continue instance_lines += [" - instance #%s:" % i] for k, v in meta.iteritems(): instance_lines += [" - %s: %s" % (k, v)] if instance_lines: check_line += instance_lines meta_lines += check_line if meta_lines: lines += meta_lines else: lines.append(" No metadata were collected.") # Emitter status lines += [ "", "Emitters", "========", "" ] if not self.emitter_statuses: lines.append(" No emitters have run yet.") else: for es in self.emitter_statuses: c = 'green' if es.has_error(): c = 'red' line = " - %s [%s]" % (es.name, style(es.status, c)) if es.status != STATUS_OK: line += ": %s" % es.error lines.append(line) return lines
def prompt(choices, mode='*'): if mode not in PROMPT_MODES: raise ValueError("mode '{}' is invalid".format(mode)) if len(choices) > 26: raise ValueError("too many choices") if mode == '*': header = "select zero or more:" max, min = float('inf'), 0 elif mode == '+': header = "select one or more:" max, min = float('inf'), 1 elif mode in [1, '1']: header = "select one:" max, min = 1, 1 elif mode == '?': header = "select zero or one:" max, min = 1, 0 letters = list(map(lambda x: chr(ord('a') + x), range(len(choices)))) num_selections = 0 selections = [] # unique indices into choices list while num_selections < min or num_selections < max: util.print(util.green(header)) for i in range(len(choices)): if i in selections: choice = " × " else: choice = " " choice += str(letters[i]) + '. ' + str(choices[i]) if i in selections: choice = util.yellow(choice) util.print(choice) try: sel = input(util.green("make a selection (or ! to commit): ")) except KeyboardInterrupt: util.exit(util.ERR_INTERRUPTED) if sel == '!': if num_selections < min: util.error("can't stop now; you must make " "{} {}".format(min, util.plural("selection", min))) continue else: break try: if letters.index(sel) in selections: selections.remove(letters.index(sel)) continue selections.append(letters.index(sel)) num_selections += 1 except ValueError: if sel == '': util.print("make a selection (or ! to commit)") else: util.error("invalid selection: not in list") continue return selections
def export_chats(chats, path, format, db, messages=None, skip=True, progress=None): """ Exports the specified chats from the database under path. @param chats list of chat dicts, as returned from SkypeDatabase @param path full path of directory where to save @param format export format (html|txt|xlsx|csv|filename.ext). If format is filename.ext, a single file is created: for single chat exports and multi chat XLSX exports (for multi-file exports, filenames are named by chats). @param db SkypeDatabase instance @param messages list messages to export if a single chat @param skip whether to skip chats with no messages @param progress function called before exporting each chat, with the number of messages exported so far @return (list of exported filenames, number of chats exported) """ files, count = [], 0 def make_filename(chat): if len(format) > 4: # Filename already given in format filename = os.path.join(path, format) else: args = collections.defaultdict(str) args.update(chat) filename = "%s.%s" % (conf.ExportChatTemplate % args, format) filename = os.path.join(path, util.safe_filename(filename)) filename = util.unique_path(filename) return filename main.logstatus("Exporting %s from %s %sto %s.", util.plural("chat", chats), db.filename, "" if len(format) > 4 else "as %s " % format.upper(), format if len(format) > 4 else path) if format.lower().endswith(".xlsx"): filename = make_filename(chats[0]) count = export_chats_xlsx(chats, filename, db, messages, skip, progress) files.append(filename) else: if not os.path.exists(path): os.makedirs(path) export_func = (export_chats_xlsx if format.lower().endswith("xlsx") else export_chat_csv if format.lower().endswith("csv") else export_chat_template) message_count = 0 for chat in chats: if skip and not messages and not chat["message_count"]: main.log("Skipping exporting %s: no messages.", chat["title_long_lc"]) if progress: progress(message_count) continue # continue for chat in chats main.status("Exporting %s.", chat["title_long_lc"]) if progress: progress(message_count) filename = make_filename(chat) msgs = messages or db.get_messages(chat) chatarg = [chat] if "xlsx" == format.lower() else chat export_func(chatarg, filename, db, msgs) message_count += chat["message_count"] files.append(filename) count = len(files) return (files, count)
def body_lines(self): # Metadata whitelist metadata_whitelist = [ 'hostname', 'fqdn', 'ipv4', 'instance-id' ] lines = [ 'Clocks', '======', '' ] try: ntp_offset, ntp_styles = get_ntp_info() lines.append(' ' + style('NTP offset', *ntp_styles) + ': ' + style('%s s' % round(ntp_offset, 4), *ntp_styles)) except Exception as e: lines.append(' NTP offset: Unknown (%s)' % str(e)) lines.append(' System UTC time: ' + datetime.datetime.utcnow().__str__()) lines.append('') # Paths to checks.d/conf.d lines += [ 'Paths', '=====', '' ] osname = config.get_os() try: confd_path = config.get_confd_path(osname) except config.PathNotFound: confd_path = 'Not found' try: checksd_path = config.get_checksd_path(osname) except config.PathNotFound: checksd_path = 'Not found' lines.append(' conf.d: ' + confd_path) lines.append(' checks.d: ' + checksd_path) lines.append('') # Hostnames lines += [ 'Hostnames', '=========', '' ] if not self.host_metadata: lines.append(" No host information available yet.") else: for key, host in self.host_metadata.iteritems(): for whitelist_item in metadata_whitelist: if whitelist_item in key: lines.append(" " + key + ": " + host) break lines.append('') # Checks.d Status lines += [ 'Checks', '======', '' ] check_statuses = self.check_statuses + get_jmx_status() if not check_statuses: lines.append(" No checks have run yet.") else: for cs in check_statuses: check_lines = [ ' ' + cs.name + ' ({})'.format(cs.check_version), ' ' + '-' * (len(cs.name) + 3 + len(cs.check_version)) ] if cs.init_failed_error: check_lines.append(" - initialize check class [%s]: %s" % (style(STATUS_ERROR, 'red'), repr(cs.init_failed_error))) if self.verbose and cs.init_failed_traceback: check_lines.extend(' ' + line for line in cs.init_failed_traceback.split('\n')) else: for s in cs.instance_statuses: c = 'green' if s.has_warnings(): c = 'yellow' if s.has_error(): c = 'red' line = " - instance #%s [%s]" % ( s.instance_id, style(s.status, c)) if s.has_error(): line += u": %s" % s.error if s.metric_count is not None: line += " collected %s metrics" % s.metric_count if s.instance_check_stats is not None: line += " Last run duration: %s" % s.instance_check_stats.get('run_time') check_lines.append(line) if s.has_warnings(): for warning in s.warnings: warn = warning.split('\n') if not len(warn): continue check_lines.append(u" %s: %s" % (style("Warning", 'yellow'), warn[0])) check_lines.extend(u" %s" % l for l in warn[1:]) if self.verbose and s.traceback is not None: check_lines.extend(' ' + line for line in s.traceback.split('\n')) check_lines += [ " - Collected %s metric%s, %s event%s & %s service check%s" % ( cs.metric_count, plural(cs.metric_count), cs.event_count, plural(cs.event_count), cs.service_check_count, plural(cs.service_check_count)), ] if cs.check_stats is not None: check_lines += [ " - Stats: %s" % pretty_statistics(cs.check_stats) ] if cs.library_versions is not None: check_lines += [ " - Dependencies:"] for library, version in cs.library_versions.iteritems(): check_lines += [ " - %s: %s" % (library, version)] check_lines += [""] lines += check_lines # Metadata status metadata_enabled = _is_affirmative(get_config().get('display_service_metadata', False)) if metadata_enabled: lines += [ "", "Service metadata", "================", "" ] if not check_statuses: lines.append(" No checks have run yet.") else: meta_lines = [] for cs in check_statuses: # Check title check_line = [ ' ' + cs.name, ' ' + '-' * len(cs.name) ] instance_lines = [] for i, meta in enumerate(cs.service_metadata): if not meta: continue instance_lines += [" - instance #%s:" % i] for k, v in meta.iteritems(): instance_lines += [" - %s: %s" % (k, v)] if instance_lines: check_line += instance_lines meta_lines += check_line if meta_lines: lines += meta_lines else: lines.append(" No metadata were collected.") # Emitter status lines += [ "", "Emitters", "========", "" ] if not self.emitter_statuses: lines.append(" No emitters have run yet.") else: for es in self.emitter_statuses: c = 'green' if es.has_error(): c = 'red' line = " - %s [%s]" % (es.name, style(es.status, c)) if es.status != STATUS_OK: line += ": %s" % es.error lines.append(line) return lines
class CollectorStatus(AgentStatus): NAME = 'Collector' def __init__(self, check_statuses=None, emitter_statuses=None, metadata=None): AgentStatus.__init__(self) self.check_statuses = check_statuses or [] self.emitter_statuses = emitter_statuses or [] self.metadata = metadata or [] @property def status(self): for check_status in self.check_statuses: if check_status.status == STATUS_ERROR: return STATUS_ERROR return STATUS_OK def has_error(self): return self.status != STATUS_OK def body_lines(self): # Metadata whitelist metadata_whitelist = ['hostname', 'fqdn', 'ipv4', 'instance-id'] lines = ['Clocks', '======', ''] try: ntp_offset, ntp_styles = get_ntp_info() lines.append(' ' + style('NTP offset', *ntp_styles) + ': ' + style('%s s' % round(ntp_offset, 4), *ntp_styles)) except Exception, e: lines.append(' NTP offset: Unknown (%s)' % str(e)) lines.append(' System UTC time: ' + datetime.datetime.utcnow().__str__()) lines.append('') # Paths to checks.d/conf.d lines += ['Paths', '=====', ''] osname = config.get_os() try: confd_path = config.get_confd_path(osname) except config.PathNotFound: confd_path = 'Not found' try: checksd_path = config.get_checksd_path(osname) except config.PathNotFound: checksd_path = 'Not found' lines.append(' conf.d: ' + confd_path) lines.append(' checks.d: ' + checksd_path) lines.append('') # Hostnames lines += ['Hostnames', '=========', ''] if not self.metadata: lines.append(" No host information available yet.") else: for key, host in self.metadata.items(): for whitelist_item in metadata_whitelist: if whitelist_item in key: lines.append(" " + key + ": " + host) break lines.append('') # Checks.d Status lines += ['Checks', '======', ''] check_statuses = self.check_statuses + get_jmx_status() if not check_statuses: lines.append(" No checks have run yet.") else: for cs in check_statuses: check_lines = [' ' + cs.name, ' ' + '-' * len(cs.name)] if cs.init_failed_error: check_lines.append( " - initialize check class [%s]: %s" % (style( STATUS_ERROR, 'red'), repr(cs.init_failed_error))) if self.verbose and cs.init_failed_traceback: check_lines.extend( ' ' + line for line in cs.init_failed_traceback.split('\n')) else: for s in cs.instance_statuses: c = 'green' if s.has_warnings(): c = 'yellow' if s.has_error(): c = 'red' line = " - instance #%s [%s]" % (s.instance_id, style(s.status, c)) if s.has_error(): line += u": %s" % s.error if s.metric_count is not None: line += " collected %s metrics" % s.metric_count check_lines.append(line) if s.has_warnings(): for warning in s.warnings: warn = warning.split('\n') if not len(warn): continue check_lines.append( u" %s: %s" % (style("Warning", 'yellow'), warn[0])) check_lines.extend(u" %s" % l for l in warn[1:]) if self.verbose and s.traceback is not None: check_lines.extend( ' ' + line for line in s.traceback.split('\n')) check_lines += [ " - Collected %s metric%s, %s event%s & %s service check%s" % (cs.metric_count, plural( cs.metric_count), cs.event_count, plural(cs.event_count), cs.service_check_count, plural(cs.service_check_count)), ] if cs.library_versions is not None: check_lines += [" - Dependencies:"] for library, version in cs.library_versions.iteritems( ): check_lines += [ " - %s: %s" % (library, version) ] check_lines += [""] lines += check_lines # Emitter status lines += ["", "Emitters", "========", ""] if not self.emitter_statuses: lines.append(" No emitters have run yet.") else: for es in self.emitter_statuses: c = 'green' if es.has_error(): c = 'red' line = " - %s [%s]" % (es.name, style(es.status, c)) if es.status != STATUS_OK: line += ": %s" % es.error lines.append(line) return lines
warn = warning.split('\n') if not len(warn): continue check_lines.append( u" %s: %s" % (style("Warning", 'yellow'), warn[0])) check_lines.extend(u" %s" % l for l in warn[1:]) if self.verbose and s.traceback is not None: check_lines.extend( ' ' + line for line in s.traceback.split('\n')) check_lines += [ " - Collected %s metric%s, %s event%s & %s service check%s" % (cs.metric_count, plural( cs.metric_count), cs.event_count, plural(cs.event_count), cs.service_check_count, plural(cs.service_check_count)), ] if cs.check_stats is not None: check_lines += [ " - Stats: %s" % pretty_statistics(cs.check_stats) ] if cs.library_versions is not None: check_lines += [" - Dependencies:"] for library, version in cs.library_versions.iteritems( ): check_lines += [
def run_export(filenames, format, chatnames, authornames, ask_password): """Exports the specified databases in specified format.""" dbs = [skypedata.SkypeDatabase(f) for f in filenames] is_xlsx_single = ("xlsx_single" == format) for db in dbs: if (ask_password and db.id and conf.SharedImageAutoDownload and format.lower().endswith("html")): prompt = "Enter Skype password for '%s': " % db.id while not skypedata.SharedImageDownload.has_login(db.id): with warnings.catch_warnings(): warnings.simplefilter("ignore") # possible GetPassWarning output(prompt, end="") # getpass output can raise errors pw = getpass.getpass("", io.BytesIO()) if not pw: continue # while try: skypedata.SharedImageDownload.login(db.id, pw) except Exception as e: log("Error signing in %s on Skype web.\n\n%s", db.id, util.format_exc(e)) prompt = "%s\nEnter Skype password for '%s': " % (e, db.id) formatargs = collections.defaultdict(str) formatargs["skypename"] = os.path.basename(db.filename) formatargs.update(db.account or {}) basename = util.safe_filename(conf.ExportDbTemplate % formatargs) dbstr = "from %s " % db if len(dbs) != 1 else "" if is_xlsx_single: export_dir = os.getcwd() filename = util.unique_path("%s.xlsx" % basename) else: export_dir = util.unique_path(os.path.join(os.getcwd(), basename)) filename = format target = filename if is_xlsx_single else export_dir try: extras = [("", chatnames)] if chatnames else [] extras += [(" with authors", authornames)] if authornames else [] output("Exporting%s%s as %s %sto %s." % (" chats" if extras else "", ",".join("%s like %s" % (x, y) for x, y in extras), format[:4].upper(), dbstr, target)) chats = sorted(db.get_conversations(chatnames, authornames), key=lambda x: x["title"].lower()) db.get_conversations_stats(chats) bar_total = sum(c["message_count"] for c in chats) bartext = " Exporting %.*s.." % (30, db.filename) # Enforce width bar = ProgressBar(max=bar_total, afterword=bartext) bar.start() result = export.export_chats(chats, export_dir, filename, db, progress=bar.update) files, count = result bar.stop() if count: bar.afterword = " Exported %s to %s. " % (db, target) bar.update(bar_total) output() log("Exported %s %sto %s as %s.", util.plural("chat", count), dbstr, target, format) else: output("\nNo messages to export%s." % ("" if len(dbs) == 1 else " from %s" % db)) os.unlink(filename) if is_xlsx_single else os.rmdir(export_dir) except Exception as e: output("Error exporting chats: %s\n\n%s" % (e, traceback.format_exc()))
def run(self): self._is_running = True while self._is_running: params = self._queue.get() self._stop_work = False self._drop_results = False if params: # {"htmls": [html result for db1, db2], # "chats": [differing chats in db1, db2]} result = { "htmls": ["", ""], "chats": [[], []], "params": params } db1, db2 = params["db1"], params["db2"] chats1 = db1.get_conversations() chats2 = db2.get_conversations() skypenames1 = [i["identity"] for i in db1.get_contacts()] skypenames2 = [i["identity"] for i in db2.get_contacts()] skypenames1.append(db1.id) skypenames2.append(db2.id) c1map = dict((c["identity"], c) for c in chats1) c2map = dict((c["identity"], c) for c in chats2) compared = [] for c1 in chats1: c2 = c2map.get(c1["identity"], None) c = c1.copy() c["messages1"] = c1["message_count"] or 0 c["messages2"] = c2["message_count"] or 0 if c2 else 0 c["c1"] = c1 c["c2"] = c2 compared.append(c) for c2 in chats2: c1 = c1map.get(c2["identity"], None) if not c1: c = c2.copy() c["messages2"] = c["message_count"] or 0 c["messages1"] = 0 c["c1"] = c1 c["c2"] = c2 compared.append(c) compared.sort(key=lambda x: x["title"]) for chat in compared: diff = self.get_chat_diff(chat, db1, db2) if self._stop_work: break # break for chat in compared for i in range(2): new_chat = not chat["c1" if i else "c2"] newstr = "" if new_chat else "new " info = export.htmltag("a", {"href": chat["identity"]}, chat["title_long"], utf=False) if new_chat: info += " - new chat" skypenames_other = [skypenames1, skypenames2][1 - i] if diff["messages"][i]: info += ", %s" % (util.plural( "%smessage" % newstr, len( diff["messages"][i]))) if diff["participants"][i] and newstr: info += ", %s" % (util.plural( "%sparticipant" % newstr, len(diff["participants"][i]))) if diff["messages"][i] or diff["participants"][i]: info += ".<br />" result["htmls"][i] += info result["chats"][i].append({ "chat": chat, "diff": diff }) if not self._drop_results \ and not len(result["chats"][i]) \ % conf.DiffResultsChunk: self._postback(result) result = { "htmls": ["", ""], "chats": [[], []], "params": params } if not self._drop_results: result["done"] = True self._postback(result)