def get_data_info(self, data_filter=None): data_filter = data_filter or {} ds_filename = self._get_ds_filename() if not os.path.exists(ds_filename): return {} data_file = HDFLockingFile(ds_filename, "r", retry_count=10, retry_wait=0.2) try: res_info = {} max_rows_org = max_rows = data_filter.get("max_rows", DEFAULT_MAX_ROWS) start_time = data_filter.get("start_time", None) end_time = data_filter.get("end_time", None) start_time_include = data_filter.get("start_time_include", True) is True should_decimate = data_filter.get("decimate", False) is True ds_time = data_file["vars/%s" % self.time_var] cur_idx = ds_time.attrs["cur_row"] res_info["ds_rows"] = cur_idx res_info["ds_size"] = len(ds_time) res_info["file_size"] = os.path.getsize(ds_filename) res_info["file_name"] = ds_filename res_info["vars"] = list(data_file["vars"]) start_row, end_row = self._get_row_interval(data_file, start_time, end_time, start_time_include) res_info["need_expand"] = self.expand_info.get("need_expand", False) if self.expand_info.get("need_expand", False): max_rows = max_rows / self.expand_info["num_steps"] # Compensate expansion res_info["should_decimate"] = should_decimate res_info["need_decimate"] = bool(should_decimate and end_row - start_row > max_rows) res_info["ts_first"] = NTP4Time.from_ntp64(ds_time.value[0].tostring()).to_unix() res_info["ts_last"] = NTP4Time.from_ntp64(ds_time.value[cur_idx - 1].tostring()).to_unix() res_info["ts_first_str"] = get_datetime_str(res_info["ts_first"] * 1000, local_time=False) res_info["ts_last_str"] = get_datetime_str(res_info["ts_last"] * 1000, local_time=False) res_info["ds_samples"] = cur_idx * self.expand_info["num_steps"] if res_info["need_expand"] else cur_idx res_info["filter_start_row"] = start_row res_info["filter_end_row"] = end_row res_info["filter_max_rows"] = max_rows res_info["filter_ts_first"] = NTP4Time.from_ntp64(ds_time.value[start_row].tostring()).to_unix() res_info["filter_ts_last"] = NTP4Time.from_ntp64(ds_time.value[end_row - 1].tostring()).to_unix() res_info["filter_ts_first_str"] = get_datetime_str(res_info["filter_ts_first"] * 1000, local_time=False) res_info["filter_ts_last_str"] = get_datetime_str(res_info["filter_ts_last"] * 1000, local_time=False) return res_info finally: data_file.close()
def _default_formatter(log_entry, **kwargs): truncate = kwargs.get("truncate", 0) color = kwargs.get("color", False) logscope = log_entry["scope"] scope_cat = logscope.split(".", 1)[0] entry_color = SCOPE_COLOR.get(logscope, None) or SCOPE_COLOR.get(scope_cat, DEFAULT_COLOR) frags = [] if color: frags.append("\033[1m\033[%sm" % entry_color) stmt_time = " [%.5f s]" % log_entry.get('statement_time') if 'statement_time' in log_entry else "" frags.append("\n%s: #%s @%s (%s)%s -> %s" % (log_entry['scope'], log_entry['seq'], log_entry['ts'], get_datetime_str(log_entry['ts'], show_millis=True), stmt_time, log_entry.get("status", "OK"))) if color: frags.append("\033[22m") # Bold off statement = log_entry.get('statement', "") if truncate: frags.append("\n" + statement[:truncate]) if len(statement) > truncate: frags.append("...") frags.append("[%s]" % (len(statement) - truncate)) else: frags.append("\n" + statement) if color: frags.append("\033[0m") if "stack" in log_entry and kwargs.get("stack", False): frags.append("\n ") frags.append("\n ".join(log_entry["stack"])) return "".join(frags)
def _default_formatter(log_entry, **kwargs): truncate = kwargs.get("truncate", 0) color = kwargs.get("color", False) logscope = log_entry["scope"] scope_cat = logscope.split(".", 1)[0] entry_color = SCOPE_COLOR.get(logscope, None) or SCOPE_COLOR.get( scope_cat, DEFAULT_COLOR) frags = [] if color: frags.append("\033[1m\033[%sm" % entry_color) stmt_time = " [%.5f s]" % log_entry.get( 'statement_time') if 'statement_time' in log_entry else "" frags.append("\n%s: #%s @%s (%s)%s -> %s" % (log_entry['scope'], log_entry['seq'], log_entry['ts'], get_datetime_str(log_entry['ts'], show_millis=True), stmt_time, log_entry.get("status", "OK"))) if color: frags.append("\033[22m") # Bold off statement = log_entry.get('statement', "") if truncate: frags.append("\n" + statement[:truncate]) if len(statement) > truncate: frags.append("...") frags.append("[%s]" % (len(statement) - truncate)) else: frags.append("\n" + statement) if color: frags.append("\033[0m") if "stack" in log_entry and kwargs.get("stack", False): frags.append("\n ") frags.append("\n ".join(log_entry["stack"])) return "".join(frags)
def _alarm_handler(self, signum, frame): if self._momento_time == self._last_switch_time: st = traceback.extract_stack(frame) from pyon.util.containers import get_ion_ts, get_datetime_str alarm_trace ='CRITICAL> gevent blocking detected at %s!\n' % get_datetime_str(get_ion_ts()) +\ 'At %s():%d of file %s.\n' % ( frame.f_code.co_name, frame.f_lineno, frame.f_code.co_filename) alarm_trace += "Stack trace: %s\n" % "".join(traceback.format_list(st)) self._last_traces.append(alarm_trace) # Optional. Let's check if it's still blocking in next iteration. signal.alarm(MAX_BLOCKING_TIME)
def get_data_info(self, data_filter=None): data_filter = data_filter or {} ds_filename = self._get_ds_filename() if not os.path.exists(ds_filename): return {} data_file = HDFLockingFile(ds_filename, "r", retry_count=10, retry_wait=0.2) try: res_info = {} max_rows_org = max_rows = data_filter.get("max_rows", DEFAULT_MAX_ROWS) start_time = data_filter.get("start_time", None) end_time = data_filter.get("end_time", None) start_time_include = data_filter.get("start_time_include", True) is True should_decimate = data_filter.get("decimate", False) is True ds_time = data_file["vars/%s" % self.time_var] cur_idx = ds_time.attrs["cur_row"] res_info["ds_rows"] = cur_idx res_info["ds_size"] = len(ds_time) res_info["file_size"] = os.path.getsize(ds_filename) res_info["file_name"] = ds_filename res_info["vars"] = list(data_file["vars"]) start_row, end_row = self._get_row_interval( data_file, start_time, end_time, start_time_include) res_info["need_expand"] = self.expand_info.get( "need_expand", False) if self.expand_info.get("need_expand", False): max_rows = max_rows / self.expand_info[ "num_steps"] # Compensate expansion res_info["should_decimate"] = should_decimate res_info["need_decimate"] = bool( should_decimate and end_row - start_row > max_rows) res_info["ts_first"] = NTP4Time.from_ntp64( ds_time.value[0].tostring()).to_unix() res_info["ts_last"] = NTP4Time.from_ntp64( ds_time.value[cur_idx - 1].tostring()).to_unix() res_info["ts_first_str"] = get_datetime_str(res_info["ts_first"] * 1000, local_time=False) res_info["ts_last_str"] = get_datetime_str(res_info["ts_last"] * 1000, local_time=False) res_info["ds_samples"] = cur_idx * self.expand_info[ "num_steps"] if res_info["need_expand"] else cur_idx res_info["filter_start_row"] = start_row res_info["filter_end_row"] = end_row res_info["filter_max_rows"] = max_rows res_info["filter_ts_first"] = NTP4Time.from_ntp64( ds_time.value[start_row].tostring()).to_unix() res_info["filter_ts_last"] = NTP4Time.from_ntp64( ds_time.value[end_row - 1].tostring()).to_unix() res_info["filter_ts_first_str"] = get_datetime_str( res_info["filter_ts_first"] * 1000, local_time=False) res_info["filter_ts_last_str"] = get_datetime_str( res_info["filter_ts_last"] * 1000, local_time=False) return res_info finally: data_file.close()
def _greenlet_blocking_monitor(self): while not self._stop_monitor: old_switch_counter = self._greenlet_switch_counter _real_sleep(MAX_BLOCKING_TIME) active_greenlet = self._active_greenlet new_switch_counter = self._greenlet_switch_counter # If we have detected a successful switch, reset the counter # to zero. This might race with it being incrememted in the # other thread, but should succeed often enough to prevent # the counter from growing without bound. if new_switch_counter != old_switch_counter: self._greenlet_switch_counter = 0 # If we detected a blocking greenlet, grab the stack trace # and log an error. The active greenlet's frame is not # available from the greenlet object itself, we have to look # up the current frame of the main thread for the traceback. else: if active_greenlet not in (None, self._active_hub): frame = sys._current_frames()[self._main_thread_id] stack = traceback.format_stack(frame) from pyon.util.containers import get_ion_ts, get_datetime_str self._last_traces.append("Greenlet appears to be blocked at %s \n" % get_datetime_str(get_ion_ts()) + "".join(stack))