def __init__(self, psutil, rpyc=None): self._psutil = psutil self.rpyc = rpyc self._conn = None self.log = utils.get_logger(__name__) # required to define the columns for the data frame storer self.fields = [ 'call_index', 'total_cpu_percent', 'percent_cpu_sys', 'percent_cpu_usr', 'percent_cpu_idle', 'percent_cpu_iow', 'phymem_percent_usage', 'load_avg', ] # this call should ensure we have the correct type self._times_tup_type = psutil.cpu_times().__class__ self.log = utils.get_logger(type(self).__name__) # initial cpu usage self._last_cpu_times = self.psutil.cpu_times()
def _consume_and_write(queue, path, store, sharr): """Insert :var:`row` received from the queue into the shared memory array at the current index and increment. Empty rows are always written to disk (keeps stores 'call-index-aligned'). """ proc = mp.current_process() slog = utils.get_logger(proc.name) log = mp.log_to_stderr(slog.getEffectiveLevel()) log.debug("starting storage writer '{}'".format(proc.name)) log.info("storage path is '{}'".format(path)) log.debug("sharr is '{}'".format(sharr)) # set up a new store instance for writing with store.writer(path, dtypes=store.dtypes) as store: # notify parent that file has been created queue.put(path) # handle no pandas/np case buff = store if sharr is None else sharr bufftype = type(buff) log.debug('buffer type is {}'.format(bufftype)) for row in iter(queue.get, Terminate): # consume and process now = time.time() # write frame to disk on buffer fill if sharr and sharr.is_full(): log.debug('writing to {} storage...'.format(store.ext)) try: # push a data frame store.put(pd.DataFrame.from_records(buff.read())) except ValueError: log.error(traceback.format_exc()) log.debug("storage put took '{}'".format(time.time() - now)) try: # push to ring buffer (or store if no pd) buff.put(row) log.debug("{} insert took '{}'".format(bufftype, time.time() - now)) except ValueError: log.error(traceback.format_exc()) log.debug("terminating frame writer '{}'".format(proc.name))
def __init__(self, name, dtype, buf_size=2**10, path=None, storetype=None): self.name = name try: self.dtype = pd.np.dtype(dtype) if pd else dtype except TypeError: # set all columns to float64 self.dtype = pd.np.dtype( list(zip(dtype, itertools.repeat('float64')))) self.log = utils.get_logger(type(self).__name__) # allocated a shared mem np structured array self._buf_size = buf_size # purely for testing self._buffer = RingBuffer(dtype=self.dtype, size=buf_size) if pd else None # parent proc read-only access to disk store self.storetype = storetype or CSVStore self._storepath = path or tmpfile(self.storetype.ext) self.store = self.storetype(self._storepath, dtypes=self.dtype) self.queue = mp.Queue() self._iput = 0 # queue put counter # disable SIGINT while we spawn signal.signal(signal.SIGINT, signal.SIG_IGN) # setup bg writer self._writer = mp.Process( target=_consume_and_write, args=(self.queue, self._storepath, self.store, self._buffer), name='{}_frame_writer'.format(self.name), ) self._writer.start() # re-enable SIGINT signal.signal(signal.SIGINT, signal.default_int_handler) # kill subproc on exit atexit.register(self.stopwriter) # ensure writer is initialized path = self.queue.get(timeout=3) assert path == self._storepath
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import os import pickle from functools import partial from collections import OrderedDict, namedtuple from switchio import utils from .storage import * # re-export(s) from .cdr import CDR log = utils.get_logger(__name__) def plot_df(df, figspec, **kwargs): """Plot a pandas data frame according to the provided `figspec` """ from .mpl_helpers import multiplot return multiplot(df, figspec=figspec, **kwargs) Measurer = namedtuple("Measurer", 'app ppkwargs storer ops') class Measurers(object): """A dict-like collection of measurement apps with sub-references to each app's `DataStorer` and optional metrics computing callables.
def __init__(self): self.log = utils.get_logger(__name__) self._call_counter = itertools.count(0)