Пример #1
0
    def test_from_json(self):
        item = {
            'usage_perc_all_services': 1.22,
            'all_services_time': 4360,
            'time_perc_all_services': 17.64,
            'mean_trend':
            '0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,769,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0',
            'min_resp_time': 769.0,
            'service_name': 'zato.stats.summary.create-summary-by-year',
            'max_resp_time': 769.0,
            'rate': 0.0,
            'mean_all_services': '63',
            'all_services_usage': 82,
            'time': 769.0,
            'usage': 1,
            'usage_trend':
            '0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0',
            'mean': 12.61
        }

        stats_elem = StatsElem.from_json(item)

        for k, v in item.items():
            value = getattr(stats_elem, k)
            eq_(v, value)
Пример #2
0
 def test_from_json(self):
     item = {'usage_perc_all_services': 1.22, 'all_services_time': 4360, 
      'time_perc_all_services': 17.64, 
      'mean_trend': '0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,769,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0', 
      'min_resp_time': 769.0, 'service_name': 'zato.stats.summary.create-summary-by-year', 
      'max_resp_time': 769.0, 'rate': 0.0, 'mean_all_services': '63', 
      'all_services_usage': 82, 'time': 769.0, 'usage': 1, 
      'usage_trend': '0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0', 
      'mean': 12.61}
     
     stats_elem = StatsElem.from_json(item)
     
     for k, v in item.items():
         value = getattr(stats_elem, k)
         eq_(v, value)
Пример #3
0
def _get_stats(client, start, stop, n, n_type, stats_type=None):
    """ Returns at most n statistics elements of a given n_type for the period
    between start and stop.
    """
    out = []
    input_dict = {'start':start, 'n':n, 'n_type':n_type}

    if stop:
        input_dict['stop'] = stop

    if stats_type == 'trends':
        service_name = 'zato.stats.trends.get-trends'
    else:
        service_name = 'zato.stats.summary.get-summary-by-range'

    response = client.invoke(service_name, input_dict)

    if response.has_data:
        for item in response.data:
            out.append(StatsElem.from_json(item))

    return out
Пример #4
0
def _get_stats(client, start, stop, n, n_type, stats_type=None):
    """ Returns at most n statistics elements of a given n_type for the period
    between start and stop.
    """
    out = []
    input_dict = {'start': start, 'n': n, 'n_type': n_type}

    if stop:
        input_dict['stop'] = stop

    if stats_type == 'trends':
        service_name = 'zato.stats.trends.get-trends'
    else:
        service_name = 'zato.stats.summary.get-summary-by-range'

    response = client.invoke(service_name, input_dict)

    if response.has_data:
        for item in response.data:
            out.append(StatsElem.from_json(item))

    return out
Пример #5
0
def _get_stats(cluster, start, stop, n, n_type, stats_type=None):
    """ Returns at most n statistics elements of a given n_type for the period
    between start and stop.
    """
    out = []
    input_dict = {'start':start, 'n':n, 'n_type':n_type}
    
    if stop:
        input_dict['stop'] = stop
        
    if stats_type == 'trends':
        service_name = 'zato:stats.get-trends'
    else:
        service_name = 'zato:stats.get-summary-by-range'
    
    zato_message, _  = invoke_admin_service(cluster, service_name, input_dict)
    
    if zato_path('response.item_list.item').get_from(zato_message) is not None:
        for msg_item in zato_message.response.item_list.item:
            out.append(StatsElem.from_xml(msg_item))
            
    return out
Пример #6
0
    def get_stats(self,
                  start,
                  stop,
                  service='*',
                  n=None,
                  n_type=None,
                  needs_trends=True,
                  stats_key_prefix=None,
                  suffixes=None):
        """ Returns statistics for a given interval, as defined by 'start' and 'stop'.
        service default to '*' for all services in that period and may be set to return
        a one-element list of information regarding that particular service. Setting 'n'
        to a positive integer will make it return only top n services.
        """
        if not stats_key_prefix:
            stats_key_prefix = self.stats_key_prefix

        stats_elems = {}
        all_services_stats = Bunch({'usage': 0, 'time': 0})

        # All mean values
        mean_all_services_list = []

        # A mean value of all the mean values (mean_all_services_list)
        mean_all_services = 0

        start = parse(start)
        stop = parse(stop)
        delta = (stop - start)

        if hasattr(delta, 'total_seconds'):
            delta_seconds = delta.total_seconds()
        else:
            delta_seconds = delta.seconds

        if not suffixes:
            suffixes = self.get_suffixes(start, stop)

        # We make several passes. First two passes are made over Redis keys, one gathers the services, if any at all,
        # and another one actually collects statistics for each service found. Next pass, a partly optional one,
        # computes trends for mean response time and service usage. Another one computes each of the service's
        # average rate and updates other attributes basing on values collected in the previous step.
        # Optionally, the last one will pick only top n elements of a given type (top mean response time
        # or top usage).

        # 1st pass
        for suffix in suffixes:
            keys = self.server.kvdb.conn.keys('{}{}:{}'.format(
                stats_key_prefix, service, suffix))
            for key in keys:
                service_name = key.replace(stats_key_prefix,
                                           '').replace(':{}'.format(suffix),
                                                       '')

                stats_elem = StatsElem(service_name)
                stats_elems[service_name] = stats_elem

                # When building statistics, we can't expect there will be data for all the time
                # elems built above so to guard against it, this is a dictionary whose keys are the
                # said elems and values are mean/usage for each elem. The values will remain
                # 0/0.0 if there is no data for the time elem, which may mean that in this
                # particular time slice the service wasn't invoked at all.
                stats_elem.expected_time_elems = OrderedDict(
                    (elem, Bunch({
                        'mean': 0,
                        'usage': 0.0
                    })) for elem in suffixes)

        # 2nd pass
        for service, stats_elem in stats_elems.items():
            for suffix in suffixes:
                key = '{}{}:{}'.format(stats_key_prefix, service, suffix)

                # We can convert all the values to floats here to ease with computing
                # all the stuff and convert them still to integers later on, when necessary.
                key_values = Bunch((
                    (name, float(value))
                    for (name,
                         value) in self.server.kvdb.conn.hgetall(key).items()))

                if key_values:

                    time = (key_values.usage * key_values.mean)
                    stats_elem.time += time

                    mean_all_services_list.append(key_values.mean)
                    all_services_stats.time += time
                    all_services_stats.usage += key_values.usage

                    stats_elem.min_resp_time = min(stats_elem.min_resp_time,
                                                   key_values.min)
                    stats_elem.max_resp_time = max(stats_elem.max_resp_time,
                                                   key_values.max)

                    for attr in ('mean', 'usage'):
                        stats_elem.expected_time_elems[suffix][
                            attr] = key_values[attr]

        mean_all_services = '{:.0f}'.format(
            sp_stats.tmean(
                mean_all_services_list)) if mean_all_services_list else 0

        # 3rd pass (partly optional)
        for stats_elem in stats_elems.values():

            stats_elem.mean_all_services = mean_all_services
            stats_elem.all_services_time = int(all_services_stats.time)
            stats_elem.all_services_usage = int(all_services_stats.usage)

            values = stats_elem.expected_time_elems.values()

            stats_elem.mean_trend_int = [int(elem.mean) for elem in values]
            stats_elem.usage_trend_int = [int(elem.usage) for elem in values]

            stats_elem.mean = float('{:.2f}'.format(
                sp_stats.tmean(stats_elem.mean_trend_int)))
            stats_elem.usage = sum(stats_elem.usage_trend_int)
            stats_elem.rate = float('{:.2f}'.format(
                sum(stats_elem.usage_trend_int) / delta_seconds))

            self.set_percent_of_all_services(all_services_stats, stats_elem)

            if needs_trends:
                stats_elem.mean_trend = ','.join(
                    str(elem) for elem in stats_elem.mean_trend_int)
                stats_elem.usage_trend = ','.join(
                    str(elem) for elem in stats_elem.usage_trend_int)

        # 4th pass (optional)
        if n:
            for stats_elem in self.yield_top_n(n, n_type, stats_elems):
                yield stats_elem

        else:
            for stats_elem in stats_elems.values():
                yield stats_elem
Пример #7
0
    def merge_slices(self, slices, n=None, n_type=None):
        """ Merges a list of stats slices into a single aggregated elem.
        """
        all_services_stats = Bunch({'usage': 0, 'time': 0, 'mean': 0})
        total_seconds = 0.0

        merged_stats_elems = {}

        merged_template = {
            'usage_perc_all_services': 0.0,
            'all_services_time': 0,
            'time_perc_all_services': 0.0,
            'min_resp_time': 0.0,
            'service_name': None,
            'max_resp_time': 0.0,
            'rate': 0.0,
            'mean_all_services': 0.0,
            'all_services_usage': 0,
            'time': 0,
            'usage': 0,
            'mean': 0.0
        }

        for slice in slices:

            seen_repeated_stats = False
            total_seconds += slice.total_seconds

            if slice.stats:

                for stats_elem in slice.stats:
                    # Each slice has a list of per-service stats. Each of the statistics
                    # has certain data repeated hence it's required we pick this data
                    # once only.
                    if not seen_repeated_stats:
                        all_services_stats.time += stats_elem.all_services_time
                        all_services_stats.usage += stats_elem.all_services_usage
                        seen_repeated_stats = True

                    # Fetch an existing elem or assign a new one
                    merged_stats_elem = merged_stats_elems.setdefault(
                        stats_elem.service_name,
                        StatsElem(stats_elem.service_name))

                    # Total time spent by this service and its total usage
                    merged_stats_elem.time += stats_elem.time
                    merged_stats_elem.usage += stats_elem.usage
                    all_services_stats.mean += stats_elem.mean

                    # Minimum and maximum execution time
                    merged_stats_elem.min_resp_time = min(
                        merged_stats_elem.min_resp_time,
                        stats_elem.min_resp_time)
                    merged_stats_elem.max_resp_time = max(
                        merged_stats_elem.max_resp_time,
                        stats_elem.max_resp_time)

                    # Temporary data, will be divided by total_seconds later on,
                    # after collecting all the stats.
                    merged_stats_elem.temp_rate += slice.total_seconds * stats_elem.rate

                    # Temporary data, aggregated later on
                    merged_stats_elem.temp_mean += stats_elem.mean
                    merged_stats_elem.temp_mean_count += 1

        if merged_stats_elems:
            mean_all_services = all_services_stats.mean / len(
                merged_stats_elems)
        else:
            mean_all_services = 0

        for value in merged_stats_elems.values():

            value.all_services_time = all_services_stats.time
            value.all_services_usage = all_services_stats.usage
            value.time = round(value.time, 1)

            if total_seconds:
                value.rate = round(value.temp_rate / total_seconds, 1)
                value.rate = round(value.temp_rate / total_seconds, 1)
                value.mean_all_services = mean_all_services

                if value.temp_mean:
                    value.mean = round(value.temp_mean / value.temp_mean_count)

                self.set_percent_of_all_services(all_services_stats, value)

        if n:
            for stats_elem in self.yield_top_n(int(n), n_type,
                                               merged_stats_elems):
                yield stats_elem
        else:
            for stats_elem in merged_stats_elems.values():
                yield stats_elem
Пример #8
0
    def get_stats(self, start, stop, service='*', n=None, n_type=None, needs_trends=True, 
            stats_key_prefix=None, suffixes=None):
        """ Returns statistics for a given interval, as defined by 'start' and 'stop'.
        service default to '*' for all services in that period and may be set to return
        a one-element list of information regarding that particular service. Setting 'n' 
        to a positive integer will make it return only top n services.
        """
        if not stats_key_prefix:
            stats_key_prefix = self.stats_key_prefix
            
        stats_elems = {}
        all_services_stats = Bunch({'usage':0, 'time':0})
    
        # All mean values
        mean_all_services_list = []
        
        # A mean value of all the mean values (mean_all_services_list)
        mean_all_services = 0
    
        start = parse(start)
        stop = parse(stop)
        delta = (stop - start)
        
        if hasattr(delta, 'total_seconds'):
            delta_seconds = delta.total_seconds()
        else:
            delta_seconds = delta.seconds
        
        if not suffixes:
            suffixes = self.get_suffixes(start, stop)
        
        # We make several passes. First two passes are made over Redis keys, one gathers the services, if any at all,
        # and another one actually collects statistics for each service found. Next pass, a partly optional one,
        # computes trends for mean response time and service usage. Another one computes each of the service's
        # average rate and updates other attributes basing on values collected in the previous step.
        # Optionally, the last one will pick only top n elements of a given type (top mean response time
        # or top usage).
        
        # 1st pass
        for suffix in suffixes:
            keys = self.server.kvdb.conn.keys('{}{}:{}'.format(stats_key_prefix, service, suffix))
            for key in keys:
                service_name = key.replace(stats_key_prefix, '').replace(':{}'.format(suffix), '')
            
                stats_elem = StatsElem(service_name)
                stats_elems[service_name] = stats_elem
                
                # When building statistics, we can't expect there will be data for all the time
                # elems built above so to guard against it, this is a dictionary whose keys are the
                # said elems and values are mean/usage for each elem. The values will remain
                # 0/0.0 if there is no data for the time elem, which may mean that in this
                # particular time slice the service wasn't invoked at all.
                stats_elem.expected_time_elems = OrderedDict(
                    (elem, Bunch({'mean':0, 'usage':0.0})) for elem in suffixes)
                
        # 2nd pass
        for service, stats_elem in stats_elems.items():
            for suffix in suffixes:
                key = '{}{}:{}'.format(stats_key_prefix, service, suffix)
                
                # We can convert all the values to floats here to ease with computing
                # all the stuff and convert them still to integers later on, when necessary.
                key_values = Bunch(
                    ((name, float(value)) for (name, value) in self.server.kvdb.conn.hgetall(key).items()))
                    
                if key_values:
    
                    time = (key_values.usage * key_values.mean)
                    stats_elem.time += time
                    
                    mean_all_services_list.append(key_values.mean)
                    all_services_stats.time += time
                    all_services_stats.usage += key_values.usage
                    
                    stats_elem.min_resp_time = min(stats_elem.min_resp_time, key_values.min)
                    stats_elem.max_resp_time = max(stats_elem.max_resp_time, key_values.max)
                    
                    for attr in('mean', 'usage'):
                        stats_elem.expected_time_elems[suffix][attr] = key_values[attr]
                        
        mean_all_services = '{:.0f}'.format(sp_stats.tmean(mean_all_services_list)) if mean_all_services_list else 0
                        
        # 3rd pass (partly optional)
        for stats_elem in stats_elems.values():
            
            stats_elem.mean_all_services = mean_all_services
            stats_elem.all_services_time = int(all_services_stats.time)
            stats_elem.all_services_usage = int(all_services_stats.usage)
            
            values = stats_elem.expected_time_elems.values()
            
            stats_elem.mean_trend_int = [int(elem.mean) for elem in values]            
            stats_elem.usage_trend_int = [int(elem.usage) for elem in values]
            
            stats_elem.mean = float('{:.2f}'.format(sp_stats.tmean(stats_elem.mean_trend_int)))
            stats_elem.usage = sum(stats_elem.usage_trend_int)
            stats_elem.rate = float('{:.2f}'.format(sum(stats_elem.usage_trend_int) / delta_seconds))
            
            self.set_percent_of_all_services(all_services_stats, stats_elem)

            if needs_trends:
                stats_elem.mean_trend = ','.join(str(elem) for elem in stats_elem.mean_trend_int)
                stats_elem.usage_trend = ','.join(str(elem) for elem in stats_elem.usage_trend_int)
                
        # 4th pass (optional)
        if n:
            for stats_elem in self.yield_top_n(n, n_type, stats_elems):
                yield stats_elem

        else:
            for stats_elem in stats_elems.values():
                yield stats_elem