Esempio n. 1
0
def test():
    """ test """
    indata = [0.1, 0.2, 0.3, 0.4, 0.4, 0.5, 0.1, 0.4]

    stats = RollingStats()
    for i in indata:
        stats.append(i)

    print stats.get()
    print calculate_stats(indata)
Esempio n. 2
0
def get_chi2(tasks, bad_users):
  """Perform chi^2 test on the stats from each worker"""
  results = {'chi2': 0.0, 'dof': 0, 'p': 0.0, 'residual': {}}

  # Aggregate results by worker
  users = {}
  for task in tasks:
    task['worker_key'] = get_worker_key(task)
    if 'worker_info' not in task:
      continue
    key = get_worker_key(task)
    if key in bad_users:
      continue
    stats = task.get('stats', {})
    wld = [float(stats.get('wins', 0)), float(stats.get('losses', 0)), float(stats.get('draws', 0))]
    if wld == [0.0, 0.0, 0.0]:
      continue
    if key in users:
      for idx in range(len(wld)):
        users[key][idx] += wld[idx]
    else:
      users[key] = wld

  if len(users) == 0:
    return results

  observed = numpy.array(users.values())
  rows,columns = observed.shape
  df = (rows - 1) * (columns - 1)
  column_sums = numpy.sum(observed, axis=0)
  row_sums = numpy.sum(observed, axis=1)
  grand_total = numpy.sum(column_sums)
  if grand_total == 0:
    return results

  expected = numpy.outer(row_sums, column_sums) / grand_total
  diff = observed - expected
  adj = numpy.outer((1 - row_sums / grand_total), (1 - column_sums / grand_total))
  residual = diff / numpy.sqrt(expected * adj)
  for idx in range(len(users)):
    users[users.keys()[idx]] = numpy.max(numpy.abs(residual[idx]))
  chi2 = numpy.sum(diff * diff / expected)
  return {
    'chi2': chi2,
    'dof': df,
    'p': 1 - scipy.stats.chi2.cdf(chi2, df),
    'residual': users,
  }
Esempio n. 3
0
def calculate_residuals(run):
  bad_users = set()
  chi2 = get_chi2(run['tasks'], bad_users)
  residuals = chi2['residual']

  # Limit bad users to 1 for now
  for _ in range(1):
    worst_user = {}
    for task in run['tasks']:
      if task['worker_key'] in bad_users:
        continue
      task['residual'] = residuals.get(task['worker_key'], 0.0)

      # Special case crashes or time losses
      stats = task.get('stats', {})
      crashes = stats.get('crashes', 0)
      time_losses = stats.get('time_losses', 0)
      if crashes + time_losses > 3:
        task['residual'] = 8.0

      if abs(task['residual']) < 2.0:
        task['residual_color'] = '#44EB44'
      elif abs(task['residual']) < 2.7:
        task['residual_color'] = 'yellow'
      else:
        task['residual_color'] = '#FF6A6A'

      if chi2['p'] < 0.01 or task['residual'] > 7.0:
        if len(worst_user) == 0 or task['residual'] > worst_user['residual']:
          worst_user['worker_key'] = task['worker_key']
          worst_user['residual'] = task['residual']

    if len(worst_user) == 0:
      break
    bad_users.add(worst_user['worker_key'])
    residuals = get_chi2(run['tasks'], bad_users)['residual']

  chi2['bad_users'] = bad_users
  return chi2
Esempio n. 4
0
def get_chi2(tasks, bad_users):
    """ Perform chi^2 test on the stats from each worker """
    results = {"chi2": 0.0, "dof": 0, "p": 0.0, "residual": {}}

    # Aggregate results by worker
    users = {}
    for task in tasks:
        task["worker_key"] = get_worker_key(task)
        if "worker_info" not in task:
            continue
        key = get_worker_key(task)
        if key in bad_users:
            continue
        stats = task.get("stats", {})
        wld = [
            float(stats.get("wins", 0)),
            float(stats.get("losses", 0)),
            float(stats.get("draws", 0)),
        ]
        if wld == [0.0, 0.0, 0.0]:
            continue
        if key in users:
            for idx in range(len(wld)):
                users[key][idx] += wld[idx]
        else:
            users[key] = wld

    if len(users) == 0:
        return results

    observed = numpy.array(list(users.values()))
    rows, columns = observed.shape
    # Results only from one worker: skip the test for workers homogeneity
    if rows == 1:
        return {"chi2": float("nan"), "dof": 0, "p": float("nan"), "residual": {}}
    column_sums = numpy.sum(observed, axis=0)
    columns_not_zero = sum(i > 0 for i in column_sums)
    df = (rows - 1) * (columns - 1)

    if columns_not_zero == 0:
        return results
    # Results only of one type: workers are identical wrt the test
    elif columns_not_zero == 1:
        results = {"chi2": 0.0, "dof": df, "p": 1.0, "residual": {}}
        return results
    # Results only of two types: workers are identical wrt the missing result type
    # Change the data shape to avoid divide by zero
    elif columns_not_zero == 2:
        idx = numpy.argwhere(numpy.all(observed[..., :] == 0, axis=0))
        observed = numpy.delete(observed, idx, axis=1)
        column_sums = numpy.sum(observed, axis=0)

    row_sums = numpy.sum(observed, axis=1)
    grand_total = numpy.sum(column_sums)

    expected = numpy.outer(row_sums, column_sums) / grand_total
    raw_residual = observed - expected
    std_error = numpy.sqrt(
        expected
        * numpy.outer((1 - row_sums / grand_total), (1 - column_sums / grand_total))
    )
    adj_residual = raw_residual / std_error
    for idx in range(len(users)):
        users[list(users.keys())[idx]] = numpy.max(numpy.abs(adj_residual[idx]))
    chi2 = numpy.sum(raw_residual * raw_residual / expected)
    return {
        "chi2": chi2,
        "dof": df,
        "p": 1 - scipy.stats.chi2.cdf(chi2, df),
        "residual": users,
    }
Esempio n. 5
0
def get_chi2(tasks, bad_users):
    """Perform chi^2 test on the stats from each worker"""
    results = {"chi2": 0.0, "dof": 0, "p": 0.0, "residual": {}}

    # Aggregate results by worker
    users = {}
    has_pentanomial = None
    for task in tasks:
        task["worker_key"] = get_worker_key(task)
        if "worker_info" not in task:
            continue
        key = get_worker_key(task)
        if key in bad_users:
            continue
        stats = task.get("stats", {})
        if has_pentanomial is None:
            has_pentanomial = "pentanomial" in stats
        if not has_pentanomial:
            wld = [
                float(stats.get("wins", 0)),
                float(stats.get("losses", 0)),
                float(stats.get("draws", 0)),
            ]
        else:
            p = stats["pentanomial"]
            # The ww and ll frequencies will typically be too small for
            # the full pentanomial chi2 test to be valid. See e.g. the last page of
            # https://www.open.ac.uk/socialsciences/spsstutorial/files/tutorials/chi-square.pdf.
            # So we combine the ww and ll frequencies with the wd and ld frequencies.
            wld = [float(p[4] + p[3]), float(p[0] + p[1]), float(p[2])]
        if wld == [0.0, 0.0, 0.0]:
            continue
        if key in users:
            for idx in range(len(wld)):
                users[key][idx] += wld[idx]
        else:
            users[key] = wld

    if len(users) == 0:
        return results

    observed = numpy.array(list(users.values()))
    rows, columns = observed.shape
    # Results only from one worker: skip the test for workers homogeneity
    if rows == 1:
        return {
            "chi2": float("nan"),
            "dof": 0,
            "p": float("nan"),
            "residual": {}
        }
    column_sums = numpy.sum(observed, axis=0)
    columns_not_zero = numpy.count_nonzero(column_sums)
    df = (rows - 1) * (columns - 1)

    if columns_not_zero == 0:
        return results
    # Results only of one type: workers are homogeneous
    elif columns_not_zero == 1:
        return {"chi2": 0.0, "dof": df, "p": 1.0, "residual": {}}
    # Results only of two types: drop the column of zeros to avoid divide by zero
    elif columns_not_zero == 2:
        observed = observed[:, ~numpy.all(observed == 0, axis=0)]
        column_sums = numpy.sum(observed, axis=0)

    row_sums = numpy.sum(observed, axis=1)
    grand_total = numpy.sum(column_sums)

    expected = numpy.outer(row_sums, column_sums) / grand_total
    raw_residual = observed - expected
    std_error = numpy.sqrt(expected * numpy.outer(
        (1 - row_sums / grand_total), (1 - column_sums / grand_total)))
    adj_residual = raw_residual / std_error
    for idx in range(len(users)):
        users[list(users.keys())[idx]] = numpy.max(numpy.abs(
            adj_residual[idx]))
    chi2 = numpy.sum(raw_residual * raw_residual / expected)
    return {
        "chi2": chi2,
        "dof": df,
        "p": 1 - scipy.stats.chi2.cdf(chi2, df),
        "residual": users,
    }