def analyzeResult(x, accuracy, perturbAt=10000, movingAvg=True, smooth=True):
  if movingAvg:
    accuracy = movingAverage(accuracy, min(len(accuracy), 100))

  x = np.array(x)
  accuracy = np.array(accuracy)
  if smooth:
    # perform smoothing convolution
    mask = np.ones(shape=(100,))
    mask = mask/np.sum(mask)
    # extend accuracy vector to eliminate boundary effect of convolution
    accuracy = np.concatenate((accuracy, np.ones((200, ))*accuracy[-1]))
    accuracy = np.convolve(accuracy, mask, 'same')
    accuracy = accuracy[:len(x)]


  perturbAtX = np.where(x > perturbAt)[0][0]

  finalAccuracy = accuracy[perturbAtX-len(mask)/2]
  learnTime = min(np.where(np.logical_and(accuracy > finalAccuracy * 0.99,
                                          x < x[perturbAtX - len(mask)/2-1]))[0])
  learnTime = x[learnTime]

  finalAccuracyAfterPerturbation = accuracy[-1]
  learnTimeAfterPerturbation = min(np.where(
    np.logical_and(accuracy > finalAccuracyAfterPerturbation * 0.99,
                   x > x[perturbAtX + len(mask)]))[0])

  learnTimeAfterPerturbation = x[learnTimeAfterPerturbation] - perturbAt

  result = {"finalAccuracy": finalAccuracy,
            "learnTime": learnTime,
            "finalAccuracyAfterPerturbation": finalAccuracyAfterPerturbation,
            "learnTimeAfterPerturbation": learnTimeAfterPerturbation}
  return result
Exemplo n.º 2
0
def loadExperiment(experiment):
    print "Loading experiment ", experiment
    data = readExperiment(experiment)
    (accuracy, x) = computeAccuracy(data['predictions'],
                                    data['truths'],
                                    data['iterations'],
                                    resets=data['resets'],
                                    randoms=data['randoms'])
    accuracy = movingAverage(accuracy, min(len(accuracy), 100))
    return (accuracy, x)
def loadExperiment(experiment):
  print "Loading experiment ", experiment
  data = readExperiment(experiment)
  (accuracy, x) = computeAccuracy(data['predictions'],
                                  data['truths'],
                                  data['iterations'],
                                  resets=data['resets'],
                                  randoms=data['randoms'])
  accuracy = movingAverage(accuracy, min(len(accuracy), 100))
  return (accuracy, x)
Exemplo n.º 4
0
def analyzeResult(x, accuracy, perturbAt=10000, movingAvg=True, smooth=True):
    if movingAvg:
        accuracy = movingAverage(accuracy, min(len(accuracy), 100))

    x = np.array(x)
    accuracy = np.array(accuracy)
    if smooth:
        # perform smoothing convolution
        mask = np.ones(shape=(100, ))
        mask = mask / np.sum(mask)
        # extend accuracy vector to eliminate boundary effect of convolution
        accuracy = np.concatenate((accuracy, np.ones((200, )) * accuracy[-1]))
        accuracy = np.convolve(accuracy, mask, 'same')
        accuracy = accuracy[:len(x)]

    perturbAtX = np.where(x > perturbAt)[0][0]

    finalAccuracy = accuracy[perturbAtX - len(mask) / 2]
    learnTime = min(
        np.where(
            np.logical_and(accuracy > finalAccuracy * 0.99,
                           x < x[perturbAtX - len(mask) / 2 - 1]))[0])
    learnTime = x[learnTime]

    finalAccuracyAfterPerturbation = accuracy[-1]
    learnTimeAfterPerturbation = min(
        np.where(
            np.logical_and(accuracy > finalAccuracyAfterPerturbation * 0.99,
                           x > x[perturbAtX + len(mask)]))[0])

    learnTimeAfterPerturbation = x[learnTimeAfterPerturbation] - perturbAt

    result = {
        "finalAccuracy": finalAccuracy,
        "learnTime": learnTime,
        "finalAccuracyAfterPerturbation": finalAccuracyAfterPerturbation,
        "learnTimeAfterPerturbation": learnTimeAfterPerturbation
    }
    return result
    # python suite.py --experiment="high-order-distributed-random-perturbed" -d
    expResults = {}
    expResultsAnaly = {}

    # HTM
    tmResults = os.path.join("tm/results",
                             "high-order-distributed-random-perturbed")
    accuracyAll = []
    exptLabel = 'HTM'
    expResultsAnaly[exptLabel] = []
    for seed in range(20):
      experiment = os.path.join(tmResults,
                                "seed" + "{:.1f}".format(seed), "0.log")
      (accuracy, x) = loadExperiment(experiment)
      expResultsAnaly[exptLabel].append(analyzeResult(x, accuracy))
      accuracy = movingAverage(accuracy, min(len(accuracy), 100))
      accuracyAll.append(np.array(accuracy))

    (meanAccuracy, stdAccuracy) = calculateMeanStd(accuracyAll)

    expResults[exptLabel] = {
      'x': x, 'meanAccuracy': meanAccuracy, 'stdAccuracy': stdAccuracy}

    # ELM
    elmResults = os.path.join("elm/results",
                             "high-order-distributed-random-perturbed")
    accuracyAll = []
    exptLabel = 'ELM'
    expResultsAnaly[exptLabel] = []
    for seed in range(10):
      experiment = os.path.join(elmResults,
    expResults = pickle.load(input)

    # load processed experiment results and plot them
    numSequenceRequired = []
    numIterationRequired = []
    lengths = np.sort(expResults.keys())

    plt.close('all')
    plt.figure(1)
    for length in lengths:
        expResult = expResults[length]
        accuracy = expResult["accuracy"]
        numIteration = expResult["numIteration"]
        numSequences = expResult["numSequences"]

        movingData = movingAverage(accuracy, min(len(accuracy), 100))
        numSequenceRequired.append(
            numSequences[np.where(np.array(movingData) >= 0.999)[0][1]])
        numIterationRequired.append(
            numIteration[np.where(np.array(movingData) >= 0.999)[0][1]])
        plt.figure(1)
        plotAccuracy((accuracy, numSequences),
                     window=100,
                     type=type,
                     label='NoiseExperiment',
                     hideTraining=True,
                     lineSize=1.0)
        plt.xlabel('# of sequences seen')

        plt.figure(2)
        plotAccuracy((expResult["accuracyAll"], expResult["numSequencesAll"]),
  expResults = pickle.load(input)

  # load processed experiment results and plot them
  numSequenceRequired = []
  numIterationRequired = []
  lengths = np.sort(expResults.keys())

  plt.close('all')
  plt.figure(1)
  for length in lengths:
    expResult = expResults[length]
    accuracy = expResult["accuracy"]
    numIteration = expResult["numIteration"]
    numSequences = expResult["numSequences"]

    movingData = movingAverage(accuracy, min(len(accuracy), 100))
    numSequenceRequired.append(
      numSequences[np.where(np.array(movingData) >= 0.999)[0][1]])
    numIterationRequired.append(
      numIteration[np.where(np.array(movingData) >= 0.999)[0][1]])
    plt.figure(1)
    plotAccuracy((accuracy, numSequences),
                 window=100,
                 type=type,
                 label='NoiseExperiment',
                 hideTraining=True,
                 lineSize=1.0)
    plt.xlabel('# of sequences seen')

    plt.figure(2)
    plotAccuracy((expResult["accuracyAll"], expResult["numSequencesAll"]),
Exemplo n.º 8
0
        # python suite.py --experiment="high-order-distributed-random-perturbed" -d
        expResults = {}
        expResultsAnaly = {}

        # HTM
        tmResults = os.path.join("tm/results",
                                 "high-order-distributed-random-perturbed")
        accuracyAll = []
        exptLabel = 'HTM'
        expResultsAnaly[exptLabel] = []
        for seed in range(20):
            experiment = os.path.join(tmResults,
                                      "seed" + "{:.1f}".format(seed), "0.log")
            (accuracy, x) = loadExperiment(experiment)
            expResultsAnaly[exptLabel].append(analyzeResult(x, accuracy))
            accuracy = movingAverage(accuracy, min(len(accuracy), 100))
            accuracyAll.append(np.array(accuracy))

        (meanAccuracy, stdAccuracy) = calculateMeanStd(accuracyAll)

        expResults[exptLabel] = {
            'x': x,
            'meanAccuracy': meanAccuracy,
            'stdAccuracy': stdAccuracy
        }

        # ELM
        elmResults = os.path.join("elm/results",
                                  "high-order-distributed-random-perturbed")
        accuracyAll = []
        exptLabel = 'ELM'