def run_experiment(experiment_var, random_seed):
evals_at_targets_df = pd.DataFrame()
for i, dim in enumerate(experiment_var):
a = 1
b = -1
# random initial solution with elements between -1 and 1
theta0 = (b - a) * np.random.rand(dim + 1, 1) + a
# allow more iterations in higher dimensions
parms.max_iterations = parms.max_iterations * dim
error_list, sample_evals = ex.run_problem(
dim, sample_size, num_targets, num_subintervals, cost_function,
theta0, balance, noise, parms, random_seed)
############# benchmark optimization run
target_values = ex.create_targets(error_list, num_targets)
benchmarker = bm.Benchmark(sample_evals, target_values, error_list)
evals_at_targets = benchmarker.benchmark()
evals_at_targets_df[i] = evals_at_targets
return evals_at_targets_df
def addDcmbmkBenchmarks(gem5FusionRoot, suites, benchmarks):
suites.append('dcmbmk')
dcmbmkSEBinDir = os.path.join(gem5FusionRoot,
'benchmarks/dcmbmk-image/bin')
dcmbmkSEInpDir = os.path.join(gem5FusionRoot,
'benchmarks/dcmbmk-image/inputs')
dcmbmkFSBinDir = os.path.join('dcmbmk/bin')
dcmbmkFSInpDir = os.path.join('dcmbmk/inputs')
# Note: this can/should be a symlink and/or get passed in
dcmbmkRcSDir = os.path.join(gem5FusionRoot, 'full_system_files/runscripts')
dcmbmkCmdLines = {}
benchNames = [
'cmem', 'diverge', 'global', 'icache1', 'icache2', 'icache3',
'icache4', 'shared', 'sync', 'texture2', 'texture4'
]
for benchName in benchNames:
bench = Benchmark(suite='dcmbmk',
name=benchName,
executable='gem5_fusion_%s' % benchName,
seBinDir=dcmbmkSEBinDir,
seInpDir=os.path.join(dcmbmkSEInpDir, benchName),
fsBinDir=dcmbmkFSBinDir,
fsInpDir=os.path.join(dcmbmkFSInpDir, benchName),
rcSDir=dcmbmkRcSDir,
simSizes=['default'],
cmdLines=dcmbmkCmdLines)
benchmarks.append(bench)
def main():
# Asks user and gets all input
Company, period_to_look, period_of_change, percent_change = Ask_dates.Ask_dates(
)
# Reset the companies benchmark files or not
Reset = False
# Create new file to store Data
Create_files.Creation(Company, Reset)
# Convert period to timeframes for API
timeframe = Period_to_timeframe.Period_to_timeframe(period_to_look)
# Calls on the Google_data's function Trending_google with our variables.
# If not enough data is available, then returns a 0. (int)
Trend_data = Google_data.Trending_google(Company, timeframe)
if (Trend_data != 0):
# Get our benchmarks
Benchmark.Calc(Trend_data,
Company[0]) # Needs to be fixed for list of companies
# Alarm!
alarm.Num(Trend_data, Company[0], percent_change, period_of_change)
else:
print "\n \nERROR MESSAGE: No Trending data! \n"
print "Trend data:" + str((Trend_data))
return 1
def estimatedRender (self, idWork, idZone):
"""Realiza el render tama�o sello y escribe los datos obtenidos en la pizarra"""
blenderFile = Util.getFileName(self.getCurrentFiles(), 'blend')
ficheroBlender = os.path.join(self.getWorkDir(), blenderFile)
salida = os.path.join(self.getWorkDir(), str(idWork) + '_' + str(idZone))
zone = self.getZone(idZone)
x1, y1 = zone.x1, zone.y1
x2, y2 = zone.x2, zone.y2
# Se toma la hora para actualizar el registro en la pizarra.
begin = time.time()
# Render con el script EstimatedRender.py.
os.system('blender -b ' + ficheroBlender + ' -P ./EstimatedRender.py -f 1 ' + salida + ' ' + str(x1) + ' ' + str(y1) + ' ' + str(x2) + ' ' + str(y2) + ' ' + str(RES))
end = time.time()
os.system('mv ' + salida + '0001.png ' + salida + EXT)
# Actualizaci�n en la pizarra con el tiempo estimado en el render tama�o "sello".
self.getBlackboard().incrementEstimatedRenderTime(int(end - begin))
if len(self.getHistoric()) == 0:
self.setCoefficient(Benchmark.getValue() / self.getBenchmarkValue())
finalTime = int(RES * (end - begin) / self.getCoefficient())
self.getBlackboard().update(idWork, idZone, finalTime)
def testBenchmarkScript(self):
test_comp = sb.launch(
'HardLimit') #this is the component you want to test.
t = 5 #time for benchmark to recalcuate confidence interval in seconds
packets = 100 #num of packets in BenchmarkGen component to calculate output rate
size = 1000 #size of packets for BenchmarkGen component to generate
samples_away = 10 #samples away from the first
plotFlag = 0 #plot results?
debugFlag = 1 #show debug information
# Benchmark start
bench1 = Benchmark.Benchmark(test_comp, t, packets, size, samples_away,
plotFlag, debugFlag)
bench1.run()
def __init__(self,
Universe='Stoxx 50',
Dates=['01-01-2015', '01-01-2017'],
Frequency=1,
Histo_Length=6,
Wght_Const=[0.0, 10.0]):
self._Universe = Universe
self._Benchmark = bench.Benchmark()
self._Dates = Dates
self._Frequency = Frequency
self._Histo_Length = Histo_Length
self._Wght_Constraint = Wght_Const
self._Wght_Histo = pd.DataFrame()
self._Perf = pd.Series()
self._Weights = None
self._Opt_Strat = 'EW'
def call(funcao, list_entradas):
if funcao == 0:
return ben.rastrigin(list_entradas)[0]
elif funcao == 1:
return ben.ackley(list_entradas)[0]
elif funcao == 2:
return ben.keane(list_entradas)[0]
elif funcao == 3:
return ben.shubert(list_entradas)[0]
elif funcao == 4:
return ben.shubert3(list_entradas)[0]
elif funcao == 5:
return ben.schaffer(list_entradas)[0]
elif funcao == 6:
return ben.zdt1(list_entradas)
elif funcao == 7:
return ben.zdt2(list_entradas)
else:
return None
def test_speed_simple():
ureg = uc.getGlobalUnitRegistry()
q = ureg.makeQuantity('100 mile/hour')
pureg = pint.UnitRegistry()
pq = pureg.Quantity(100, 'mile/hour')
bm = Benchmark()
bm.run(lambda: q.to("km/s"))
uc_runtime = bm.measurement
bm.run(lambda: pq.to("km/s"))
pint_runtime = bm.measurement
print("convert: 100 mph -> km/s")
print("Pint", pint_runtime)
print("UnitConvert", uc_runtime)
print("Speedup", Speedup(pint_runtime, uc_runtime))
def test_speed_medium():
print()
ureg = uc.getGlobalUnitRegistry()
q = ureg.makeQuantity('100 W')
pureg = pint.UnitRegistry()
pq = pureg.Quantity(100, 'W')
bm = Benchmark()
bm.run(lambda: pq.to("g cm mm / hour / min / ms"))
pint_runtime = bm.measurement
bm.run(lambda: q.to("g cm mm / hour / min / ms"))
uc_runtime = bm.measurement
print("convert: 100 W -> g cm mm / hr / min / ms")
print("Pint", pint_runtime)
print("UnitConvert", uc_runtime)
print("Speedup", Speedup(pint_runtime, uc_runtime))
def test_pint_calc_with_uc_conversion():
print()
ureg = uc.getGlobalUnitRegistry()
pureg = pint.UnitRegistry()
l = pureg.Quantity(100, 'in')
w = pureg.Quantity(100, 'cm')
A = l * w
bm = Benchmark()
bm.run(lambda: A.to("acre"))
pint_runtime = bm.measurement
bm.run(lambda: pureg.Quantity(
ureg.makeQuantity(str(A)).to("acre").value(), "acre"))
uc_runtime = bm.measurement
print("pint calc with uc conversion")
print("Pint", pint_runtime)
print("UnitConvert", uc_runtime)
print("Speedup", Speedup(pint_runtime, uc_runtime))
def __init__(self):
print("LogisticRegression initializing")
# Chargement initial des données (mails)
csvValuesColumnNumber = 57
csvFilePath = "spambase/spambase.data"
mailDataset = pd.read_csv(csvFilePath, header=None) # names=names,
mailDataset.drop(columns=[
26, 27
]) # Drop columns "Georges & 650" contextual false-positives
# Split des colonnes en deux : les valeurs (dataFieldsValues) et le label pour chaque mail (dataLabels)
# permettant de savoir si c'est un spam (1) ou non
dataFieldsValues = mailDataset.iloc[:, :-1].values
# : signifie "tout" -> :-1 signifie "toutes les colonnes sauf la dernière"
dataLabels = mailDataset.iloc[:, csvValuesColumnNumber].values
# train_test_split(counts, df['label'], test_size=0.1, random_state=69)
# Split des lignes de spambase et shuffle pour avoir un échantillon aléatoire
# X_train : valeurs d'entraînement
# y_train : labels d'entraînement (associés à chaque valeur)
# X_test : valeurs pour le test
# y_test : labels pour vérifier le test
iterationNumber = 2
# Permet d'avoir des jeux de test identiques pour chaque itération
a2_X_train = []
a2_X_test = []
a2_y_train = []
a2_y_test = []
# Jeu de tests scalé
a2_X_train_scaled = []
a2_X_test_scaled = []
from sklearn.preprocessing import StandardScaler
for i in range(0, iterationNumber):
X_train, X_test, y_train, y_test = train_test_split(
dataFieldsValues, dataLabels, test_size=0.2,
shuffle=True) # test_size = 1 - train_size
a2_X_train.append(X_train)
a2_X_test.append(X_test)
a2_y_train.append(y_train)
a2_y_test.append(y_test)
scaler = StandardScaler()
scaler.fit(X_train)
X_train_scaled = scaler.transform(X_train)
X_test_scaled = scaler.transform(X_test)
a2_X_train_scaled.append(X_train_scaled)
a2_X_test_scaled.append(X_test_scaled)
predictionArrayErrorRatio = [
] # prédiction, valeurs à comparer à y_test
predictionArrayName = []
predictionArrayTimeTookMs = []
iterationNumber = len(a2_X_train)
if iterationNumber <= 0:
return
for iIteration in range(0, iterationNumber):
errorOccured = False
randSeed = int(
time.time() * 10000000000
) % 4294967295 # Modulo la valeur d'un int non signé : 2^32 - 1
print("predictWith randSeed = " + str(randSeed))
np.random.seed(randSeed)
startTimeMs = int(time.time() * 1000)
print("predictWith " + "LR")
from sklearn.linear_model import LogisticRegression
classifier = LogisticRegression(random_state=0,
solver='lbfgs',
multi_class='ovr',
max_iter=100000)
classifier.fit(a2_X_train[iIteration],
a2_y_train[iIteration]) # X_train_scaled
y_predict = classifier.predict(
a2_X_test[iIteration]) # X_test_scaled
if not errorOccured:
print(classification_report(a2_y_test[iIteration], y_predict))
elapsedTimeMs = int(time.time() * 1000) - startTimeMs
localPredictErrorRatio = np.mean(
y_predict != a2_y_test[iIteration])
predictionArrayErrorRatio.append(localPredictErrorRatio)
predictionArrayName.append("LR")
predictionArrayTimeTookMs.append(elapsedTimeMs)
predictionArrayErrorRatioScaled = [
] # prédiction, valeurs à comparer à y_test
predictionArrayNameScaled = []
predictionArrayTimeTookMsScaled = []
iterationNumber = len(a2_X_train_scaled)
if iterationNumber <= 0:
return
for iIteration in range(0, iterationNumber):
errorOccured = False
randSeed = int(
time.time() * 10000000000
) % 4294967295 # Modulo la valeur d'un int non signé : 2^32 - 1
print("predictWith randSeed = " + str(randSeed))
np.random.seed(randSeed)
startTimeMs = int(time.time() * 1000)
print("predictWith " + "LR")
from sklearn.linear_model import LogisticRegression
classifier = LogisticRegression(random_state=0,
solver='lbfgs',
multi_class='ovr',
max_iter=100000)
classifier.fit(a2_X_train_scaled[iIteration],
a2_y_train[iIteration]) # X_train_scaled
y_predict = classifier.predict(
a2_X_test_scaled[iIteration]) # X_test_scaled
if not errorOccured:
print(classification_report(a2_y_test[iIteration], y_predict))
elapsedTimeMs = int(time.time() * 1000) - startTimeMs
localPredictErrorRatio = np.mean(
y_predict != a2_y_test[iIteration])
predictionArrayErrorRatioScaled.append(localPredictErrorRatio)
predictionArrayNameScaled.append("LR")
predictionArrayTimeTookMsScaled.append(elapsedTimeMs)
Benchmark.drawBenchmarkForMultipleValues(
'Non Scalé - Taux d\'erreur en fonction de l\'algo utilisé',
'Algo utilisé', 'Erreur moyenne', predictionArrayErrorRatio,
predictionArrayName)
Benchmark.drawBenchmarkForMultipleValues(
'Scalé - Taux d\'erreur en fonction de l\'algo utilisé',
'Algo utilisé', 'Erreur moyenne', predictionArrayErrorRatioScaled,
predictionArrayNameScaled)
Benchmark.drawBenchmarkForMultipleValues(
"Non Scalé - Temps pris par algorithme", "Algo utilisé",
"Temps pris (ms)", predictionArrayTimeTookMs, predictionArrayName)
Benchmark.drawBenchmarkForMultipleValues(
"Scalé - Temps pris par algorithme", "Algo utilisé",
"Temps pris (ms)", predictionArrayTimeTookMsScaled,
predictionArrayNameScaled)
def notifyBenchmarkValue (self):
"""Notifica al Master el tiempo empleado en la ejecuci�n del benchmark"""
self.getMaster().benchmarkValue(int(Benchmark.getValue()))
## Higher order benchmarking The above solves the `CartPole-v0` env. However, we want to benchmark more envs. In addition, a single solve isn't representative, especially given the randomness inherent in many of these environments. What would give more specific info is to do the above benchmarking, but run it a number of times to create a distribution. This is easily doable with the `Evolve` class and the `Benchmark.py` functions. Briefly, an `Evolve` object creates an `Agent` class object for a given env, and then `Evolve.evolve()` does RWG to try and solve that env. It returns the solve time (in number of generations). `Benchmark.benchmark_envs()` takes a list of envs. For each, it creates a dist of the solve times, by (some specified number of times) creating a new `Evolve` object and solving the env. Here's a simple usage, from `scripts/benchmark_example.py`: ``` import path_utils import Benchmark Benchmark.benchmark_envs(['CartPole-v0'], N_dist=100, N_gen=1000) ``` This will only benchmark `CartPole-v0`. It will create a distribution of solve times from `N_dist` instances of that env. Each one will have a max number of generations `N_gen` (if it doesn't solve in that time, it gets marked as the maximum time; this might be suboptimal because it's underestimating these outliers). This produces: <p align="center"> <img width="640" height="480" src="misc/CartPole-v0_solve_gen_dist.png"> </p> Something curious: even though it seems to have a well-defined Gamma-like (?) distribution shape, there are always some at the maximum `N_gen` (meaning they didn't solve). This is curious, since every iteration of `evolve()` is independent. However, since we're just testing for `mean_score` > `best_score`, it's possible that it gets a "lucky" set of weights that got a high score for its 3 episode trials, but couldn't solve it. Then, later sets that might not get as high a 3-episode score, but *would* solve it, don't get tested. This has to be looked at more. In addition, it creates a timestamped directory in the `outputs` directory for the benchmarking run. Within that, it creates: * For each env benchmarked, a directory with the FF plot for each run
#!/opt/local/bin/python3.3 from Benchmark import * if __name__ == "__main__": b = Benchmark() print( b.validate_with_knn() ) print( b.validate_with_rf() )
theta, error_list, sample_evals = solver.gradient_descent(cost_function,
mini_batcher,
X_train,
y_train,
theta0)
print("Bias = ", theta[0])
print("Coefficients = ", theta[1:])
############# Create benchmarker #############
num_targets = 20
target_values = ex.create_targets(error_list, num_targets)
benchmarker = bm.Benchmark(sample_evals,
target_values,
error_list)
############# predicting output for test set #############
y_pred, err, majority_err, minority_err = benchmarker.test_error(cost_function,
X_test,
y_test,
theta,
verbose = True)
############# Plots #############
# plot data
num_points = 200
# plt.figure()
# mp.plot_data(cost_function, num_points, 8, dataset.data)
#!/usr/bin/env python
#
# Script to use Benchmark.py class
#
#
#
from ossie.utils import sb
import Benchmark
## Component to test
upzero_comp = sb.launch('UpZero')
tunefilter_comp = sb.launch('TuneFilterDecimate')
fastfilter_comp = sb.launch('fastfilter')
# Benchmark Parameters
test_comp = upzero_comp #this is the component you want to test.
t = 5 #time for benchmark to recalcuate confidence interval in seconds
packets = 100 #num of packets in BenchmarkGen component to calculate output rate
size = 1000 #size of packets for BenchmarkGen component to generate
samples_away = 10 #samples away from the first
plotFlag = 0 #plot results?
debugFlag = 1 #show debug information
# Benchmark start
bench1 = Benchmark.Benchmark(test_comp, t, packets, size, samples_away,
plotFlag, debugFlag)
bench1.run()
print 'script done'
import path_utils
import Benchmark
import os
'''
For testing various benchmarking examples.
'''
#['CartPole-v0', 'MountainCar-v0', 'MountainCarContinuous-v0', 'Pendulum-v0']
Benchmark.benchmark_envs([
'CartPole-v0', 'MountainCar-v0', 'MountainCarContinuous-v0', 'Pendulum-v0'
],
N_gen=2000,
N_dist=25,
NN='FFNN_multilayer',
N_hidden_layers=0,
N_hidden_units=0,
act_fn='linear')
exit()
Benchmark.benchmark_vary_params(
{
'env_name': 'MountainCar-v0',
'NN': 'FFNN_multilayer'
}, {
'N_hidden_units': [2, 4],
'N_hidden_layers': [0, 1],
'act_fn': ['tanh', 'relu']
},
N_gen=5,
