async def _on_telemetry(self, sid, data):
# Visualize data received data
print('Telemetry received: ', data)
# Communication
return_msg = dict()
if data is not None:
meas_list = str.split(data['sensor_measurement'])
# Parse data accordingly to sensor type
if meas_list[0] == "L":
measurement = Measurement(
sensor=Sensor.LIDAR,
data=np.array([float(x) for x in meas_list[1:3]]),
ts_us=int(meas_list[3]))
groundtruth = [float(x) for x in meas_list[4:8]]
elif meas_list[0] == "R":
measurement = Measurement(
sensor=Sensor.RADAR,
data=np.array([float(x) for x in meas_list[1:4]]),
ts_us=int(meas_list[4]))
groundtruth = [float(x) for x in meas_list[5:9]]
else:
raise ValueError('Not correct sensor type')
self.ekf.process_measurement(measurement)
estimation = self.ekf.get_estimation()
# Stack history of estimation vs groundtruth
self.estimations_history.append(estimation)
self.groundtruth_history.append(groundtruth)
# Calculate RMSE
estimation_rmse = rmse(self.estimations_history,
self.groundtruth_history)
# Create message
return_msg['estimate_x'] = float(estimation[0])
return_msg['estimate_y'] = float(estimation[1])
return_msg['rmse_x'] = estimation_rmse[0]
return_msg['rmse_y'] = estimation_rmse[1]
return_msg['rmse_vx'] = estimation_rmse[2]
return_msg['rmse_vy'] = estimation_rmse[3]
# Message sending
print("Sending data: ", return_msg)
await self._sio.emit('estimate_marker', return_msg)
else:
# If data is not valid, send dummy response
await self._sio.emit('manual', {})
def __init__(self):
self.tracker = TrackerManager()
self.measurement = Measurement()
self.headmovement_trigger_counter = 0
self.headmovement_ref_position = [0, 0, 1]
self.auto_trigger_by_headmovement = False
self._timer = QtCore.QTimer()
self._timer.timeout.connect(self.callback_thread)
self.timer_interval_ms = 20
self._timer.start(20)
self.measurement_running_flag = False
self.measurement_position = []
self.measurement_valid = False
self.measurement_history = np.array([])
self.measurement_trigger = False
self.reference_measurement_trigger = False
self.measurement_done_lock = threading.Lock()
self.gui_handle = []
self.measurements = np.array([])
self.raw_signals = np.array([])
self.raw_feedbackloop = np.array([])
self.measurements_reference = np.array([])
self.raw_signals_reference = np.array([])
self.raw_feedbackloop_reference = np.array([])
self.positions = np.array([])
self.positions_list = MeasurementListModel()
self.hp_irs = np.array([])
self.raw_signals_hp = np.array([])
self.raw_feedbackloop_hp = np.array([])
self.numHPMeasurements = 0
self.numMeasurements = 0
self.guidance_running = False
self.recommended_points = {}
self.point_recommender = pointrecommender.PointRecommender(
self.tracker)
#self.point_recommender.get_head_rotation_to_point(260, 40)
today = date.today()
self.current_date = today.strftime("%d_%m_%Y")
self.reproduction_mode = False
self.reproduction_running = False
self.reproduction_player = None
self.send_osc_data = False
self.osc_send_ip = '127.0.0.1'
self.osc_send_port = 1337
self.osc_send_address = '/guided_hrtfs/angle'
self.osc_send_client = None
def runner__get_new_state_of_world(self, identifier):
#
# VARIABLES
#
from environment import Environment
from measurement import Measurement
from runner import Runner
#
# CODE
#
# we need the environment
environment_directory = 'environments/'
environment = Environment(identifier)
environment.read_environment_file("./" + environment_directory +
identifier + ".xml")
environment.initialize()
# and we need to create an exogenous network
environment.network.create_exogenous_network(
environment.parameter.network_density)
# and also a measurement
measurement = Measurement()
runner = Runner(identifier, environment, measurement)
print runner
print
runner.get_new_state_of_world(1)
print runner
runner.get_new_state_of_world(0)
print runner
runner.get_new_state_of_world(-1)
print runner
def tune_with_TPE(dp, xgb_model, opt):
"""
Hyperparameter tuning using TPE:
https://github.com/hyperopt/hyperopt/wiki/FMin
for XGBOOST params doc see:
https://github.com/dmlc/xgboost/blob/master/doc/parameter.md
"""
# add xgb_model,measurer
X_train, X_test, y_train, y_test = get_data(dp)
xgb_model.set_data((X_train, X_test, y_train, y_test))
n2p_ratio = len(y_train[y_train == 0]) / len(y_train[y_train == 1])
# set optimizer parameters
opt.set_max_eval(2)
# # add params to handle cv task
# skf = StratifiedKFold(n_splits=3,shuffle=True,random_state=100)
# X = np.concatenate([X_train,X_test])
# y = np.concatenate([y_train,y_test])
# xgb_model.set_data(xgb.DMatrix(X,label=y))
# define search space
search_space = {"n_estimators": hp.quniform("n_estimators",100,1500,50),
"eta": hp.quniform("eta",0.025,0.1,0.025),
"max_depth": hp.choice("max_depth",[3,4]),
"min_child_weight": hp.quniform('min_child_weight',1,4,1),
"subsample": hp.quniform("subsample",0.9,1.0,0.1),
"colsample_bytree": hp.quniform("colsample_bytree",0.8,1.0,0.1),
"gamma": hp.quniform("gamma",0,0.2,0.05),
"scale_pos_weight": hp.quniform("scale_pos_weight",\
n2p_ratio/10,n2p_ratio,n2p_ratio/10),
# "alpha": hp.choice("alpha",[1e-5,1e-2,1,10]),
"nthread":8,
"silent":1,
"seed":100,
"measurer": Measurement(),
"xgb_model": xgb_model,
# "folds":list(skf.split(X,y)),
# "tree_method":"auto",
# "predictor":"cpu_predictor"
}
opt.set_search_space(search_space)
opt.set_objective(objective)
# start optimization
best = opt.optimize()
opt.save_trials(CURRENT_DIR)
# retrain and save best model
train_dmat, _ = xgb_model.get_train_test_Dmat()
best_model = xgb.train(best,
train_dmat,
num_boost_round=int(best["n_estimators"]))
xgb_model.set_model(best_model)
xgb_model.save_model(save_path=MODEL_DIR,
model_name=str(best),
prefix="TPE_200_BestModel")
def run(self, *args, initial_port=None, baud=None, dry=None):
port = self.get_port(initial_port)
ser = Serial(port, baud)
measurement = Measurement()
while 1:
try:
while not ser.is_open:
port = self.get_port(initial_port)
print('.', end='', flush=True)
if port:
ser.setPort(port)
try:
ser.open()
print('Serial connected', port, baud)
break
except SerialException as e:
print('\n', e)
time.sleep(5)
serial_line = ser.readline().decode("utf-8")
measurement.parse_line(serial_line)
if measurement.completed:
if measurement.is_complete():
if dry:
print(measurement.as_dict())
else:
ok = send_measurement(measurement)
print('measurement send,', ok)
measurement = Measurement()
except KeyboardInterrupt:
print('Serial reader closed, keyboard interrupt')
ser.close()
break
except SerialException as e:
print('Serial exception', e)
ser.close()
except Exception:
# Something went very wrong ?? Very dirty fix for now
update_script = '/home/pi/update.sh'
if os.path.isfile(update_script):
subprocess.call(update_script)
def addMeasurement(self, name, lumi, lumiErr):
#verify that this name is not already used
for meas in self.measurements:
if meas.name == name:
raise RuntimeError("Measurement %s already exists in TopLevelXML %s. Please use a different name." % (name, self.name))
pass
#add measurement to the list
self.measurements.append(Measurement(name, lumi, lumiErr))
return self.measurements[len(self.measurements) - 1]
def get_shopping_list(menu):
ingredients = dict()
for meal in menu:
try:
meal_ingredients = recipes[meal]
except KeyError:
continue # TODO deal with lack of recipe in recipe book
for meal_ingredient, amount in meal_ingredients:
if meal_ingredient in ingredients:
ingredients[meal_ingredient] += Measurement(amount)
else:
ingredients[meal_ingredient] = Measurement(amount)
ingredient_list = list()
for ingredient, amount in ingredients:
ingredient_list.append([ingredient, f'{amount.value} {amount.unit}'])
return ingredient_list
def main():
test_file_path = os.path.join(TEST_DIR, TEST_FILENAME)
packets = read_exchange_data(test_file_path)
packet_exchanges = get_exchange_times(packets)
measurement = Measurement(packet_exchanges, TEST_FILENAME)
#measurement.filter_unreceived()
measurement.show_send_periods()
measurement.show_success_chance()
measurement.show_anomalies_in_tx()
return
def load_measurements(src_path, dest_list, predicate = lambda m: True):
print("Loading from", src_path)
with open(src_path) as dat:
for ln in dat:
if Measurement.CSV_HEADER in ln: continue
try:
m = Measurement(ln[:-1])
if predicate and predicate(m):
dest_list.append(m)
except Exception as e:
print(e)
print(src_path, "loaded successfully")
def test_fire_sales(self,
args): # TODO not consistent with other test names
import logging
import networkx as nx
from environment import Environment
from runner import Runner
from measurement import Measurement
#
# INITIALIZATION
#
environment_directory = str(args[1])
identifier = str(args[2])
log_directory = str(args[3])
measurement_directory = str(args[4])
# Configure logging parameters so we get output while the program runs
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %H:%M:%S',
filename=log_directory + identifier + ".log",
level=logging.INFO)
logging.info('START logging for run: %s',
environment_directory + identifier + ".xml")
environment = Environment()
environment.initialize(environment_directory, identifier)
runner = Runner()
measurement = Measurement()
#
# UPDATE STEP
#
for i in range(environment.parameters.numSimulations):
environment.initialize(environment_directory, identifier)
runner.initialize(environment)
measurement.initialize() # clear the previous measurement
# do the run
runner.do_run(measurement, "info")
# do the histograms, i.e. add the current measurement to the histogram
measurement.do_histograms()
logging.info('')
#
# MEASUREMENT AND LOGGING
#
measurement.write_histograms(measurement_directory, environment)
logging.info('FINISHED logging for run: %s \n',
environment_directory + identifier + ".xml")
def test_zrangebyscore():
fakeAnchorId = "fakeAnchorId"
fakeTagId = "fakeTagId"
fakeRssi = "0"
fakeTimestamp = 0
Measurement.zremrangebyrank(r, fakeTagId, 0, -1)
fake_measurement = Measurement(fakeAnchorId, fakeTagId, fakeRssi, fakeTimestamp)
fake_measurement.save(r)
measurements = Measurement.zrangebyscore(r, fakeTagId, fakeTimestamp, fakeTimestamp)
assert(len(measurements) == 1)
assert(measurements[0].rssi == fakeRssi)
assert(measurements[0].tagId == fakeTagId)
assert(measurements[0].timestamp == fakeTimestamp)
assert(measurements[0].anchorId == fakeAnchorId)
def make_predictions(self, trained_model, data):
X_train, X_test, y_train, y_test = data
#get predictions
print(type(trained_model))
print("Making predictions!")
train_pred = trained_model.predict(X_train)
train_pred_prob = trained_model.predict_proba(X_train)
test_pred = trained_model.predict(X_test)
test_pred_prob = trained_model.predict_proba(X_test)
print(test_pred_prob)
# print metrics
measurer = Measurement()
print("Train resutls:")
measurer.print_measurements(y_train, train_pred, train_pred_prob[:, 1])
print("Test resutls:")
measurer.print_measurements(y_test, test_pred, test_pred_prob[:, 1])
def _perform_measurements(self):
"""TODO: Docstring for _perform_measurements.
:returns: TODO
"""
shutil.rmtree(config.DEFAULT_LOG_FILE_DIR, ignore_errors=True)
makedirs(config.DEFAULT_LOG_FILE_DIR)
with open(self.metafile_path, 'w') as mf:
meta_writer = csv.writer(mf)
meta_writer.writerow(['filename', 'taskname', 'alg'] +
self._metafile_header_params())
for alg_executable in config.ALGORITHMS:
for point in Collector.parameter_space():
measurement = Measurement(alg_executable)
measurement.run(self.text_files_dir, self.regex_files_dir,
point, meta_writer, self.task)
def setup_2obs2parts_sameparams():
out = {}
params = {'beta1': 1, 'beta2': 1, 'z': 1, 'var': 1/(2*np.pi)}
meas_data = pd.DataFrame(
{
'control': [1, 2],
'control_2': [1, 1],
'meas1': [-1, 5],
'meas2': [0, 5],
'meas3': [-1, 6]
},
index = [[1, 2], [1, 1]]
)
out['meas_obj'] = Measurement([params, params, params], meas_data)
out['facs_data'] = np.array([[-3, -2], [2, 3]])
return out
def visualize():
test_file_path = os.path.join(TEST_DIR, TEST_FILENAME)
exchange_data = read_exchange_data(test_file_path)
packet_exchanges = Exchange.list_to_exchange(exchange_data)
measurement = Measurement(exchanges=packet_exchanges,
test_name=TEST_FILENAME)
measurement.filter_unreceived()
measurement.filter_by_range(low=FILTER_MIN, high=FILTER_MAX)
measurement.plot()
results = count_results(packet_exchanges)
print_sorted(results)
print("Filtered:")
results = filter_results_by_frequency(results)
print_sorted(results)
def prepare_sampling(file):
# INPUT: list of tuples
print('prepare sampling')
unsampledMeasures = list()
print('sampling file: {}'.format(file))
with open(file, 'r') as f:
for row in f.readlines():
# 2017-01-03T10:04:45
liste = row.split(',')
datetime_object = datetime.datetime.strptime(
liste[0], '%Y-%m-%dT%H:%M:%S')
unsampledMeasures.append(
Measurement(datetime_object, liste[1],
liste[2].replace('\n', '')))
print('Starting Sampling!')
startSampling(unsampledMeasures)
def test_get_url_response_time_almostok(self, mock_requests):
# Mock correct response
classinst = Measurement("http://example.com", 5)
mock_requests.return_value.status_code = 503
mock_requests.return_value.headers = {'Date': 'Wed, 31 Mar 2021 08:47:43 GMT'}
mock_requests.return_value.text = "Mocked request response gateway timeout"
ret = classinst.get_url_response_time()
metricsDict = dict()
metricsDict['url'] = "http://example.com"
metricsDict['status_code'] = 503
metricsDict['receivedtext'] = "Mocked request response gateway timeout"
# do not use assertDictContainsSubset, deprecated
self.assertEqual(dict(ret, **metricsDict), ret, "Wrong return from mock")
self.assertIn("resptimems", ret, "No response time measurement in return from function when return code != 200")
def addMeasurement(self, name, lumi, lumiErr):
"""
Add measurement to this configuration
@param name Name of measurement
@param lumi Luminosity to use
@param lumiErr Error on the luminosity to use
"""
#verify that this name is not already used
for meas in self.measurements:
if meas.name == name:
raise RuntimeError("Measurement %s already exists in fitConfig %s. Please use a different name." % (name, self.name))
pass
#add measurement to the list
self.measurements.append(Measurement(name, lumi, lumiErr))
return self.measurements[len(self.measurements) - 1]
def measurement__write_measurement(self, identifier):
#
# VARIABLES
#
from environment import Environment
from measurement import Measurement
from runner import Runner
#
# CODE
#
# we need the environment
environment_directory = 'environments/'
measurement_directory = 'measurements/'
environment = Environment(identifier)
environment.read_environment_file("./" + environment_directory +
identifier + ".xml")
environment.initialize()
# and we need to create an exogenous network
environment.network.create_exogenous_network(
environment.parameter.network_density)
# and also a measurement
measurement = Measurement()
for agent in environment.network.network.nodes():
F0s = 1.0
F1s = 1.0
# we need the previous decisions of neighbors
neighbors_previous_decision = []
for neighbor in environment.network.network.neighbors(agent):
neighbors_previous_decision.append(neighbor.previous_x)
agent.compute_decision(neighbors_previous_decision,
environment.parameter.num_agents, F0s, F1s)
runner = Runner(identifier, environment, measurement)
print runner
runner.do_run()
print runner
measurement.write_measurement(measurement_directory + identifier)
def bRun_exec(self, bRun, cbAlgorithm):
activeAlgorithm = cbAlgorithm.get_active_id()
if activeAlgorithm == "1":
wAlgorithm = Superposition_1()
elif activeAlgorithm == "2":
wAlgorithm = Superposition_2()
elif activeAlgorithm == "11":
wAlgorithm = Measurement()
else:
wAlgorithm = Gtk.Window()
wAlgorithm.set_title("Algorithm")
wAlgorithm.add(Gtk.Label("this is a dummy window"))
wAlgorithm.set_transient_for(self)
#wAlgorithm.set_type_hint(Gtk.gdk.WINDOW_TYPE_HINT_DIALOG)
wAlgorithm.connect("destroy", lambda *w: Gtk.main_quit())
wAlgorithm.show_all()
def tune_hyperparams_scikit(self, model, X_train, y_train):
"""
Tuning parameters of model using scikit learn GridSearch
"""
print("Started CV task at " +
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
# create gini scorer
measurer = Measurement()
gini_score = measurer.get_gini_scorer()
params = [{
"learning_rate": [0.01],
"n_estimators": [100, 200],
"seed": [100],
"max_depth": [2],
"min_child_weight": [1, 5],
"subsample": [1]
}]
print("Running GridSearch")
gscv = GridSearchCV(model,
params,
cv=2,
n_jobs=-1,
scoring=gini_score,
verbose=3,
error_score=0)
gscv.fit(X_train, y_train)
best_model = gscv.best_estimator_
# save best model
save_model(best_model, MODEL_DIR, prefix="CV5-bestModel-")
# save CV results
gscv_resutls = pd.DataFrame(gscv.cv_results_)
gscv_resutls.to_csv("GridSearchRestest.csv", index=False)
print("Finished CV task at " +
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
def xgbcv(self, X, y, params=None):
"""
Tuning parameters of model using XGBoost CVs
"""
print("Started CV task at " +
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
# create gini scorer
measurer = Measurement()
gini_score = measurer.xgb_gini_scorer
# create StratifiedKFold object
skf = StratifiedKFold(n_splits=3, shuffle=True, random_state=100)
# dmat
dtrain = xgb.DMatrix(X, label=y)
if not params:
params = {
"learning_rate": 0.1,
"n_estimators": 20,
"seed": 100,
"max_depth": 3,
"min_child_weight": 1,
"subsample": 1,
"tree_method": "gpu_hist",
"predictor": "cpu_predictor",
"objective": 'binary:logistic'
}
cv_results = xgb.cv(params,
dtrain,
num_boost_round=int(params["n_estimators"]),
feval=gini_score,
folds=list(skf.split(X, y)),
verbose_eval=5)
cv_results.to_csv("cv_resutls.csv", index=False)
print("Finished CV task at " +
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
def test_instanciate_ok(self):
classinst = Measurement("http://example.com", 5)
self.assertIsInstance(classinst, Measurement, "result is not an instance of Measurement class")
measurement_directory = str(args[4])
# Configure logging parameters so we get output while the program runs
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %H:%M:%S',
filename=log_directory + identifier + ".log",
level=logging.INFO)
logging.info('START logging for run: %s',
environment_directory + identifier + ".xml")
environment = Environment(identifier)
environment.read_environment_file("./" + environment_directory +
identifier + ".xml")
environment.initialize()
measurement = Measurement()
#
# UPDATE STEP
#
runner = Runner(identifier, environment, measurement)
if environment.parameter.compute_goodness == "True": # compute_goodness = False if we have a normal run
#print "NOW RUNNING: " + environment.identifier
#start_time = datetime.now()
#print "START: " + str(start_time)
runner.compute_goodness()
#end_time = datetime.now()
#print "END: " + str(end_time)
#print "TIME: " + str(end_time - start_time) + "\n"
def test_basic():
assert Measurement(2, 3) + Measurement(4, 4) == Measurement(6, 5)
def test_found_text_in_body_nothing(self):
classinst = Measurement("http://example.com", 5, '^hop')
found = classinst.found_text_in_body('fffhop')
self.assertEqual(found, None, 'Should find nothing when looking up')
def test_setup_producer_get_none_if_producer_ko(self, mock_kafka_producer):
classinst = Measurement("http://example.com", 5)
prodRes = classinst.setup_producer('kafkaservers')
self.assertEqual(prodRes, None, 'Should return None if producer cannot be set')
def test_instanciate_nok_badurl(self, mock_requests):
classinst = Measurement("example com", 5)
mock_requests.return_value = None
ret = classinst.get_url_response_time()
self.assertEqual(None, ret, "Bad URL given, failed")
def startMeasurement(filepath, file_type, covalent_radii_cut_off, c, n1, n2,
calculate):
if n1 > n2:
print("Final m cannot be smaller than initial m")
return
print("Starting script...")
slab = read(filepath)
total_particles = len(slab)
print("Slab %s read with success. Total particles %d" %
(filepath, total_particles))
print("Creating graph...")
if file_type == 'V':
G = generateGraphFromSlabVinkFile(slab, covalent_radii_cut_off)
else:
G = generateGraphFromSlab(slab, covalent_radii_cut_off)
total_nodes = G.get_total_nodes()
if total_nodes == 0 or G.get_total_edges() == 0:
print("No edges found in graph. Check covalent_radii_cut_off")
return
print("Graph created with success. Nodes found: %d" % total_nodes)
cell = slab.get_cell()
# somente para celulas ortogonais
if not is_orthorhombic(cell):
print("Unit cell is not orthorhombic")
return
measurement = Measurement()
measurement.fromFile(filepath, covalent_radii_cut_off, c, n1, n2)
measurement.createFile()
(pbcX, pbcY, pbcZ) = slab.get_pbc()
(dmin, dmax) = getMaxMinSlabArray(slab)
maxM = getNumberRandomPositions(n2 - 1, total_particles)
configurationalEntropy = bg.ConfigurationalEntropy(G, 3, len(slab), pbcX,
pbcY, pbcZ, maxM)
configurationalEntropy.add_positions(slab.get_positions(wrap=True))
print("Generating random positions...")
configurationalEntropy.init_search(0, cell[0][0], 0, cell[1][1], 0,
cell[2][2], n2 - 1, maxM, dmin, dmax)
print("Random positions generated.")
H_n_values = []
for n in range(n1, n2):
measurement.start()
m = getNumberRandomPositions(n, total_particles)
H_n, H1n = run(configurationalEntropy, m, n, c)
measurement.writeResult(n, m, H_n, H1n)
measurement.end()
H_n_values.append((n, H_n, H1n, "Y"))
calculateConfigurationalEntropy(n1, n2, H_n_values, c, calculate)
print("Program ended correctly")
def test_instanciate_ok_not_so_badurl(self):
classinst = Measurement("example.com", 5)
self.assertIsInstance(classinst, Measurement, "result is not an instance of Measurement class")
