def evaluation(data_type,trec_path,dev_query_embedding2id,passage_embedding2id,dev_I,dev_D,test_set="trec2019",dev_split_num=-1,split_idx=-1):
if data_type ==0 :
if test_set== "marcodev":
qrels="../data/raw_data/msmarco-docdev-qrels.tsv"
elif test_set== "trec2019":
qrels="../data/raw_data/2019qrels-docs.txt"
elif data_type ==1:
if test_set == "marcodev":
qrels="../data/raw_data/qrels.dev.small.tsv"
else:
logging.error("wrong data type")
exit()
trec_path=trec_path.replace(".trec",".formatted.trec")
met = Metric()
if split_idx >= 0:
split_file_path=qrels+f"{dev_split_num}_fold.split_dict"
with open(split_file_path,'rb') as f:
split=pickle.load(f)
else:
split=None
ndcg10 = met.get_metric(qrels, trec_path, 'ndcg_cut_10',split,split_idx)
mrr10 = met.get_mrr(qrels, trec_path, 'mrr_cut_10',split,split_idx)
mrr100 = met.get_mrr(qrels, trec_path, 'mrr_cut_100',split,split_idx)
print(f" evaluation for {test_set}, trec_file {trec_path}, split_idx {split_idx} \
ndcg_cut_10 : {ndcg10}, \
mrr_cut_10 : {mrr10}, \
mrr_cut_100 : {mrr100}"
)
return ndcg10
def test_pair_dist_starting_at_zero_dist(self):
sigma = 0.1
test_metric = Metric(sigma, edge=1)
positions = [[0.2, 0.2], [0.2 + 2 * sigma, 0.2]]
self.assertAlmostEqual(
test_metric.pair_dist(positions[0], positions[1], 10, [1, 0]), 0,
1, "Should collide")
def invert_metric(self, g):
""" Invert a given metric g and return the inverted metric
as a shorthand dictionary of sympy.core.symbols.Symbol objects.
"""
N = int(np.sqrt(len(g.elements)))
g_matrix = sp.MatrixSymbol('g_matrix', N, N)
g_matrix = sp.Matrix(g_matrix)
print('Converting the metric to sp.Matrix object...')
for i in range(N):
for j in range(N):
m = g.index_dict[i]
n = g.index_dict[j]
mn = m + n
g_matrix[i, j] = g.elements[mn]
print('Inverting the matrix...')
g_matrix_inv = g_matrix**(-1)
g_inv = Metric(index_dict=g.index_dict)
for i in range(N):
for j in range(N):
m = g_inv.index_dict[i]
n = g_inv.index_dict[j]
mn = m + n
g_inv.elements[mn] = g_matrix_inv[i, j]
print('Succesfully inverted the matrix.')
return g_inv
def test1(n):
from numpy import array
# One equilibrium of Ar4 LJ cluster (in coordinates of
# c2v_tetrahedron1 Func):
w = 0.39685026
A = array([w, w, +w])
# Another equilibrium:
B = array([w, w, -w])
# Halfway between A and B:
C = (A + B) / 2.0
C = array([w + 0.01, w - 0.01, 0.0])
xs = array([A, C, B])
from test.testfuns import c2v_tetrahedron1, diagsandhight
from path import MetricPath
from metric import Metric
from numpy import linspace
z = c2v_tetrahedron1()
# z = diagsandhight()
# r = 1.12246195815
# A = array([r, r, r / sqrt(2.)])
# B = array([r, r, -r / sqrt(2.)])
# C = array([r, r * sqrt(2.), 0.])
# xs = array([A, C, B])
p = MetricPath(xs, Metric(z).norm_up)
x0 = map(p, linspace(0., 1., n))
from ase import Atoms
from qfunc import QFunc
from func import compose
pes = compose(QFunc(Atoms("Ar4")), z)
from rc import Volume
vol = compose(Volume(), z)
def callback(x, e, g, t):
# from pts.tools.jmol import jmol_view_path
print "energies=", e # map(pes, x)
print "volume=", map(vol, x)
# jmol_view_path(map(z, x), syms=["Ar"]*4, refine=1)
pass
print "BEFORE:"
callback(x0, map(pes, x0), map(pes.fprime, x0), None)
x1, info = soptimize(pes, x0, tangent1, rc=vol, callback=callback)
# print "info=", info
print "AFTER:"
callback(x1, map(pes, x1), map(pes.fprime, x1), None)
def list_metrics(self, start_date, end_date):
# Create CloudWatch client
cloudwatch = boto3.client('cloudwatch')
metrics = []
statistics_type = 'Average'
period = 3600 #hour
# List metrics through the pagination interface
paginator = cloudwatch.get_paginator('list_metrics')
for response in paginator.paginate(Namespace=self.namespace):
for row in response['Metrics']:
metric_name = row['MetricName']
for row2 in row['Dimensions']:
dimension_name = row2['Name']
dimension_value = row2['Value']
if (self.namespace == 'AWS/EC2' and metric_name in ["CPUUtilization","NetworkOut","NetworkIn","EBSWriteBytes","EBSReadBytes","DiskReadBytes","DiskWriteBytes","DiskWriteOps","DiskReadOps"]) \
or self.namespace != 'AWS/EC2':
metric = Metric(metric_name, self.namespace,
dimension_name, dimension_value,
statistics_type, period, start_date,
end_date)
metrics.append(metric)
return metrics
def compute_suspicious_score(self):
scores = []
for i in xrange(1, self.total_stmt + 1):
cp, cf, np, nf = self.compute_statistics(i)
metric = Metric(cp, cf, np, nf)
scores.append(metric.odds_ratio())
return scores
def select_metric_value(m):
"""Interactive selection of a metric value
Input:
m : list of values that can be unpacked into valid
parameters for constructing a Metric
Return:
a valid index for the Metric
"""
m = Metric(*m)
default_metric_value = m.index
print("\n{0} {1} {2} {0}".format(10 * "+", m.name, m.short_name))
while True:
for v in m.values:
print(v, v.description)
idx = input('Select one [{0}]: '.format(default_metric_value)).upper()
if not idx:
idx = default_metric_value
print('Selected metric value ###|', idx, '|###')
try:
m.index = idx
except AssertionError:
print('Not valid')
else:
return m.index
def test_getMetricWithMeasure(self):
metric = Metric('random forest')
metric.addValue('accuracy', 0.75)
metric.addValue('recall', 0.9)
metric.addValue('precision', 0.2)
metric.addValue('acc', 0.98)
metric.addValue('test_value', 500)
actual_result = metric.getMetricWithMeasure()
expected_result = {'accuracy': 0.75, 'recall': 0.9, 'precision': 0.2, 'acc': 0.98, 'test_value': 500}
self.assertEqual( actual_result, expected_result )
actual_result = metric.getMetricWithMeasure('acc')
expected_result = {'acc': 0.98}
self.assertEqual( actual_result, expected_result )
actual_result = metric.getMetricWithMeasure(['recall', 'acc', 'test_value'])
expected_result = {'recall': 0.9, 'acc': 0.98, 'test_value': 500}
self.assertEqual( actual_result, expected_result )
try:
actual_result = metric.getMetricWithMeasure( 480 )
fail(self)
except ValueError as ve:
self.assertEqual( str(ve), 'Metric.getMetricWithMeasure must be given either a string of metric or array of strings of metrics desired.' )
def test_addValue(self):
metric = Metric('random forest')
metric.addValue('accuracy', 0.75)
self.assertEqual( metric.getValue('accuracy'), 0.75 )
metric.addValue('recall', 0.9)
metric.addValue('precision', 0.2)
self.assertEqual( metric.getValue('accuracy'), 0.75 )
self.assertEqual( metric.getValue('recall'), 0.9 )
self.assertEqual( metric.getValue('precision'), 0.2 )
metric.addValue('accuracy', 0.99)
metric.addValue('accuracy', 0.3)
metric.addValue('accuracy', 0.25672837482)
self.assertEqual( metric.getValue('accuracy'), 0.25672837482 )
try:
metric.addValue('accuracy', ['hello!'])
fail(self)
except ValueError as ve:
self.assertEqual( str(ve), 'Metric.addValue must have \'m_type\' as string and \'value\' as integer or floating point number instead of type(m_type) => <class \'str\'> and type(value) => <class \'list\'>')
try:
metric.addValue(['accuracy'], 0.9)
fail(self)
except ValueError as ve:
self.assertEqual( str(ve), 'Metric.addValue must have \'m_type\' as string and \'value\' as integer or floating point number instead of type(m_type) => <class \'list\'> and type(value) => <class \'float\'>')
def get_info(self):
distance = None
try:
date = datetime.datetime.utcnow()
GPIO.output(self.pin_trigger, GPIO.HIGH)
time.sleep(0.00001)
GPIO.output(self.pin_trigger, GPIO.LOW)
while GPIO.input(self.pin_echo) == 0:
pulse_start_time = time.time()
while GPIO.input(self.pin_echo) == 1:
pulse_end_time = time.time()
pulse_duration = pulse_end_time - pulse_start_time
distance = round(pulse_duration * 17150, 2)
except Exception as e:
logger.error(f'Error in distance: {e}')
self.reset()
if distance is not None:
name = self.metric_prefix + 'distance'
if self.output == 'WF':
name = 'Distance'
if len(self.metric_prefix) > 0:
name = self.metric_prefix + '.' + name
if self.format == 'f':
distance = distance / 30.48
if self.format == 'i':
distance = distance / 30.48 * 12
self.metrics.append(Metric(name, distance, date))
def test_wall_dist(self):
test_metric = Metric(sigma=0.1, edge=1)
pos = [0.5, 0.5]
for wall in [1, -1, 2, -2]:
v_hat = [0, 0]
v_hat[np.abs(wall) - 1] = np.sign(wall)
dist_to_wall, wall_type = test_metric.wall_dist(pos, v_hat, 1)
self.assertEqual(dist_to_wall, 0.4, "Wrong distance to wall")
self.assertEqual(wall_type, wall, "Wrong Classification of wall")
def __init__(self, args):
"""
Create Evaluator object which handles the evaluation of a specified model,
the tracking of computed metrics, and saving of results.
:param args: ArgParse object which holds the path the experiment configuration file along
with other key experiment options.
"""
# get experiment configuration file
osj = os.path.join
oss = os.path.split
base_path = os.path.dirname(os.path.abspath(__file__))
config = get_config(base_path, args.config)
self.config = config
config_path = osj(base_path, 'config')
# create experiment, logging, and model checkpoint directories
model_name = os.path.split(args.model)[1].split('.')[0]
self.eval_dir = osj(base_path, oss(args.config)[0], 'eval_' + args.data + '_' + model_name)
self.visual_path = osj(self.eval_dir, 'visuals')
if os.path.exists(self.eval_dir):
print("Error: This evaluation already exists")
print("Cancel Session: 0")
print("Overwrite: 1")
if input() == str(1):
shutil.rmtree(self.eval_dir)
else:
exit(1)
os.mkdir(self.eval_dir)
os.mkdir(self.visual_path)
# Device
self.device = torch.device("cpu" if args.gpuid == 'cpu' else "cuda:{}".format(args.gpuid))
# Model - load pre-trained
model = get_model(config)
self.model = model
self.load_model(args.model)
self.model.to(self.device)
# Loss metric
self.criterion = nn.MSELoss()
# Dataset and DataLoader
self.data_type = args.data
self.dataset = ShapeNet(config, config_path, args.data)
self.tracked_samples = extract_categories(self.dataset.samples, 5)
self.loader = DataLoader(self.dataset, batch_size=config['batch_size'],
shuffle=True, num_workers=1, drop_last=False)
print("Commencing evaluation with {} model on {} split".format(config['model'], args.data))
# Metrics
self.metrics = Metric(config)
self.metric_tracker = MetricTracker(config, self.eval_dir, args.data)
self.epoch = 0
def get_info(self):
current_value = GPIO.input(self.pin)
while current_value == self.initial_value:
#do nothing, wait for a change
current_value = GPIO.input(self.pin)
time.sleep(0.00001)
# changed
self.initial_value = current_value
self.metrics.append(Metric(self.name, current_value, datetime.datetime.utcnow()))
def get_info(self):
date = datetime.datetime.utcnow()
humidity, temp = Adafruit_DHT.read_retry(self.code, self.pin)
if temp is not None:
name = self.metric_prefix + 'humidity'
if self.output == 'WF':
name = 'Humidity'
if len(self.metric_prefix) > 0:
name = self.metric_prefix + '.' + name
self.metrics.append(Metric(name, humidity, date))
if self.format == 'f':
temp = (temp * 9 / 5) + 32
name = self.metric_prefix + 'temperature'
if self.output == 'WF':
name = 'Temperature'
if len(self.metric_prefix) > 0:
name = self.metric_prefix + '.' + name
self.metrics.append(Metric(name, temp, date))
def test_metric_improvement(self):
params = {"name": "logloss"}
m = Metric(params)
y_true = np.array([0, 0, 1, 1])
y_predicted = np.array([0, 0, 0, 1])
score_1 = m(y_true, y_predicted)
y_true = np.array([0, 0, 1, 1])
y_predicted = np.array([0, 0, 1, 1])
score_2 = m(y_true, y_predicted)
self.assertTrue(m.improvement(score_1, score_2))
def test_generate_normal(self):
sys = SystemDataGenerator(name='test00')
metric_vals = {'name': 'cpu_busy',
'min_val': 0,
'max_val': 10,
'mean': 10,
'sd': 3}
metric = Metric(**metric_vals)
metric_value = sys.generate_metric_value_normal(metric)
assert metric_value[metric.name] > 0
def createMetric(category, metric, value):
# Build a metric type name for the given metric
m = METRIC_TYPE + ' / ' + category + ' / ' + metric
# Return a metric description object
return Metric(source=METRIC_SOURCE,
metric=m,
resource=METRIC_RESOURCE,
node=METRIC_NODE,
value=value)
def convertUptime(self, data, date):
hasUser = False
if "user" in data.lower().strip():
hasUser = True
data = list(data.replace(",", "").split(' '))
if len(data) > 1:
for i in range(len(data)):
data[i] = list(filter(None, data[i].split(' ')))
else:
data = list(filter(None, data[0].split(' ')))
data = [[data[0], data[1]], [data[2]], [data[4], data[5]],
[data[6], data[7], data[8], data[9], data[10]]]
#check if uptime has days
uptime = 0
i = 0
if (hasUser and len(data) > 3) or (len(data) > 2 and not hasUser):
if (len(data[0]) > 2):
uptime += int(data[i][2]) * 24 * 60
i += 1
uptime += self.convertDurationToInt(data[i][0])
else:
if len(data[i]) == 5:
uptime = int(data[i][2]) * 24 * 60
uptime += self.convertDurationToInt(data[i][4])
elif len(data[i]) == 6:
uptime = int(data[i][2]) * 24 * 60
uptime += int(data[i][4])
else:
uptime += self.convertDurationToInt(data[i][2])
self.metrics.append(Metric("uptime", uptime, date))
i += 1
#users
if hasUser:
users = int(data[i][0])
self.metrics.append(Metric("users", users, date))
i += 1
#load
minMean = float(data[i][2])
fiveMinMean = float(data[i][3])
fifteenMean = float(data[i][4])
self.metrics.append(Metric("load1min.mean", minMean, date))
self.metrics.append(Metric("load5min.mean", fiveMinMean, date))
self.metrics.append(Metric("load15min.mean", fifteenMean, date))
def test_step_size(self):
sigma = 0.1
test_metric = Metric(sigma, edge=1)
positions = [[0.2, 0.2], [0.7, 0.2]]
current_step_size, step_type, sphere_or_wall_ind = \
test_metric.step_size(positions, 0, [1, 1], .2)
self.assertEqual(current_step_size, .2, "Problem with free step")
current_step_size, step_type, sphere_or_wall_ind = \
test_metric.step_size(positions, 0, [1, 0], 1)
self.assertAlmostEqual(current_step_size, .3, 1,
"Problem with pair collision")
def test_equals(self):
metric1 = Metric('some name')
metric1.addValue('acc', 0.98)
metric1.addValue('recall', 0.72)
metric2 = Metric('some other name')
metric2.addValue('acc', 0.98)
metric2.addValue('recall', 0.72)
self.assertEqual( metric1, metric2 )
metric1 = Metric('some name')
metric1.addValue('acc', 0.98)
metric1.addValue('recall', 0.72)
metric2 = Metric('some other name')
metric2.addValue('acc', 0.20)
metric2.addValue('recall', 0.5)
self.assertNotEqual( metric1, metric2 )
def slave_task_metrics(mesos):
ms = []
for pid in mesos.slaves():
slave = mesos.slaves()[pid]
prefix = "slave.[].executors.singularity.tasks.[]"
metrics = [
Metric("cpus_system_time_secs", prefix + ".cpus.system_time_secs"),
Metric("cpus_user_time_secs", prefix + ".cpus.user_time_secs"),
Metric("cpus_limit", prefix + ".cpus.limit"),
Metric("mem_limit_bytes", prefix + ".mem.limit_bytes"),
Metric("mem_rss_bytes", prefix + ".mem.rss_bytes"),
]
for m in metrics:
for t in slave["task_details"]:
# TODO:m.Add(t, "key1", "key2", ...)
m.Add(t["statistics"], [pid, t["executor_id"]])
ms += metrics
return ms
def __init__(self, init_list, goal_list):
"""Initialise Solver object. Raise ValueError if solution not possible."""
self.initial_state = copy.deepcopy(self.list_to_grid(init_list))
self.goal_state = copy.deepcopy(self.list_to_grid(goal_list))
self.frontier = []
self.explored = set()
self.metrics = Metric(self.frontier)
def init_S2(self):
"""Initialize the metric on the 2-sphere, S2.
"""
theta = sp.symbols(self.S2[0])
phi = sp.symbols(self.S2[1])
g = Metric(index_dict=self.S2)
g.convert_to_shorthand()
g.elements['thetatheta'] = 1
g.elements['phiphi'] = (sp.sin(theta))**2
return g
def test_wall_dist_starting_at_the_wall(self):
test_metric = Metric(sigma=0.1, edge=1)
pos = [0.1, 0.5]
dist_to_wall, wall_type = test_metric.wall_dist(pos, [-1, 0], 1)
self.assertEqual(dist_to_wall, 0, "Wrong distance to wall")
self.assertEqual(wall_type, -1, "Wrong Classification of wall")
dist_to_wall, wall_type = test_metric.wall_dist(pos, [0, 1], 1)
self.assertEqual(dist_to_wall, 0.4, "Wrong distance to wall")
self.assertEqual(wall_type, 2, "Wrong Classification of wall")
def test_create(self):
params = {"name": "logloss"}
m = Metric(params)
y_true = np.array([0, 0, 1, 1])
y_predicted = np.array([0, 0, 1, 1])
score = m(y_true, y_predicted)
self.assertTrue(score < 0.1)
y_true = np.array([0, 0, 1, 1])
y_predicted = np.array([1, 1, 0, 0])
score = m(y_true, y_predicted)
self.assertTrue(score > 1.0)
def test_pair_dist(self):
sigma = 0.1
test_metric = Metric(sigma, edge=1)
positions = [[0.2, 0.2], [0.7, 0.2]]
self.assertAlmostEqual(
test_metric.pair_dist(positions[0], positions[1], 10, [1, 0]),
0.5 - 2 * sigma, 1, "Wrong distance between pairs")
positions = [[0.2, 0.2], [0.7, 0.7]]
self.assertAlmostEqual(
test_metric.pair_dist(positions[0], positions[1], 10, [0, 1]),
float('inf'), 1, "Should not collide")
def train(self, metric_data, prediction_duration=15):
prediction_range = prediction_duration
# convert incoming metric to Metric Object
self.metric = self.metric + Metric(
metric_data
) # because the rolling_data_window_size is set, this df should not bloat
data = self.metric.metric_values
vals = np.array(data["y"].tolist())
print("Training Model .....")
forecast_values = self.fourier_extrapolation(vals, prediction_range,
1) # int(len(vals)/3))
dataframe_cols = {}
dataframe_cols["yhat"] = np.array(forecast_values)
# find most recent timestamp from original data and extrapolate new timestamps
_LOGGER.debug("Creating Dummy Timestamps.....")
minimum_time = min(data["ds"])
dataframe_cols["timestamp"] = pd.date_range(
minimum_time, periods=len(forecast_values), freq="min")
# create dummy upper and lower bounds
_LOGGER.debug("Computing Bounds .... ")
upper_bound = np.array([(np.ma.average(
forecast_values[:i],
weights=np.linspace(0, 1, num=len(forecast_values[:i]))) +
(np.std(forecast_values[:i]) * 2))
for i in range(len(forecast_values))])
upper_bound[0] = np.mean(
forecast_values[0]) # to account for no std of a single value
lower_bound = np.array([(np.ma.average(
forecast_values[:i],
weights=np.linspace(0, 1, num=len(forecast_values[:i]))) -
(np.std(forecast_values[:i]) * 2))
for i in range(len(forecast_values))])
lower_bound[0] = np.mean(
forecast_values[0]) # to account for no std of a single value
dataframe_cols["yhat_upper"] = upper_bound
dataframe_cols["yhat_lower"] = lower_bound
# create series and index into predictions_dict
_LOGGER.debug("Formatting Forecast to Pandas ..... ")
forecast = pd.DataFrame(data=dataframe_cols)
forecast = forecast.set_index("timestamp")
self.predicted_df = forecast
_LOGGER.debug(forecast)
def eval(self, X, y, metrics=[], fold_num=None):
if len(X) == len(y):
# If we aren't running cross validation, then clear out the metrics
if not fold_num:
self.metrics_manager.clearMetrics()
for model_name in self.models_dict.keys():
model = self.models_dict[model_name]
if fold_num and type(fold_num) == int:
metric_object = Metric(model_name, fold=fold_num)
else:
metric_object = Metric(model_name)
# If a ROC curve is requested, graph it!
if 'roc' in metrics or 'ROC' in metrics:
fpr, tpr, roc_auc = model.calcROC(X, y,
self.type_of_experiment)
self.metrics_manager.addToROCCurve(model_name, fpr, tpr,
roc_auc)
model_metrics = model.eval(X, y, metrics)
for measure_name in model_metrics.keys():
metric_object.addValue(measure_name,
model_metrics[measure_name])
self.metrics_manager.addMetric(metric_object)
if not fold_num:
return self.metrics_manager.getMetrics()
else:
if self.name:
raise ValueError(
'Data and target provided to \'{}\' must be same length:\n\tlen of data: {}\n\tlen of target: {}'
.format(self.name, str(len(X)), str(len(y))))
else:
raise ValueError(
'Provided data and target must be same length:\n\tlen of data: {}\n\tlen of target: {}'
.format(str(len(X)), str(len(y))))
def test_eval(self):
iris = load_iris()
X_train, y_train = iris.data[:120], iris.target[:120]
X_test, y_test = iris.data[120:], iris.target[120:]
rf = Model('rf', RandomForestClassifier, {'random_state':0})
rf.run(X_train, y_train)
metrics = rf.eval(X_test, y_test, metrics=['acc'])
rf_metric = Metric('rf')
rf_metric.addValue('acc', 0.7666666666666667)
expected_metrics = rf_metric.getValues()
self.assertEqual( metrics, expected_metrics )
def convertDF(self, data, date):
data = self.convertStringToList(data)
length = len(data)
for i in range(length):
if i > 0:
colLength = len(data[i])
for c in range(colLength):
if c > 0 and c < 4:
name = 'disk.' + data[0][c]
tag = {}
tag['filesystem'] = data[i][0]
tag['mount'] = data[i][5]
value = int(data[i][c])
metric = Metric(name, value, date)
metric.tags = tag
self.metrics.append(metric)
