def post(self):
assert util.development() or oauth.is_current_user_admin()
util.log_upload_data(self.request.path, self.request.get("data"))
data = StringIO.StringIO(self.request.get("data"))
for line in data:
data = json.loads(line)
# We first load the fileset into the database
# For use later, we also add a list of filenames in the fileset
m = model.Metric(key_name=data["name"],
display_name=data["display name"],
distortion=data["distortion"],
yaxis=data.get("yaxis", None))
m.put()
model.metrics().invalidate()
def post(self):
assert util.development() or oauth.is_current_user_admin()
util.log_upload_data(self.request.path, self.request.get("data"))
data = StringIO.StringIO(self.request.get("data"))
for line in data:
data = json.loads(line)
# We first load the fileset into the database
# For use later, we also add a list of filenames in the fileset
m = model.Metric(key_name=data["name"],
display_name=data["display name"],
distortion=data["distortion"],
yaxis=data.get("yaxis", None))
m.put()
model.metrics().invalidate()
def get_adhoc_improvement(metrics, config, filenames, commit):
# Mostly copied from main.py with some notable changes
response = []
# Find the baseline based on the raw URL variables
parent = main.find_baseline(",".join(metrics), config, ",".join(filenames),
commit)
result = []
for m in metrics:
if model.metrics()[m].distortion:
improvement = main.rd_improvement
else:
improvement = main.mean_improvement
if parent:
baseline_data = main.fetch_metric_for_fileset(
m, config, filenames, parent)
average, results = main.calculate_improvement(
m, config, filenames, commit, baseline_data, improvement)
else:
results = dict([f, 0.0] for f in filenames)
for f, composite in results.iteritems():
response.append({
'metric': m,
'config': config,
'baseline': parent,
'filename': f,
'value': composite
})
return response
def fetch_config_info(metric, config, filename, commit):
'''This function fetches the data for a given metric, config, filename,
commit tuple. This functionality is used multiple places, such as
CodecMetricHandler and AverageImprovementHandler.'''
indexes = model.CodecMetricIndex.all(keys_only = True)
indexes = indexes.filter('metrics =', metric)
indexes = indexes.filter('config_name =', config)
indexes = indexes.filter('files =', filename)
indexes = indexes.filter('commit =', commit)
keys = [k.parent() for k in indexes]
if len(keys) == 0:
return None
metric_data = model.metrics()[metric]
result=[]
for cm in db.get(keys): # cm = codec metric
# we get the runtime and config flags
config_flags = cm.config_flags
runtime_flags = cm.runtime_flags
commit = cm.commit
result.append((commit, config_flags, runtime_flags))
# Sanity checks - we only want one runtime configuration
assert len(result) == 1
# We go ahead and return the tuple
result = result[0]
return result
def get(self, metric, config, filename, commit):
"""Fetches the requested metric data as JSON"""
if not metric or not config or not filename or not commit:
self.error(404)
return
filename = urllib.unquote(filename)
commit = urllib.unquote(commit)
if commit[0] == "~":
result = {
'yaxis':
"Percent Improvement",
'data':
fetch_time_series(metric, config, filename, commit)[filename],
}
else:
result = {
'yaxis': model.metrics()[metric].yaxis,
'data': fetch_codec_metric(metric, config, filename, commit),
}
# Return the result
if result['data']:
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(pretty_json(result))
else:
self.error(404)
def get(self, metric, config, filename, commit):
"""Fetches the requested metric data as JSON"""
if not metric or not config or not filename or not commit:
self.error(404)
return
filename = urllib.unquote(filename)
commit = urllib.unquote(commit)
if commit[0] == "~":
result = {'yaxis': "Percent Improvement",
'data': fetch_time_series(metric, config, filename,
commit)[filename],
}
else:
result = {'yaxis': model.metrics()[metric].yaxis,
'data': fetch_codec_metric(metric, config, filename,
commit),
}
# Return the result
if result['data']:
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(pretty_json(result))
else:
self.error(404)
def fetch_config_info(metric, config, filename, commit):
'''This function fetches the data for a given metric, config, filename,
commit tuple. This functionality is used multiple places, such as
CodecMetricHandler and AverageImprovementHandler.'''
indexes = model.CodecMetricIndex.all(keys_only=True)
indexes = indexes.filter('metrics =', metric)
indexes = indexes.filter('config_name =', config)
indexes = indexes.filter('files =', filename)
indexes = indexes.filter('commit =', commit)
keys = [k.parent() for k in indexes]
if len(keys) == 0:
return None
metric_data = model.metrics()[metric]
result = []
for cm in db.get(keys): # cm = codec metric
# we get the runtime and config flags
config_flags = cm.config_flags
runtime_flags = cm.runtime_flags
commit = cm.commit
result.append((commit, config_flags, runtime_flags))
# Sanity checks - we only want one runtime configuration
assert len(result) == 1
# We go ahead and return the tuple
result = result[0]
return result
def valid_cost(ep):
metrics = []
L_v_all = 0.
all_keys = valid_image.keys()
metric_per = np.zeros((6, 4))
for k in all_keys:
output_prob_np = np.zeros(valid_gt[k].shape)
for i in range(0, output_prob_np.shape[0], batch_size):
output_prob_np[i:i+batch_size] = sess.run(out_bin, feed_dict=\
{x: valid_image[k][i:i+batch_size], \
y_: valid_gt[k][i:i+batch_size], is_train:False})
if plane == 'A1':
output_prob_np = np.swapaxes(output_prob_np, 0, 1)
output_prob_np = np.swapaxes(output_prob_np, 2, 1)
valid_gt[k] = np.swapaxes(valid_gt[k], 0, 1)
valid_gt[k] = np.swapaxes(valid_gt[k], 2, 1)
if plane == 'A2':
output_prob_np = np.swapaxes(output_prob_np, 0, 1)
valid_gt[k] = np.swapaxes(valid_gt[k], 0, 1)
for nn in range(6):
class_met = model.metrics(output_prob_np[..., nn + 1],
valid_gt[k][..., nn + 1])
metric_per[nn, :] = metric_per[nn, :] + class_met.metrics_cal_cpu()
metric_per = metric_per / (len(all_keys))
print("Ave. Dice=", np.mean(metric_per, axis=0)[3])
save(output_prob_np,
directory + '/out_prob_' + str(ep).zfill(3) + '_' + str(k) + '.nii')
def query(self, metric, config, filename, commit):
result = None
metric_cache = model.metrics()
for m in _split_field(metric):
for cfg in _split_field(config):
for f in _split_filename(filename):
for cm in _split_field(commit):
if not result:
result = self.query_(metric_cache, m,cfg,f,cm)
else:
r = self.query_(metric_cache, m,cfg,f,cm)
for idx in range(4):
result[idx] = result[idx].intersection(r[idx])
return result
def query(self, metric, config, filename, commit):
result = None
metric_cache = model.metrics()
for m in _split_field(metric):
for cfg in _split_field(config):
for f in _split_filename(filename):
for cm in _split_field(commit):
if not result:
result = self.query_(metric_cache, m, cfg, f, cm)
else:
r = self.query_(metric_cache, m, cfg, f, cm)
for idx in range(4):
result[idx] = result[idx].intersection(r[idx])
return result
def _metrics(cfg: dict):
top_n = cfg['top_n']
cfg_model, cfg_dataset = cfg['model'], cfg['dataset']
cfg_results = cfg['results']
_, col_files, col_reviewers = cfg_dataset['cols']
x_df = utils \
.read_csv(cfg_dataset['path'], usecols=cfg_dataset['cols']) \
.pipe(utils.to_list_of_strings, col=col_files) \
.pipe(utils.to_list_of_strings, col=col_reviewers)
model_recommender = model.deserialize(cfg_model['path'])
metrics_df = model.metrics(model_recommender,
x_df,
cfg_dataset['cols'],
top_n=top_n)
if cfg_results.get('save', True):
utils.save_csv(cfg_results['out'], metrics_df, index=True)
def fetch_metric_for_fileset(metric, config, files, commit):
"""This function is a bulk version of fetch_codec_metric()"""
indexes = model.CodecMetricIndex.all(keys_only = True)
indexes = indexes.filter('metrics =', metric)
indexes = indexes.filter('config_name =', config)
indexes = indexes.filter('commit =', commit)
keys = [k.parent() for k in indexes]
if len(keys) == 0:
return None
metric_data = model.metrics()[metric]
results_by_file = {}
for cm in db.get(keys):
for filename, runs in cm.data.iteritems():
if filename not in files:
continue
result = results_by_file.get(filename, [])
for run in runs:
this_run_data = []
if metric_data.distortion:
this_run_data.append(run["Bitrate"])
this_run_data.append(run[metric])
result.append(this_run_data)
results_by_file[filename] = result
# Sanity checks
for filename, result in results_by_file.iteritems():
for r in result[1:]:
assert len(r) == len(result[0])
# Result is a list of lists. Sort by the first element of the nested
# list.
results_by_file[filename] = sorted(result, key=lambda x:x[0])
return results_by_file
def fetch_metric_for_fileset(metric, config, files, commit):
"""This function is a bulk version of fetch_codec_metric()"""
indexes = model.CodecMetricIndex.all(keys_only=True)
indexes = indexes.filter('metrics =', metric)
indexes = indexes.filter('config_name =', config)
indexes = indexes.filter('commit =', commit)
keys = [k.parent() for k in indexes]
if len(keys) == 0:
return None
metric_data = model.metrics()[metric]
results_by_file = {}
for cm in db.get(keys):
for filename, runs in cm.data.iteritems():
if filename not in files:
continue
result = results_by_file.get(filename, [])
for run in runs:
this_run_data = []
if metric_data.distortion:
this_run_data.append(run["Bitrate"])
this_run_data.append(run[metric])
result.append(this_run_data)
results_by_file[filename] = result
# Sanity checks
for filename, result in results_by_file.iteritems():
for r in result[1:]:
assert len(r) == len(result[0])
# Result is a list of lists. Sort by the first element of the nested
# list.
results_by_file[filename] = sorted(result, key=lambda x: x[0])
return results_by_file
def fetch_codec_metric(metric, config, filename, commit):
'''This function fetches the data for a given metric, config, filename,
commit tuple. This functionality is used multiple places, such as
CodecMetricHandler and AverageImprovementHandler.'''
indexes = model.CodecMetricIndex.all(keys_only = True)
indexes = indexes.filter('metrics =', metric)
indexes = indexes.filter('config_name =', config)
indexes = indexes.filter('files =', filename)
indexes = indexes.filter('commit =', commit)
keys = [k.parent() for k in indexes]
if len(keys) == 0:
return None
metric_data = model.metrics()[metric]
result=[]
for cm in db.get(keys):
for run in cm.data[filename]:
this_run_data = []
if metric_data.distortion:
this_run_data.append(run["Bitrate"])
this_run_data.append(run["target_bitrate"])
this_run_data.append(run[metric])
result.append(this_run_data)
# Sanity checks
for r in result[1:]:
assert len(r) == len(result[0])
# Result is a list of lists. Sort by the first element of the nested
# list.
result = sorted(result, key=lambda x:x[0])
return result
def fetch_codec_metric(metric, config, filename, commit):
'''This function fetches the data for a given metric, config, filename,
commit tuple. This functionality is used multiple places, such as
CodecMetricHandler and AverageImprovementHandler.'''
indexes = model.CodecMetricIndex.all(keys_only=True)
indexes = indexes.filter('metrics =', metric)
indexes = indexes.filter('config_name =', config)
indexes = indexes.filter('files =', filename)
indexes = indexes.filter('commit =', commit)
keys = [k.parent() for k in indexes]
if len(keys) == 0:
return None
metric_data = model.metrics()[metric]
result = []
for cm in db.get(keys):
for run in cm.data[filename]:
this_run_data = []
if metric_data.distortion:
this_run_data.append(run["Bitrate"])
this_run_data.append(run["target_bitrate"])
this_run_data.append(run[metric])
result.append(this_run_data)
# Sanity checks
for r in result[1:]:
assert len(r) == len(result[0])
# Result is a list of lists. Sort by the first element of the nested
# list.
result = sorted(result, key=lambda x: x[0])
return result
def get_adhoc_improvement(self, metrics, configs, filenames, commits):
"""Calculates the requested composite metrics and outputs as JSON"""
# Find the baseline based on the raw URL variables
parent = find_baseline(metrics, configs, filenames, commits)
# We format the end of the table with extra info
if parent:
parent_str = parent[:9]
else:
parent_str = "None found"
result = []
metrics = util.field_list(metrics)
configs = util.field_list(configs)
filenames = util.filename_list(filenames)
commits = util.field_list(commits)
# Fix for the case that a commit in commits has no parent
# In this case we choose the oldest commit as the parent, ie the one
# without a parent.
if not parent:
parent = commits[-1]
metrics_cache = model.metrics()
for m in metrics:
if metrics_cache[m].distortion:
improvement = rd_improvement
else:
improvement = mean_improvement
for cfg in configs:
baseline_data = fetch_metric_for_fileset(
m, cfg, filenames, parent)
for cm in commits:
col = [] # Each m, cfg, cm combination will be a column in
# the table
average, results = calculate_improvement(
m, cfg, filenames, cm, baseline_data, improvement)
for f, composite in results.iteritems():
col.append([f, composite])
# Build the column name
col_name = []
if len(metrics) > 1:
col_name.append(m)
if len(configs) > 1:
col_name.append(cfg)
if len(col_name) == 0 or len(commits) > 1:
col_name.append(cm[:9])
col_name = "/".join(col_name)
col.append(['OVERALL: (' + parent_str + ')', average])
result.append({'col': col_name, 'data': col})
# return the results
result = {
'baseline': parent,
'data': result,
'commits': ','.join(commits)
}
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(pretty_json(result))
def get_adhoc_improvement(self, metrics, configs, filenames, commits):
"""Calculates the requested composite metrics and outputs as JSON"""
# Find the baseline based on the raw URL variables
parent = find_baseline(metrics, configs, filenames, commits)
# We format the end of the table with extra info
if parent:
parent_str = parent[:9]
else:
parent_str = "None found"
result = []
metrics = util.field_list(metrics)
configs = util.field_list(configs)
filenames = util.filename_list(filenames)
commits = util.field_list(commits)
# Fix for the case that a commit in commits has no parent
# In this case we choose the oldest commit as the parent, ie the one
# without a parent.
if not parent:
parent = commits[-1]
metrics_cache = model.metrics()
for m in metrics:
if metrics_cache[m].distortion:
improvement = rd_improvement
else:
improvement = mean_improvement
for cfg in configs:
baseline_data = fetch_metric_for_fileset(m, cfg, filenames,
parent)
for cm in commits:
col = [] # Each m, cfg, cm combination will be a column in
# the table
average, results = calculate_improvement(
m, cfg, filenames, cm, baseline_data, improvement)
for f, composite in results.iteritems():
col.append([f, composite])
# Build the column name
col_name = []
if len(metrics) > 1:
col_name.append(m)
if len(configs) > 1:
col_name.append(cfg)
if len(col_name) == 0 or len(commits) > 1:
col_name.append(cm[:9])
col_name = "/".join(col_name)
col.append(['OVERALL: (' + parent_str + ')', average])
result.append({'col': col_name,
'data': col})
# return the results
result = {'baseline': parent,
'data': result,
'commits': ','.join(commits)
}
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(pretty_json(result))
