def test_artifacts_default_configuration(self):
self.assertEqual(
CIHPCMongo.artifacts_default_configuration(),
dict(
db_name='default',
col_timers_name='timers',
col_files_name='files',
col_history_name='hist',
col_running_name='running',
))
self.assertEqual(
CIHPCMongo.artifacts_default_configuration('foobar'),
dict(
db_name='foobar',
col_timers_name='timers',
col_files_name='files',
col_history_name='hist',
col_running_name='running',
))
def prepare(self, project, base64data=None):
if base64data:
self.options = json.loads(
base64.decodebytes(base64data.encode()).decode())
else:
logger.warning(f'no options given')
self.options = dict()
logger.info(strings.to_json(self.options))
self.config = ProjectConfig.get(project)
self.mongo = CIHPCMongo.get(project)
def prepare(project, base64data=None):
if base64data:
options = json.loads(
base64.decodebytes(base64data.encode()).decode()
)
else:
logger.warning(f'no options given')
options = dict()
logger.info(strings.to_json(options))
config = ProjectConfig.get(project)
mongo = CIHPCMongo.get(project)
return namedtuple('ViewInit', ['options', 'config', 'mongo'])(options, config, mongo)
ax = sea.distplot(A,
color=pallete[0],
hist_kws=dict(alpha=0.2),
**kwargs)
ax2 = ax.twiny()
ax2 = sea.distplot(B,
color=pallete[1],
hist_kws=dict(alpha=0.5),
ax=ax2,
axlabel='log(|dur|)',
**kwargs)
plt.title('$P(N): %1.1f$%% (%1.1f%%)' % (p1 * 100, p2 * 100))
global_configuration.cfg_secret_path = '/home/jan-hybs/projects/ci-hpc/cfg/secret.yaml'
mongo = CIHPCMongo.get('flow123d')
cursor = mongo.reports.aggregate([
{
'$match': {
# 'problem.stage': 'test-full',
'problem.stage': 'test-serial',
'git.commit': 'c0906bb4e3542132da5e4cc3368c6d50976f8058',
}
},
{
'$project': {
'_id': 1,
'timers': 1,
# 'git': 1,
# 'system': 1,
import scipy as sc
from matplotlib import pyplot as plt
import seaborn as sea
# plt.style.use("dark_background")
# sea.set_style("whitegrid")
sea.set_style("darkgrid", {"axes.facecolor": ".9"})
# sea.set()
np.random.seed(1234)
# from IPython.core.display import display, HTML
# display(HTML("<style>.container { width: 75% !important; } .rendered_htmlm {max-width: 850px;}</style>"))
global_configuration.cfg_secret_path = '/home/jan-hybs/projects/ci-hpc/cfg/secret.yaml'
mongo = CIHPCMongo.get('bench-stat')
alpha_style = lambda a, s: dict(alpha=a, linestyle=s)
qs = dict(
q1=list(range(0, 6)),
q2=list(range(6, 12)),
q3=list(range(12, 18)),
q4=list(range(18, 24)),
)
qs_step = 6
qs = {'q%d'%(i+1): list(range(i*qs_step, (i+1)*qs_step)) for i, x in enumerate(range(0, 24, qs_step))}
def figsave(name:str = None, fmt='pdf', i=1, **kwargs):
def frame_view(project, base64data=''):
if base64data:
options = json.loads(
base64.decodebytes(base64data.encode()).decode()
)
else:
options = dict()
print(
strings.to_json(options)
)
config = ProjectConfig.get(project)
_ids = [objectid.ObjectId(x) for y in options['_ids'] for x in y]
field_set = config.fields.required_fields()
filter_dict = options['filters']
projection_list = set(filter_dict.keys()) | field_set
projection_list.add(config.frame_view.unwind)
db_find_fields = dict(zip(projection_list, itertools.repeat(1)))
# add _ids to selector
db_find_filters = du.filter_keys(
filter_dict,
forbidden=("", None, "*")
)
db_find_filters['_id'] = {'$in': _ids}
with Timer('db find & apply', log=logger.debug):
mongo = CIHPCMongo.get(project)
data_frame = pd.DataFrame(
mongo.aggregate(
match=db_find_filters,
project=db_find_fields,
unwind='$%s' % config.frame_view.unwind
)
)
if data_frame.empty:
return FrameView.error_empty_df(db_find_filters)
config.fields.apply_to_df(data_frame)
chart_options = du.dotdict(
y=config.frame_view.fields.timers.duration.name,
x=config.frame_view.fields.timers.name.name,
n=config.frame_view.fields.timers.path.name,
groupby={},
colorby=config.frame_view.groupby,
)
if not config.frame_view.fields.timers.path:
data_frame[config.frame_view.fields.timers.path.name] = config.frame_view.fields.timers.name.name
print(chart_options)
charts = list()
for group_values, group_keys, group_names, group_data in FrameView.group_by(data_frame, chart_options.groupby):
group_title = du.join_lists(group_names, group_values, '{} = <b>{}</b>', '<br />')
group_title = du.join_lists(group_names, group_values, '{} = <b>{}</b>', '<br />')
series = list()
colors_iter = iter(config.color_palette.copy() * 5)
for color_values, color_keys, color_names, color_data in FrameView.group_by(group_data, chart_options.colorby):
color_title = du.join_lists(color_names, color_values, '{} = {}', ', ')
color = next(colors_iter)
print(color_title)
with Timer('agg ' + color_title, log=logger.info):
# color_data = color_data.groupby(chart_options.x).agg({
# chart_options.y: 'mean',
# chart_options.n: 'first'
# }).sort_values(by=chart_options.y, ascending=False).head(50)
small_values = color_data[color_data[chart_options.y] < 0.5]
color_data = color_data[color_data[chart_options.y] >= 0.5]
small_values_grouped = small_values.groupby(chart_options.x).agg({
chart_options.y: 'mean',
}).sum()
color_data = color_data.append({
chart_options.y: small_values_grouped[chart_options.y],
chart_options.x: 'values < 0.5',
chart_options.n: 'sum of the means of the values lesser than 0.5 sec',
}, ignore_index=True)
color_data_grouped = color_data.groupby(chart_options.x).agg({
chart_options.y: {
'25%': lambda x: np.percentile(x, 25),
'75%': lambda x: np.percentile(x, 75),
},
chart_options.n: 'first',
}).reset_index()
print(color_data_grouped)
columnrange = pd.DataFrame()
columnrange['y'] = list(color_data_grouped[chart_options.x])
columnrange['n'] = list(color_data_grouped[chart_options.n]['first'])
columnrange['low'] = list(color_data_grouped[chart_options.y]['25%'])
columnrange['high'] = list(color_data_grouped[chart_options.y]['75%'])
columnrange = columnrange.sort_values(by='high', ascending=False).reset_index(drop=True)
a, b = list(columnrange['y']), list(columnrange['n'])
columnrange.drop(columns=['n'], inplace=True)
series.append(dict(
type='columnrange',
extra={
'path': dict(zip(a, b))
},
data=du.dropzero(du.fillna(columnrange.round(3))),
name='Q1-Q3 (%s)' % color_title,
color=color(0.7)),
)
color_data = color_data.reset_index()
scatter = pd.DataFrame()
scatter['y'] = list(color_data[chart_options.x])
scatter['x'] = list(color_data[chart_options.y])
scatter['n'] = list(color_data[chart_options.n])
scatter = scatter.sort_values(by='x', ascending=False).reset_index(drop=True)
a, b = list(scatter['y']), list(scatter['n'])
paths = list(scatter['n'])
scatter.drop(columns=['n'], inplace=True)
series.append(dict(
type='scatter',
extra={
'path': dict(zip(a, b)),
},
data=du.dropzero(du.fillna(scatter.round(3))),
name='mean (%s)' % color_title,
color=color(0.7)),
)
charts.append(dict(
title=group_title,
xAxis=dict(title=dict(text=None)),
yAxis=dict(title=dict(text=None)),
series=series,
))
return dict(
status=200,
data=charts
)