def experiment_metric_chart(request, project_id):
try:
# Get the filter parameters from the request
params = request.query_params
metric = None
max_min_flag = None
# Check if the metric type exists
if 'max-min-flag' in params:
max_min_flag = int(params['max-min-flag'])
if max_min_flag not in [1,0]:
raise ValueError('Passed in incorrect max_min_flag')
if 'metric' in params:
metric = str(params['metric'])
metric_query = '''
select distinct value as metric
from modelchimp_machinelearningmodel ml,
jsonb_array_elements(ml.evaluation_parameters::jsonb -> 'metric_list')
where project_id = %s
'''
metric_query = metric_query % (project_id,)
metric_query_result = execute_query(metric_query)
metric_query_result = [ mqr['metric'] for mqr in metric_query_result]
if metric not in metric_query_result:
raise ValueError('Metric does not exist')
except Exception as e:
return Response("Error: %s" % e, status=status.HTTP_400_BAD_REQUEST)
query = '''
select
id,
name,
short_name,
%s(value::decimal) as value
from (select
id,
name,
CASE WHEN name = experiment_id THEN SUBSTRING(name,0,8)
ELSE name
END as short_name,
value ->> 'epoch' as epoch,
value ->> 'value' as value
from modelchimp_machinelearningmodel ml,
jsonb_array_elements(ml.evaluation_parameters::jsonb -> 'evaluation' -> '%s')
where project_id = %s ) a
group by id, name, short_name
'''
query = query % (
'max' if max_min_flag == 1 else 'min',
metric,
project_id,
)
result_raw = execute_query(query)
return Response(result_raw, status=status.HTTP_200_OK)
def get(self, request, model_id, *args, **kwargs):
ml_obj = MachineLearningModel.objects.get(id=model_id)
user = self.request.user
metric_query = f'''
select distinct json_object_keys(metric_dict::json) as metric
from modelchimp_experimentimage mei
where json_typeof(metric_dict::json) = 'object'
and ml_model_id = {model_id}
'''
epoch_query = f'''
select distinct epoch
from modelchimp_experimentimage mei
where epoch is not null
and ml_model_id = {model_id}
'''
metric_result = execute_query(metric_query)
epoch_result = execute_query(epoch_query)
return Response(
{
'metric_filter': metric_result,
'epoch_filter': epoch_result
},
status=status.HTTP_200_OK)
def experiment_duration_chart(request, project_id):
try:
# Get the filter parameters from the request
params = request.query_params
tag = None
if 'tag' in params:
tag = str(params['tag'])
tag_query = '''
select distinct value as tag
from modelchimp_experiment ml,
jsonb_array_elements(ml.durations::jsonb -> 'tag_list')
where project_id = %s
'''
tag_query = tag_query % (project_id, )
tag_query_result = execute_query(tag_query)
tag_query_result = [tag['tag'] for tag in tag_query_result]
if tag not in tag_query_result:
raise ValueError('Duration tag does not exist')
except Exception as e:
return Response("Error: %s" % e, status=status.HTTP_400_BAD_REQUEST)
query = '''
select
id,
name,
short_name,
sum(value::decimal) as value
from (select
id,
name,
CASE WHEN name = experiment_id THEN SUBSTRING(name,0,8)
ELSE name
END as short_name,
value ->> 'epoch' as epoch,
value ->> 'value' as value
from modelchimp_experiment ml,
json_array_elements(ml.durations::json -> 'duration' -> '%s')
where project_id = %s ) a
group by id, name, short_name
'''
query = query % (
tag,
project_id,
)
result_raw = execute_query(query)
return Response(result_raw, status=status.HTTP_200_OK)
def param_metric_meta(request, project_id):
'''
List of model parameters as columns to be shown on customize menu
'''
try:
project_id = int(project_id)
except Exception as e:
return Response("Error: %s" % e, status=status.HTTP_400_BAD_REQUEST)
# Check the user has permission for the project
try:
Membership.objects.get(user=request.user, project=project_id)
except Membership.DoesNotExist:
return Response(status=status.HTTP_403_FORBIDDEN)
result = dict()
query = '''
select distinct json_object_keys(parameters::json) as name
from modelchimp_experiment ml
where json_typeof(parameters::json) = 'object'
and project_id = %s
'''
query = query % (project_id, )
result['parameter'] = execute_query(query)
query = '''
select distinct value as name
from modelchimp_experiment ml,
jsonb_array_elements(ml.metrics::jsonb -> 'metric_list')
where project_id = %s
order by name
'''
query = query % (project_id, )
result_metric = execute_query(query)
result['metric'] = []
for metric in result_metric:
result['metric'].append({'name': f"{metric['name']}$0"})
result['metric'].append({'name': f"{metric['name']}$1"})
return Response(result, status=status.HTTP_200_OK)
def experiment_metric_filter(request, project_id):
try:
# Get the filter parameters from the request
params = request.query_params
if 'param' in params:
param = str(params['param'])
param_query = '''
select distinct json_object_keys(model_parameters::json) as param
from modelchimp_machinelearningmodel ml
where json_typeof(model_parameters::json) = 'object'
and project_id = %s
'''
param_query = param_query % (project_id,)
param_query_result = execute_query(param_query)
param_query_result = [ pqr['param'] for pqr in param_query_result]
if param not in param_query_result:
raise ValueError('Model Parameter does not exist')
else:
param = None
except Exception as e:
return Response("Error: %s" % e, status=status.HTTP_400_BAD_REQUEST)
query = '''
select distinct value as metric
from modelchimp_machinelearningmodel ml,
jsonb_array_elements(ml.evaluation_parameters::jsonb -> 'metric_list')
where project_id = %s %s
order by metric
'''
query = query % (
project_id,
"and model_parameters->>'%s' is not null" % (param, ) if param else ''
)
result_raw = execute_query(query)
return Response(result_raw, status=status.HTTP_200_OK)
def experiment_parameter_metric_filter(request, project_id):
query = '''
select distinct json_object_keys(parameters::json) as param
from modelchimp_experiment ml
where json_typeof(parameters::json) = 'object'
and project_id = %s
'''
query = query % (project_id, )
result_raw = execute_query(query)
return Response(result_raw, status=status.HTTP_200_OK)
def experiment_duration_filter(request, project_id):
query = '''
select distinct value as tag
from modelchimp_experiment ml,
jsonb_array_elements(ml.durations::jsonb -> 'tag_list')
where project_id = %s
order by tag
'''
query = query % (project_id, )
result_raw = execute_query(query)
return Response(result_raw, status=status.HTTP_200_OK)
def get_experiment_level_duration_data(self, project_id, tag):
duration_query = '''
select value ->> 'epoch' as epoch,
value ->> 'value' as value
from modelchimp_machinelearningmodel ml,
json_array_elements(ml.epoch_durations::json -> 'duration' -> '%s')
where id = %s
'''
duration_query = duration_query % (tag, project_id)
result = execute_query(duration_query)
return result
def get_experiment_level_metric_data(self, project_id, metric_name):
metric_query = '''
select value ->> 'epoch' as epoch,
value ->> 'value' as value
from modelchimp_machinelearningmodel ml,
json_array_elements(ml.evaluation_parameters::json -> 'evaluation' -> '%s')
where id = %s
'''
metric_query = metric_query % (metric_name, project_id)
result = execute_query(metric_query)
return result
def param_metric_data(request, project_id):
'''
Send the data of the model parameters to be displayed in the table
'''
try:
param_fields = request.data.getlist('param_fields[]')
except Exception as e:
return Response("Error: %s" % e, status=status.HTTP_400_BAD_REQUEST)
query = '''
select distinct id,key,value
from modelchimp_experiment ml, json_each_text(parameters::json)
where json_typeof(parameters::json) = 'object'
and project_id = %s
and key in (%s)
'''
query = query % (project_id, ",".join(
["'" + param + "'" for param in param_fields]))
result_raw = execute_query(query)
return Response(result_raw, status=status.HTTP_200_OK)
def experiment_parameter_metric_chart(request, project_id):
try:
# Get the filter parameters from the request
params = request.query_params
metric = None
max_min_flag = None
param = None
# Check if the metric type exists
if 'max-min-flag' in params:
max_min_flag = int(params['max-min-flag'])
if max_min_flag not in [1, 0]:
raise ValueError('Passed in incorrect max_min_flag')
if 'metric' in params:
metric = str(params['metric'])
metric_query = '''
select distinct value as metric
from modelchimp_experiment ml,
jsonb_array_elements(ml.metrics::jsonb -> 'metric_list')
where project_id = %s
'''
metric_query = metric_query % (project_id, )
metric_query_result = execute_query(metric_query)
metric_query_result = [
mqr['metric'] for mqr in metric_query_result
]
if metric not in metric_query_result:
raise ValueError('Metric does not exist')
if 'param' in params:
param = str(params['param'])
param_query = '''
select distinct json_object_keys(parameters::json) as param
from modelchimp_experiment ml
where json_typeof(parameters::json) = 'object'
and project_id = %s
'''
param_query = param_query % (project_id, )
param_query_result = execute_query(param_query)
param_query_result = [pqr['param'] for pqr in param_query_result]
if param not in param_query_result:
raise ValueError('Model Parameter does not exist')
except Exception as e:
return Response("Error: %s" % e, status=status.HTTP_400_BAD_REQUEST)
max_sql = 'DESC' if max_min_flag == 1 else ''
query = f'''
select id,
name,
short_name,
param,
value,
max_rank
from (select
id,
name,
CASE WHEN name = experiment_id THEN SUBSTRING(name,0,8)
ELSE name
END as short_name,
parameters->>'{param}' as param,
value ->> 'epoch' as epoch,
value -> 'value' as value,
row_number() OVER (PARTITION BY parameters->>'{param}' ORDER BY value -> 'value' {max_sql}) as max_rank
from modelchimp_experiment ml,
jsonb_array_elements(ml.metrics::jsonb -> 'evaluation' -> '{metric}')
where project_id = {project_id}
and parameters->>'{param}' is not null) a
where max_rank = 1
'''
result_raw = execute_query(query)
return Response(result_raw, status=status.HTTP_200_OK)
