def plots(request): # Check the dataset exists dataset_id = request.GET.get('dataset', None) dataset2_id = request.GET.get('dataset2', None) dataset = None dataset2 = None if dataset_id is not None: try: dataset_id = int(dataset_id) if dataset2_id is not None: dataset2_id = int(dataset2_id) except ValueError: raise Http404() datasets = HTSDataset.objects.in_bulk([dataset_id, dataset2_id]) if not datasets: raise Http404() try: dataset = datasets[dataset_id] if dataset2_id in datasets: dataset2 = datasets[dataset2_id] except KeyError: raise Http404() _assert_has_perm(request, dataset, 'view_plots') return render( request, 'plots.html', { 'default_dataset': dataset, 'second_dataset': dataset2, 'navbar_hide_dataset': True })
def plate_mapper(request, dataset_id, num_wells=None): editable = True plate_sizes = set() plate_size = namedtuple('plate_size', 'numCols numRows numWells') if dataset_id is None: if num_wells is None: return render(request, 'plate_mapper_choose_size.html') num_wells = int(num_wells) if num_wells not in STANDARD_PLATE_SIZES: raise Http404() width, height = PlateMap.plate_size_from_num_wells(num_wells) plates = [Plate(id='MASTER', width=width, height=height)] dataset = None plate_sizes.add(plate_size(width, height, num_wells)) else: plates = list( Plate.objects.filter(dataset_id=dataset_id).order_by( 'id').select_related('dataset')) if plates: dataset = plates[0].dataset else: try: dataset = HTSDataset.objects.get(pk=dataset_id) except HTSDataset.DoesNotExist: raise Http404() if dataset.deleted_date is not None: raise Http404() if dataset.owner_id != request.user.id: editable = False _assert_has_perm(request, dataset, 'view_plate_layout') for plate in plates: plate_sizes.add( plate_size(plate.width, plate.height, plate.num_wells)) plate_sizes = sorted(plate_sizes, key=lambda ps: ps.numWells) response = TemplateResponse( request, 'plate_mapper.html', { 'num_wells': num_wells, 'dataset': dataset, 'editable': editable, 'plate_sizes': plate_sizes, 'plates': plates, 'cell_lines': list(CellLine.objects.all().values('id', 'name')), 'drugs': list(Drug.objects.all().values('id', 'name')) }) return response
def download_dip_rates(request, dataset_id): try: dataset = HTSDataset.objects.get(pk=dataset_id, deleted_date=None) except HTSDataset.DoesNotExist: raise Http404() _assert_has_perm(request, dataset, 'download_data') if not license_accepted(request, dataset): return _plain_response('You must accept the dataset license to ' 'download this file') try: full_path = _generate_dip_rates(dataset) except NoDataException: return _plain_response('No data found for this request') output_filename = '{}_dip_rates.tsv'.format(dataset.name) return serve_file(request, full_path, rename_to=output_filename, content_type='text/tab-separated-values')
def download_fit_params(request, dataset_id, stat_type): file_type = 'fit_params_{}_tsv'.format(stat_type) file_name = 'fit_params_{}_{}.tsv'.format(stat_type, dataset_id) file_type_protocol = 1 param_names = { 'dip': ('aa', 'aa_obs', 'emax', 'emax_rel', 'emax_obs', 'emax_obs_rel', 'einf', 'ec50', 'ic50', 'hill'), 'viability': ('aa', 'aa_obs', 'emax', 'emax_obs', 'einf', 'ec50', 'ic50', 'hill') } try: dataset = HTSDataset.objects.get(pk=dataset_id, deleted_date=None) except (HTSDataset.DoesNotExist, ValueError): return _plain_response('This dataset does not exist, or you do not ' 'have permission to access it.') _assert_has_perm(request, dataset, 'download_data') if not license_accepted(request, dataset): return _plain_response('You must accept the dataset license to ' 'download this file') mod_date = timezone.now() file = _cached_file(dataset, file_type, file_type_protocol) # Additional cache invalidation: curve fits were generated after the # cached file if file: if file.creation_date < CurveFitSet.objects.get( dataset_id=dataset_id, stat_type=stat_type).calculation_end: file = None if file: full_path = file.file.name else: try: # Fetch the DIP rates from the DB base_params = df_curve_fits(dataset.id, stat_type, drug_ids=None, cell_line_ids=None) except NoDataException: return _plain_response( 'The requested parameter set does not exist for the ' 'specified dataset') # Fit Hill curves and compute parameters fp = fit_params_from_base(base_params, custom_ic_concentrations={50}, custom_ec_concentrations={50}, include_auc=False, include_aa=True, include_hill=True, include_emax=True, include_einf=True, include_response_values=False) fp.reset_index('dataset_id', drop=True, inplace=True) # Remove -ve AA values fp.loc[fp['aa'] < 0.0, 'aa'] = np.nan # Filter for the default list of parameters only fp = fp.filter(items=param_names[stat_type]) full_path = os.path.join(settings.DOWNLOADS_ROOT, file_name) fp.to_csv(full_path, sep='\t') df, created = HTSDatasetFile.objects.get_or_create( dataset=dataset, file_type=file_type, defaults={ 'file_type_protocol': file_type_protocol, 'file': full_path }) if not created: df.file_type_protocol = file_type_protocol df.file = full_path df.creation_date = mod_date df.save() output_filename = '{}_{}_params.tsv'.format(dataset.name, stat_type) return serve_file(request, full_path, rename_to=output_filename, content_type='text/tab-separated-values')
def ajax_get_dataset_groupings(request, dataset_id, dataset2_id=None): dataset_ids = [dataset_id] if dataset2_id is not None: dataset_ids.append(dataset2_id) plates = Plate.objects.filter( dataset__in=dataset_ids).select_related('dataset') datasets = set([p.dataset for p in plates]) if len(datasets) == 0: # Maybe we just have no plates, get datasets to check permissions datasets = HTSDataset.objects.filter(id__in=dataset_ids) if len(datasets) == 0: raise Http404() elif len(datasets) != len(dataset_ids): raise Http404() for dataset in datasets: _assert_has_perm(request, dataset, 'view_plots') if not license_accepted(request, dataset): return HttpResponse(LICENSE_UNSIGNED.format(dataset.name), status=400) if len(plates) == 0: return HttpResponse( 'This dataset has no plate files. Data will need to be added ' 'before plots can be used on this dataset.', status=400) groupings_dict = dataset_groupings(list(datasets)) cell_line_ids = [cl['id'] for cl in groupings_dict['cellLines']] drug_ids = [] for dr in groupings_dict['drugs']: drug_ids.append(dr['id']) if isinstance(dr['id'], int) else \ drug_ids.extend(dr['id']) cell_line_tags, drug_tags = _get_tags(request, cell_line_ids, drug_ids) groupings_dict['drugTags'] = [] groupings_dict['cellLineTags'] = [] last_cat = None append_to = groupings_dict['drugTags'] for tag in drug_tags: if tag.tag_category != last_cat: groupings_dict['drugTags'].append({ 'optgroup': tag.tag_category, 'options': [] }) last_cat = tag.tag_category append_to = groupings_dict['drugTags'][-1]['options'] append_to.append({'id': tag.id, 'name': tag.tag_name}) special_tags = { 'optgroup': 'Special Tags', 'options': [{ 'id': TAG_EVERYTHING_ELSE, 'name': 'Everything else' }] } groupings_dict['drugTags'].append(special_tags) last_cat = None append_to = groupings_dict['cellLineTags'] for tag in cell_line_tags: if tag.tag_category != last_cat: groupings_dict['cellLineTags'].append({ 'optgroup': tag.tag_category, 'options': [] }) last_cat = tag.tag_category append_to = groupings_dict['cellLineTags'][-1]['options'] append_to.append({'id': tag.id, 'name': tag.tag_name}) groupings_dict['cellLineTags'].append(special_tags) if groupings_dict['singleTimepoint'] is False: groupings_dict['plates'] = [{ 'id': p.id, 'name': p.name } for p in plates] else: groupings_dict['plates'] = [] return JsonResponse(groupings_dict)
def _dose_response_plot(request, dataset, dataset2_id, permission_required, drug_id, cell_line_id, plot_type, template=default_plotly_template): if dataset2_id is not None: try: dataset2 = HTSDataset.objects.get(pk=dataset2_id) except HTSDataset.DoesNotExist: raise Http404() _assert_has_perm(request, dataset2, permission_required) if dataset.name == dataset2.name: return HttpResponse( 'Cannot compare two datasets with the same ' 'name. Please rename one of the datasets.', status=400) datasets = dataset if not dataset2_id else [dataset, dataset2] color_by = request.GET.get('colorBy', 'off') if color_by == 'off': color_by = None drug_tag_ids = [int(dt) for dt in request.GET.getlist('dT')] color_groups = None aggregate_drugs = request.GET.get('aggregateDrugs', False) == "true" if not drug_id and drug_tag_ids: drug_id, drug_groups = _process_aggreate( request, 'drugs', drug_tag_ids, aggregate_drugs or color_by == 'dr', datasets) if aggregate_drugs: aggregate_drugs = drug_groups if color_by == 'dr': color_groups = drug_groups cell_line_tag_ids = [int(ct) for ct in request.GET.getlist('cT')] aggregate_cell_lines = request.GET.get('aggregateCellLines', False) \ == "true" if not cell_line_id and cell_line_tag_ids: cell_line_id, cell_line_groups = _process_aggreate( request, 'cell_lines', cell_line_tag_ids, aggregate_cell_lines or color_by == 'cl', datasets) if aggregate_cell_lines: aggregate_cell_lines = cell_line_groups if color_by == 'cl': color_groups = cell_line_groups if color_groups: color_groups = _make_tags_unique(color_groups) elif color_by == 'cl': # The tags will just be the cell lines themselves color_groups = { cl.name: [cl.name] for cl in CellLine.objects.filter( id__in=cell_line_id).order_by('name') } elif color_by == 'dr': # Ditto for drugs color_groups = { dr.name: [dr.name] for dr in Drug.objects.filter(id__in=drug_id).order_by('name') } if color_groups and len(color_groups) > MAX_COLOR_GROUPS: return HttpResponse( 'Cannot plot using more than {} unique colors. Please remove ' 'some entries or turn off coloring to proceed.'.format( MAX_COLOR_GROUPS), status=400) if not cell_line_id: return HttpResponse('Please enter at least one cell line', status=400) if not drug_id: return HttpResponse('Please enter at least one drug', status=400) response_metric = request.GET.get('drMetric', 'dip') if response_metric not in ('dip', 'viability', 'compare'): return HttpResponse( 'Unknown metric. Supported values: dip, ' 'viability, compare.', status=400) def _setup_dr_par(name, needs_toggle=False): if needs_toggle and \ request.GET.get(name + 'Toggle', 'off') != 'on': return None par_name = request.GET.get(name, None) if par_name is not None and '_custom' in par_name: rep_value = request.GET.get(name + 'Custom', None) if int(rep_value) < 0: raise ValueError() par_name = par_name.replace('_custom', rep_value) return par_name try: dr_par = _setup_dr_par('drPar') except ValueError: return HttpResponse( 'Parameter custom value ' 'needs to be a positive integer', status=400) try: dr_par_two = _setup_dr_par('drParTwo', needs_toggle=True) except ValueError: return HttpResponse( 'Parameter two custom value ' 'needs to be a positive integer', status=400) try: dr_par_order = _setup_dr_par('drParOrder', needs_toggle=True) except ValueError: return HttpResponse( 'Parameter order custom value ' 'needs to be a positive integer', status=400) # 'compare' plots are only available for one dataset and metric if response_metric == 'compare': if dataset2_id is not None: return HttpResponse( '"compare" mode not compatible with two ' 'datasets', status=400) if dr_par_two is not None: return HttpResponse( 'Parameter two not available with "compare" ' 'mode', status=400) if dr_par_order is not None: return HttpResponse( 'Parameter order not available with "compare" ' 'mode', status=400) if plot_type == 'drc': return HttpResponse( 'Dose response curves not available with ' '"compare" mode', status=400) if dr_par.endswith('_rel'): return HttpResponse( 'Relative metrics are not available with ' '"compare" mode', status=400) # Work out any non-standard parameters we need to calculate # e.g. non-standard IC concentrations ic_concentrations = set() ec_concentrations = set() e_values = set() e_rel_values = set() regexes = { IC_REGEX: ic_concentrations, EC_REGEX: ec_concentrations, E_REGEX: e_values, E_REL_REGEX: e_rel_values } need_aa = False need_hill = False need_emax = False need_einf = False for param_idx, param in enumerate((dr_par, dr_par_two, dr_par_order)): if param is None: continue if param == 'label' and param_idx == 2: continue if param == 'aa_obs': continue if param == 'aa': need_aa = True continue if param == 'hill': need_hill = True continue if param.startswith('emax'): need_emax = True continue if param == 'einf': need_einf = True continue for regex, value_list in regexes.items(): match = regex.match(param) if not match: continue try: value = int(match.groups(0)[0]) if value < 0 or value > 100: raise ValueError() value_list.add(value) break except ValueError: return HttpResponse( 'Invalid custom value - must be ' 'an integer between 1 and 100', status=400) else: return HttpResponse('Unknown parameter: {}'.format(param), status=400) dataset_ids = dataset.id if dataset2_id is None else [ dataset.id, dataset2_id ] # Fit Hill curves and compute parameters if response_metric == 'compare': all_metrics = ('dip', 'viability') else: all_metrics = (response_metric, ) try: base_params = [ df_curve_fits(dataset_ids, metric, drug_id, cell_line_id) for metric in all_metrics ] except NoDataException: return HttpResponse( 'No data found for this request. This drug/cell ' 'line/assay combination may not exist.', status=400) include_response_values = False ctrl_resp_data = None expt_resp_data = None if plot_type == 'drc': single_drug = len( base_params[0].index.get_level_values('drug').unique()) == 1 single_cl = len( base_params[0].index.get_level_values('cell_line').unique()) \ == 1 if single_cl and single_drug: try: if response_metric == 'dip': ctrl_resp_data, expt_resp_data = df_dip_rates( dataset_id=dataset_ids, drug_id=drug_id, cell_line_id=cell_line_id, use_dataset_names=True) else: expt_resp_data, ctrl_resp_data = _get_viability_scores( datasets, drug_id, cell_line_id, viability_time=base_params[0]._viability_time) except NoDataException: return HttpResponse( 'No data found for this request. This drug/' 'cell line/assay combination may not exist.', status=400) include_response_values = True need_emax = True ic_concentrations = {50} ec_concentrations = {50} with warnings.catch_warnings(record=True) as w: fit_params = [ fit_params_from_base( base_param_set, ctrl_resp_data=ctrl_resp_data, expt_resp_data=expt_resp_data, include_response_values=include_response_values, custom_ic_concentrations=ic_concentrations, custom_ec_concentrations=ec_concentrations, custom_e_values=e_values, include_aa=need_aa, include_hill=need_hill, include_emax=need_emax, include_einf=need_einf) for base_param_set in base_params ] # Currently only care about warnings if plotting AA if plot_type == 'drpar' and (dr_par == 'aa' or dr_par_two == 'aa'): w = [i for i in w if issubclass(i.category, AAFitWarning)] if w: return HttpResponse(w[0].message, status=400) if response_metric == 'compare': # Create new dataframe import pandas as pd fit_params = pd.concat([ fit_params[0]['label'], fit_params[0][dr_par], fit_params[1][dr_par] ], join='inner', axis=1) fit_params.columns = [ 'label', 'dip__{}'.format(dr_par), 'viability__{}'.format(dr_par) ] fit_params._viability_time = base_params[1]._viability_time fit_params._drmetric = 'compare' dr_par, dr_par_two = fit_params.columns[1:] else: fit_params = fit_params[0] if plot_type == 'drpar': if dr_par is None: return HttpResponse('Dose response parameter is a required field', status=400) try: plot_fig = plot_drc_params( fit_params, fit_param=dr_par, fit_param_compare=dr_par_two, fit_param_sort=dr_par_order, aggregate_cell_lines=aggregate_cell_lines, aggregate_drugs=aggregate_drugs, color_by=color_by, color_groups=color_groups, multi_dataset=dataset2_id is not None, template=template) except CannotPlotError as e: return HttpResponse(e, status=400) else: dip_absolute = request.GET.get('drcType', 'rel') == 'abs' plot_fig = plot_drc(fit_params, is_absolute=dip_absolute, color_by=color_by, color_groups=color_groups, template=template) return plot_fig
def ajax_get_plot(request, file_type='json'): if file_type == 'csv': permission_required = 'download_data' else: permission_required = 'view_plots' try: plot_type = request.GET['plotType'] template = request.GET.get('theme', default_plotly_template) if template not in ALLOWED_TEMPLATES: return HttpResponse('Please select an allowed template', status=400) dataset_id = int(request.GET['datasetId']) dataset2_id = request.GET.get('dataset2Id', None) if dataset2_id == "": dataset2_id = None if dataset2_id is not None: dataset2_id = int(dataset2_id) cell_line_id = request.GET.getlist('c') drug_id = request.GET.getlist('d') cell_line_id = [int(cl) for cl in cell_line_id] drug_ids = [] for dr in drug_id: try: drug_ids.append(int(dr)) except ValueError: drug_ids.append([int(d) for d in dr.split(",")]) drug_id = drug_ids assay = request.GET.get('assayId') yaxis = request.GET.get('logTransform', 'None') except (KeyError, ValueError): raise Http404() try: dataset = HTSDataset.objects.get(pk=dataset_id) except HTSDataset.DoesNotExist: raise Http404() _assert_has_perm(request, dataset, permission_required) if not license_accepted(request, dataset): return HttpResponse(LICENSE_UNSIGNED.format(dataset.name), status=400) if plot_type == 'tc': if len(drug_id) != 1 or len(cell_line_id) != 1: return HttpResponse( 'Please select exactly one cell line and ' 'drug for time course plot', status=400) try: df_data = df_doses_assays_controls(dataset=dataset, drug_id=drug_id, cell_line_id=cell_line_id, assay=assay) except NoDataException: return HttpResponse( 'No data found for this request. This ' 'drug/cell line/assay combination may not ' 'exist.', status=400) if assay is None: assay = df_data.assays.index.get_level_values('assay')[0] overlay_dip_fit = request.GET.get('overlayDipFit', 'false') == 'true' if overlay_dip_fit and assay != df_data.dip_assay_name: return HttpResponse( 'Can only overlay DIP rate on cell ' 'proliferation assays', status=400) plot_fig = plot_time_course(df_data, log_yaxis=yaxis == 'log2', assay_name=assay, show_dip_fit=overlay_dip_fit, subtitle=dataset.name, template=template) elif plot_type in ('drc', 'drpar'): if all(isinstance(d, int) for d in drug_id): plot_fig = _dose_response_plot(request, dataset, dataset2_id, permission_required, drug_id, cell_line_id, plot_type, template) else: if dataset2_id is not None: return HttpResponse( 'Please select a single dataset at a time to view drug ' 'combination heat plots', status=400) plot_fig = _drug_combination_heatmap(request, dataset, drug_id, cell_line_id, template) if isinstance(plot_fig, HttpResponse): return plot_fig elif plot_type == 'qc': qc_view = request.GET.get('qcView', None) if qc_view == 'ctrldipbox': ctrl_dip_data = df_ctrl_dip_rates(dataset_id) if ctrl_dip_data is None: return HttpResponse( 'No control wells with DIP rates ' 'available were detected in this ' 'dataset.', status=400) plot_fig = plot_ctrl_dip_by_plate(ctrl_dip_data, template=template) elif qc_view == 'ctrlcellbox': # Try to fetch from cache cache_key = f'dataset_{dataset_id}_plot_ctrlcellbox' cur_plotver = 1 plot_cached = cache.get(cache_key) if plot_cached: plot_fig = plot_cached['plot_fig'] if plot_cached is None or plot_cached['dataset_last_modified'] < dataset.modified_date or \ plot_cached['plot_version'] < cur_plotver: # Create plot groupings = dataset_groupings(dataset) if not groupings['singleTimepoint']: return HttpResponse( 'This plot type is only available for ' 'single time-point datasets', status=400) try: df_data = df_control_wells(dataset_id=dataset, assay=assay) except NoDataException: return HttpResponse('No data found for this request.', status=400) if (df_data['value'] == 100.0).all(): return HttpResponse( 'The raw data for this dataset is given as relative viability, so no control ' 'wells are available', status=400) plot_fig = plot_ctrl_cell_counts_by_plate( df_data, subtitle=dataset.name, template=template) # Push to cache cache.set( cache_key, { 'dataset_last_modified': dataset.modified_date, 'plot_version': cur_plotver, 'plot_fig': plot_fig }) elif qc_view == 'dipplatemap': plate_id = request.GET.get('plateId', None) try: plate_id = int(plate_id) except ValueError: return HttpResponse('Integer plateId required', status=400) pl_data = ajax_load_plate(request, plate_id, return_as_platedata=True, use_names=True) plot_fig = plot_plate_map(pl_data, color_by='dip_rates', template=template) else: return HttpResponse('Unimplemented QC view: {}'.format(qc_view), status=400) else: return HttpResponse('Unimplemented plot type: %s' % plot_type, status=400) as_attachment = request.GET.get('download', '0') == '1' if file_type == 'json': j = json.dumps(plot_fig, cls=PlotlyJSONEncoder) response = HttpResponse(j, content_type='application/json') elif file_type == 'csv': response = HttpResponse(plotly_to_dataframe(plot_fig).to_csv(), content_type='text/csv') elif file_type == 'html': template = 'plotly_plot{}.html'.format( '_standalone' if as_attachment else '') context = { 'data': json.dumps(plot_fig, cls=PlotlyJSONEncoder), 'page_title': strip_tags(plot_fig['layout']['title']['text']) } if as_attachment: context['plotlyjs'] = get_plotlyjs() response = render(request, template, context) else: return HttpResponse('Unknown file type: %s' % file_type, status=400) if as_attachment: try: title = plot_fig['layout']['title']['text'] except KeyError: title = 'Plot' response['Content-Disposition'] = \ 'attachment; filename="{}.{}"'.format(strip_tags(title), file_type) return response
def ajax_load_plate(request, plate_id, extra_return_args=None, return_as_platedata=False, use_names=False): if not request.user.is_authenticated and settings.LOGIN_REQUIRED: return JsonResponse({}, status=401) try: p = Plate.objects.filter(id=plate_id).select_related('dataset').get() except Plate.DoesNotExist: raise Http404() _assert_has_perm(request, p.dataset, 'view_plate_layout') if not license_accepted(request, p.dataset): return HttpResponse(LICENSE_UNSIGNED.format(p.dataset.name), status=400) field_ext = '__name' if use_names else '_id' # Blank well data wells = [] for cl in range(p.num_wells): wells.append({ 'cellLine': None, 'drugs': AutoExtendList(), 'doses': AutoExtendList(), 'dipRate': None }) # Populate cell lines cl_query = Well.objects.filter(plate_id=plate_id).values( 'well_num', 'cell_line' + field_ext) for w in cl_query: wells[w['well_num']]['cellLine'] = w['cell_line' + field_ext] # Populate drugs drugs = WellDrug.objects.filter(well__plate_id=plate_id).values( 'well__well_num', 'drug' + field_ext, 'order', 'dose') for dr in drugs: # prefetched above wells[dr['well__well_num']]['drugs'][dr['order']] = dr['drug' + field_ext] wells[dr['well__well_num']]['doses'][dr['order']] = dr['dose'] # Populate DIP rates for ws in WellStatistic.objects.filter(well__plate_id=plate_id, stat_name='dip_rate').values( 'well__well_num', 'value'): # Need to remove NaNs for proper JSON support wells[ws['well__well_num']]['dipRate'] = ws['value'] if \ ws['value'] is not None and not math.isnan(ws['value']) else None plate = { 'datasetName': p.dataset.name, 'plateId': p.id, 'plateName': p.name, 'numCols': p.width, 'numRows': p.height, 'wells': wells } if return_as_platedata: return PlateData.from_dict(plate) return_dict = {'success': True, 'plateMap': plate} if extra_return_args is not None: return_dict.update(extra_return_args) return JsonResponse(return_dict)