def get_examples(self, entry): from lang.dictionary.models import Example return Example.objects.filter( (Q(example_translations__translation__object=entry) & Q(example_translations__translation__subject=self)) | (Q(example_translations__translation__object=self) & Q(example_translations__translation__subject=entry)))
def search_with_text(self, text, language=None): qs = self.get_queryset().with_translations().filter( Q(text__icontains=text) & ~Q(type=ENTRY_TYPE_SENTENCE)) if language: qs = qs.filter(language=language) return qs.order_by('text')
def _load_with_translations(cls, word): word.translations.extend([ Word(t.source if t.translated == word else t.translated) for t in TranslationModel.objects.filter( Q(source=word) | Q(translated=word)) ]) return word
def update_salmon_versions(experiment: Experiment): quant_results = get_quant_results_for_experiment(experiment, filter_old_versions=False) # We first need to find the last created result and get its salmon version. # get_quant_results_for_experiment returns a set, not a queryset. last_created = None for quant_result in quant_results: if not last_created: last_created = quant_result else: if quant_result.created_at > last_created.created_at: last_created = quant_result latest_salmon_version = last_created.organism_index.salmon_version total_samples_queued = 0 for quant_result in quant_results: if latest_salmon_version != quant_result.organism_index.salmon_version: # we found a quant result associated with an experiment where we need to run salmon # hopefully each computational result is associated with a single sample for sample in quant_result.samples.all(): original_files = list(sample.original_files.all()) if not len(original_files): continue # Ensure that there's no processor jobs for these original files that the foreman # might want to retry (failed | hung | lost) has_open_processor_job = (ProcessorJob.objects.filter( original_files=original_files[0], pipeline_applied=ProcessorPipeline.SALMON).filter( Q(success=False, retried=False, no_retry=False) | Q( success=None, retried=False, no_retry=False, start_time__isnull=False, end_time=None, batch_job_id__isnull=False, ) | Q( success=None, retried=False, no_retry=False, start_time=None, end_time=None, )).exists()) if has_open_processor_job: continue create_downloader_job(original_files, force=True) total_samples_queued += 1 logger.info( "Re-ran Salmon for %d samples in experiment %s.", total_samples_queued, experiment.accession_code, )
def get(self, request, type='', **kwargs): if type not in [ 'submited', 'confirmed', 'completed', 'canceled', 'absent' ]: return HttpResponseNotFound() title_small = '' condition = '' query = Q() if type == 'submited': title_small = '已提交(等待确认)' condition = title_small query = Q(status=0) elif type == 'confirmed': title_small = '已确认' condition = title_small query = Q(status=100) elif type == 'completed': title_small = '已完成' condition = title_small query = Q(status__in=(200, )) elif type == 'canceled': title_small = '已取消' condition = title_small query = Q(status__in=(-101, -102, -103, -104, -105, -201)) rrs_o = ReserveRecord.objects.filter( main_time__reservee=request.user).filter(query) rrs = list(rrs_o) rris = [] for rr in rrs: rris.append({ 'type': ', '.join([x.name for x in rr.type.all()]), 'extra_time': [x for x in rr.time.all() if x != rr.main_time], 'color': { -201: 'danger', -101: 'danger', -102: 'danger', -103: 'danger', -104: 'danger', 0: 'info', 100: 'success', 200: 'info', }.get(rr.status, 'info'), # 不同状态不同颜色(Class)success/warning/info/danger }) rrzs = list(zip(rrs, rris)) self.extra_context = { 'title_small': title_small, 'condition': condition, 'rrzs': rrzs } return super().get(request, type=type, **kwargs)
def _get_dataset_stats(cls, range_param): """Returns stats for processed datasets""" filter_query = Q() for username in Stats.EMAIL_USERNAME_BLACKLIST: filter_query = filter_query | Q(email_address__startswith=username) filter_query = filter_query | Q( email_address__endswith="@alexslemonade.org") processed_datasets = Dataset.objects.filter( is_processed=True, email_address__isnull=False).exclude(filter_query) result = processed_datasets.aggregate( total=Count("id"), aggregated_by_experiment=Count( "id", filter=Q(aggregate_by="EXPERIMENT")), aggregated_by_species=Count("id", filter=Q(aggregate_by="SPECIES")), scale_by_none=Count("id", filter=Q(scale_by="NONE")), scale_by_minmax=Count("id", filter=Q(scale_by="MINMAX")), scale_by_standard=Count("id", filter=Q(scale_by="STANDARD")), scale_by_robust=Count("id", filter=Q(scale_by="ROBUST")), ) if range_param: # We don't save the dates when datasets are processed, but we can use # `last_modified`, since datasets aren't modified again after they are processed result["timeline"] = cls._get_intervals( processed_datasets, range_param, "last_modified").annotate(total=Count("id"), total_size=Sum("size_in_bytes")) return result
def _get_experiments_processed(self): """ total experiments with at least one sample processed """ experiments_with_sample_processed = (Experiment.objects.annotate( processed_samples_count=Count( "samples", filter=Q(samples__is_processed=True)), ).filter( Q(processed_samples_count__gt=1)).count()) experiments_with_sample_quant = (ComputedFile.objects.filter( filename="quant.sf", result__samples__is_processed=False).values_list( "result__samples__experiments", flat=True).distinct().count()) return experiments_with_sample_processed + experiments_with_sample_quant
def update_salmon_versions(experiment: Experiment): quant_results = (get_quant_results_for_experiment( experiment, filter_old_versions=False).order_by( "-organism_index__created_at").prefetch_related( "organism_index").prefetch_related("samples__original_files")) total_samples_queued = 0 latest_salmon_version = None for quant_result in quant_results: if not latest_salmon_version: # we can safely ignore the latest salmon version, that will be the first # quant result. Note we are ordering by -organism_index__created_at latest_salmon_version = quant_result.organism_index.salmon_version elif latest_salmon_version != quant_result.organism_index.salmon_version: # we found a quant result associated with an experiment where we need to run salmon # hopefully each computational result is associated with a single sample for sample in quant_result.samples.all(): original_files = list(sample.original_files.all()) if not len(original_files): continue # Ensure that there's no processor jobs for these original files that the foreman # might want to retry (failed | hung | lost) has_open_processor_job = (ProcessorJob.objects.filter( original_files=original_files[0], pipeline_applied=ProcessorPipeline.SALMON).filter( Q(success=False, retried=False, no_retry=False) | Q( success=None, retried=False, no_retry=False, start_time__isnull=False, end_time=None, nomad_job_id__isnull=False, ) | Q( success=None, retried=False, no_retry=False, start_time=None, end_time=None, )).exists()) if has_open_processor_job: continue create_downloader_job(original_files, force=True) total_samples_queued += 1 logger.info( "Re-ran Salmon for %d samples in experiment %s.", total_samples_queued, experiment.accession_code, )
def update_num_samples(self): """ Update our cache values """ aggregates = self.samples.aggregate( num_total_samples=Count("id"), num_processed_samples=Count("id", filter=Q(is_processed=True)), num_downloadable_samples=Count( "id", filter=Q(is_processed=True, organism__qn_target__isnull=False) ), ) self.num_total_samples = aggregates["num_total_samples"] self.num_processed_samples = aggregates["num_processed_samples"] self.num_downloadable_samples = aggregates["num_downloadable_samples"] self.save()
def _inject_sc_umbrellas(self): """Update supply chain list with umbrella details This method over-rides supply chain list with removing chains that are part of an umbrella and inserting unique umbrella details in their place, which is expected to be 1, per department. """ unique_tuples = self.get_unique_umbrella_tuples() qs = self.object_list.filter( Q(supply_chain_umbrella__isnull=True) | Q(id__in=[x[1] for x in unique_tuples])) # Tried to unify the query set with required fields like name, slug etc which # worked fine for pure supply chains while for chains under umbrella annotation queries # got bigger and needed Subquery as Djnago doesn't support expressions. Even with # subqueries, there were exceptions being thrown. Further investigation could resolve this # However in the interest of time left and per KISS priciples, retuning list of dicts # with info required for the page! chains = list() for item in qs.iterator(): if item.supply_chain_umbrella: u = item.supply_chain_umbrella sc = u.supply_chains.all().order_by( "-last_submission_date").first() last_updated = date_tag(sc.last_submission_date, "j M Y") sa_count = self._get_umbrella_sa_count(u) chains.append({ "name": u.name, "slug": u.slug, "sa_count": sa_count, "last_updated": last_updated, }) else: chains.append({ "name": item.name, "slug": item.slug, "sa_count": item.strategic_action_count, "last_updated": date_tag(item.last_submission_date, "j M Y"), }) return chains
def groups_with_remaining_depth(self, remaining): """ Return all groups that have a specific depth to the right. When 'remaining' is, 0: return all leaf groups 1: there is exactly one other group between this one and the tasks .. 3: maximum group depth for 'remaining' """ assert 0 <= remaining <= 3 q = Q() depth = remaining + 1 while depth < 5: q |= Q(job__project__structure_depth=depth) & Q(depth=depth - remaining) depth += 1 return self.filter(q)
def get_queryset(self): filter_query = Q() for username in EMAIL_USERNAME_BLACKLIST: filter_query = filter_query | Q(email_address__startswith=username) for domain in EMAIL_DOMAIN_BLACKLIST: filter_query = filter_query | Q(email_address__endswith=domain) processed_datasets = ( super() .get_queryset() .filter(is_processed=True, email_address__isnull=False) .exclude(filter_query) ) return processed_datasets
def make_regions_cte(cte): return Region.objects.filter(parent__isnull=True).values( "name", path=F("name"), depth=Value(0, output_field=int_field), is_planet=Value(0, output_field=int_field), ).union( cte.join(Region, parent=cte.col.name).annotate( # annotations for filter and CASE/WHEN conditions parent_name=ExpressionWrapper( cte.col.name, output_field=text_field, ), parent_depth=ExpressionWrapper( cte.col.depth, output_field=int_field, ), ).filter(~Q(parent_name="mars"), ).values( "name", path=Concat( cte.col.path, Value("\x01"), F("name"), output_field=text_field, ), depth=cte.col.depth + Value(1, output_field=int_field), is_planet=Case( When(parent_depth=0, then=Value(1)), default=Value(0), output_field=int_field, ), ), all=True, )
def get_dataset(organisms: List[Organism]): """ Builds a dataset with the samples associated with the given organisms """ dataset = {} filter_query = Q() for organism in organisms: filter_query = filter_query | Q(organisms=organism) experiments = Experiment.objects.filter(filter_query).prefetch_related("samples") for experiment in queryset_iterator(experiments): experiment_samples = experiment.samples.filter( organism__in=organisms, is_processed=True, has_raw=True ).values_list("accession_code", flat=True) dataset[experiment.accession_code] = list(experiment_samples) return dataset
def search(self, query): if not query: return self tsquery = Func(Value('russian'), Value(query), function='plainto_tsquery') return self.annotate( query_size_annotation=Func(tsquery, function='numnode', output_field=models.IntegerField()), found_annotation=Separator( '@@', Func(Value('russian'), F('content'), function='post_content_to_tsvector'), tsquery, output_field=models.BooleanField())).filter( Q(query_size_annotation=0) | Q(found_annotation=True))
def apply_filter(self, search_txt): ''' Applies filter given on all columns of the table :param search_txt: the string to filter on :return: changes self.data ''' filters = Q() #loop on each field and apply filters on them for column_nr,column_info in self.columns.items(): if column_info['searchable'] == 'true': if 'isvisible' in column_info: if column_info['isvisible'] == 'true': field_ref = column_info['data'] + '__icontains' filters.add(Q(**{field_ref: search_txt}), Q.OR) else: field_ref = column_info['data']+'__icontains' filters.add(Q(**{field_ref:search_txt}),Q.OR) self.data = self.data.filter(filters)
def non_season_players(self, season): sqs = Season.objects.filter(pk=season.pk) inactive = Player.objects \ .filter(~Q(seasons__in=sqs)) \ .annotate(active=Value(False, models.BooleanField())) \ .annotate(uname=Concat('user__last_name', Value(', '), 'user__first_name')) return inactive.order_by('uname')
def get_queryset(self): queryset = super().get_queryset() searched_item = self.request.GET.get('s') if ':' in searched_item: key, value = searched_item.split(':') if key == 'state': if value == 'actif' \ or value == 'true' \ or value == 'True': return queryset.filter(active=True) elif value == 'inactif' \ or value == 'false' \ or value == 'False': return queryset.filter(active=False) if searched_item.startswith('-'): searched_item = re.search(r'^-(?:\s?)(.*)', searched_item).group(1) terms = ~Q(name__icontains=searched_item) & ~Q(reference__icontains=searched_item) \ & ~Q(collection__name__icontains=searched_item) else: terms = Q(name__icontains=searched_item) | Q(reference__icontains=searched_item) \ | Q(collection__name__icontains=searched_item) return queryset.filter(terms)
def index(request): uid = request.user.id last_message = Subquery( Message.objects.filter(room_id=OuterRef('chat_room')).order_by( '-id').values('message')[:1]) last_message_read_by = Subquery( Message.objects.filter( room_id=OuterRef('chat_room')).order_by('-id').annotate( count=Count('read_by__id')).values('count')[:1]) last_sender = Subquery( Message.objects.filter(room_id=OuterRef('chat_room')).order_by( '-id').values('sender__first_name')[:1]) # chat_room_annotation = Subquery( # ChatRoom.objects.filter( # Q(name=OuterRef('room_name1').bitor(Q(name=OuterRef('room_name2')))) # ).values('name') # ) users_with_chat_room = User.objects.exclude(id=request.user.id).annotate( room_name1=Concat(F('id'), Value('_'), Value(uid)), room_name2=Concat(Value(uid), Value('_'), F('id'))).filter( Q(chat_rooms__name=F('room_name1')) | Q(chat_rooms__name=F('room_name2'))).annotate( chat_room=F('chat_rooms__id')) users_with_chat_room_messages = users_with_chat_room.annotate( last_message=last_message, last_message_read_by=last_message_read_by, last_sender=last_sender, ) context = { 'users': users_with_chat_room_messages | User.objects.exclude(id=request.user.id).exclude( id__in=users_with_chat_room_messages.values_list('id')), 'customer': Customer.objects.last() # 'users': User.objects.exclude(id=request.user.id) } return render(request, 'chat2/index.html', context=context)
def update_salmon_all_experiments(): """Creates a tximport job for all eligible experiments.""" eligible_experiments = (Experiment.objects.filter( technology="RNA-SEQ", num_processed_samples=0).annotate(num_salmon_versions=Count( "samples__results__organism_index__salmon_version", distinct=True, filter=Q(samples__results__processor__name=ProcessorEnum. SALMON_QUANT.value["name"]), )).filter(num_salmon_versions__gt=1)) for experiment in eligible_experiments: update_salmon_versions(experiment)
def get(self, request, version, format=None): range_param = request.query_params.dict().pop("range", "day") start_date = get_start_date(range_param) jobs = (ProcessorJob.objects.filter( created_at__gt=start_date).annotate( reason=Left("failure_reason", 80)).values("reason").annotate( job_count=Count("reason"), sample_count=Count( "original_files__samples", distinct=True, filter=Q(original_files__samples__is_processed=False), ), ).order_by("-job_count")) return paginate_queryset_response(jobs, request)
def get_queryset(self): public_result_queryset = CompendiumResult.objects.filter( result__is_public=True) latest_version = self.request.query_params.get("latest_version", False) if latest_version: version_filter = Q( primary_organism=OuterRef("primary_organism"), quant_sf_only=OuterRef("quant_sf_only"), ) latest_version = ( public_result_queryset.filter(version_filter).order_by( "-compendium_version").values("compendium_version")) return public_result_queryset.annotate( latest_version=Subquery(latest_version[:1])).filter( compendium_version=F("latest_version")) return public_result_queryset
def upvote(self, request, pk=None): post = self.get_object() user = request.user ip = request_ip(request) if user.is_anonymous: user = None vote = Activity.objects.filter(Q(posts__id=post.id), user=user, activity_type=Activity.UP_VOTE, ip=ip).exists() if not vote: post.activities.create(user=user, content_object=post, activity_type=Activity.UP_VOTE, ip=ip) post.views += 1 post.save() return Response(not vote, status=status.HTTP_204_NO_CONTENT)
def update_season_players(self, season, request): player_keys = [int(k) for k in request.POST.getlist('members')] print(f"Keys: {sorted(player_keys)}") sel_players = Player.objects.filter(pk__in=player_keys).order_by('pk') curr_players = Player.objects.filter( Q(seasons__in=[season])).order_by('pk') print(f"Selected: {sel_players.all().values_list('pk')}") print(f"Current: {curr_players.all().values_list('pk')}") to_del = curr_players.difference(sel_players).order_by('pk') to_add = sel_players.difference(curr_players).order_by('pk') for p in to_del.all(): print(f"Deleting {p}") SeasonPlayer.objects.get(season=season, player=p).delete() for p in to_add.all(): print(f"Adding {p}") SeasonPlayer.objects.create(season=season, player=p, blockmember=True).save()
def _get_dataset_stats(cls, range_param): """Returns stats for processed datasets""" result = Dataset.processed_filtered_objects.aggregate( total=Count("id"), aggregated_by_experiment=Count("id", filter=Q(aggregate_by="EXPERIMENT")), aggregated_by_species=Count("id", filter=Q(aggregate_by="SPECIES")), scale_by_none=Count("id", filter=Q(scale_by="NONE")), scale_by_minmax=Count("id", filter=Q(scale_by="MINMAX")), scale_by_standard=Count("id", filter=Q(scale_by="STANDARD")), scale_by_robust=Count("id", filter=Q(scale_by="ROBUST")), ) if range_param: # We don't save the dates when datasets are processed, but we can use # `last_modified`, since datasets aren't modified again after they are processed result["timeline"] = cls._get_intervals( Dataset.processed_filtered_objects, range_param, "last_modified" ).annotate(total=Count("id"), total_size=Sum("size_in_bytes")) return result
def get_queryset(self): """ ref https://www.django-rest-framework.org/api-guide/filtering/#filtering-against-query-parameters """ invalid_filters = check_filters( self, special_filters=[ "ids", "organism__name", "dataset_id", "experiment_accession_code", "accession_codes", ], ) if invalid_filters: raise InvalidFilters(invalid_filters) queryset = (Sample.public_objects.prefetch_related( "organism").prefetch_related( Prefetch( "results", queryset=ComputationalResult.objects.order_by("time_start") )).prefetch_related("results__processor").prefetch_related( "results__computationalresultannotation_set"). prefetch_related("results__computedfile_set").filter( **self.get_query_params_filters())) # case insensitive search https://docs.djangoproject.com/en/2.1/ref/models/querysets/#icontains filter_by = self.request.query_params.get("filter_by", None) if filter_by: queryset = queryset.filter( Q(accession_code__icontains=filter_by) | Q(title__icontains=filter_by) | Q(sex__icontains=filter_by) | Q(age__icontains=filter_by) | Q(specimen_part__icontains=filter_by) | Q(genotype__icontains=filter_by) | Q(disease__icontains=filter_by) | Q(disease_stage__icontains=filter_by) | Q(cell_line__icontains=filter_by) | Q(treatment__icontains=filter_by) | Q(race__icontains=filter_by) | Q(subject__icontains=filter_by) | Q(compound__icontains=filter_by) | Q(time__icontains=filter_by)) return queryset
def follow_recommendation(self): recommendation = Profile.objects.exclude( Q(handle__in = self.following.values('handle')) | Q(handle = self.handle) ).order_by('?')[:3] return recommendation
def generate_related_model_names(apps, schema_editor): PageManager = get_page_model_manager(apps, schema_editor) pages = PageManager.filter( Q(type='shopelectro_category') | Q(type='shopelectro_product')) pages.update(type=Page.MODEL_TYPE, related_model_name=F('type'))
def get_all(self): return self.get_queryset().with_translations().filter( Q(language=LANGUAGE_EN) & ~Q(type=ENTRY_TYPE_SENTENCE)).order_by('text').all()
def _get_job_stats(cls, jobs, range_param): start_filter = Q() if range_param: start_date = get_start_date(range_param) start_filter = start_filter | Q(start_time__gte=start_date) | Q( start_time__isnull=True) result = jobs.filter(start_filter).aggregate( total=Count("id"), successful=Count("id", filter=Q(success=True)), failed=Count("id", filter=Q(success=False)), pending=Count( "id", filter=Q( start_time__isnull=True, success__isnull=True, created_at__gt=JOB_CREATED_AT_CUTOFF, ), ), open=Count( "id", filter=Q( start_time__isnull=False, success__isnull=True, created_at__gt=JOB_CREATED_AT_CUTOFF, ), ), ) # via https://stackoverflow.com/questions/32520655/get-average-of-difference-of-datetime-fields-in-django result["average_time"] = (jobs.filter(start_filter).filter( start_time__isnull=False, end_time__isnull=False, success=True).aggregate( average_time=Avg(F("end_time") - F("start_time")))["average_time"]) if not result["average_time"]: result["average_time"] = 0 else: result["average_time"] = result["average_time"].total_seconds() if range_param: result["timeline"] = cls._get_intervals( jobs, range_param).annotate( total=Count("id"), successful=Count("id", filter=Q(success=True)), failed=Count("id", filter=Q(success=False)), pending=Count("id", filter=Q(start_time__isnull=True, success__isnull=True)), open=Count("id", filter=Q(start_time__isnull=False, success__isnull=True)), ) return result