def _topics_in_progress(self, lesson_topics: BaseQuery) -> Set[Topic]: """loop through given lesson topics, check for rows that do not have is_finished in other rows - these are the in progress topics. """ topics = (lt.topic for lt in lesson_topics.all()) in_progress_topics = itertools.filterfalse( lambda topic: (LessonTopic.query.filter_by(topic_id=topic.id). filter_by(is_finished=True).first()), topics, ) return set(list(in_progress_topics))
def calculate_job_query_stat(query_string: str, query: BaseQuery): jobs = list(query.all()) result = {"count": len(jobs), "cpu_h": 0, "state": {}, "perf": {"avg": {}}} states = current_app.app_config.cluster["JOB_STATES"] sensors = current_app.app_config.monitoring["SENSOR_LIST"] for state in states: result["state"][state] = 0 for sensor in sensors: result["perf"]["avg"]["avg_" + sensor] = 0 if len(jobs) == 0: return result perf_count = 0 for job, tag, perf in jobs: result["cpu_h"] += (job.t_end - job.t_start) * job.num_cores / 3600 result["state"][job.state] += 1 has_perf_metric = False for sensor in sensors: try: result["perf"]["avg"]["avg_" + sensor] += getattr( perf, "avg_" + sensor) except: pass else: has_perf_metric = True if has_perf_metric: perf_count += 1 # except Exception as e: # print("error calculating stat for {0}, skipped".format(job.id), file=sys.stderr) # print(e) if perf_count > 0: for sensor in sensors: result["perf"]["avg"]["avg_" + sensor] = result["perf"]["avg"].get( "avg_" + sensor, 0) / perf_count return result
class Summary(): """list distinct instance on sa node between start_ts and end_ts """ query = None def __init__(self, start_ts, end_ts): """Build a query to get a distinct list of instance_id bewteen sart and end ts.""" az_query = db.session.query(Hypervisor).join(AvailabilityZone).\ filter(Hypervisor.availability_zone_id == AvailabilityZone.id).\ filter(AvailabilityZone.name.like("sa%")).\ with_entities(Hypervisor.id).subquery() self.query = BaseQuery([Snapshot, InstanceState], db.session()).\ filter(Snapshot.ts >= start_ts, Snapshot.ts < end_ts).\ filter(InstanceState.snapshot_id == Snapshot.id).\ with_entities(InstanceState.instance_id).\ distinct(InstanceState.instance_id).\ filter(InstanceState.hypervisor_id.in_(az_query)) def value(self): return [item[0] for item in self.query.all()]
class Summary(): """list distinct instance on sa node between start_ts and end_ts """ query = None def __init__(self, start_ts, end_ts): """Build a query to get a distinct list of instance_id bewteen sart and end ts.""" az_query = db.session.query(Hypervisor).join(AvailabilityZone).\ filter(Hypervisor.availability_zone_id == AvailabilityZone.id).\ filter(AvailabilityZone.name.like("sa%")).\ with_entities(Hypervisor.id).subquery() self.query = BaseQuery([Snapshot, InstanceState], db.session()).\ filter(Snapshot.ts >= start_ts, Snapshot.ts < end_ts).\ filter(InstanceState.snapshot_id == Snapshot.id).\ with_entities(InstanceState.instance_id).\ distinct(InstanceState.instance_id).\ filter(InstanceState.hypervisor_id.in_(az_query)) def value(self): return [item[0] for item in self.query.all()]
def apply_filter(query: BaseQuery) -> BaseQuery: for param, value in request.args.items(): if param not in {"fields", "sort", "page", "limit"}: if param == "username": query = query.filter( GithubUserInfo.username.like(f"%{value}%")) continue if param == "language": langs_list = value.split(" ") for lang in langs_list: query = query.join(GithubUserInfo.repositories).filter( GithubRepositories.languages.contains(lang)) continue if param.startswith("number_of_repositories"): if param.endswith("[gt]"): users_ids_list = [] subquery = (query.join( GithubUserInfo.repositories).distinct( GithubUserInfo.id).all()) for user in subquery: if len(user.repositories) > int(value): users_ids_list.append(user.id) query = query.filter( GithubUserInfo.id.in_(users_ids_list)) continue elif param.endswith("[gte]"): users_ids_list = [] subquery = (query.join( GithubUserInfo.repositories).distinct( GithubUserInfo.id).all()) for user in subquery: if len(user.repositories) >= int(value): users_ids_list.append(user.id) query = query.filter( GithubUserInfo.id.in_(users_ids_list)) continue elif param.endswith("[lt]"): users_ids_list = [] subquery = (query.join( GithubUserInfo.repositories).distinct( GithubUserInfo.id).all()) for user in subquery: if len(user.repositories) < int(value): users_ids_list.append(user.id) query = query.filter( GithubUserInfo.id.in_(users_ids_list)) continue elif param.endswith("[lte]"): users_ids_list = [] subquery = (query.join( GithubUserInfo.repositories).distinct( GithubUserInfo.id).all()) for user in subquery: if len(user.repositories) <= int(value): users_ids_list.append(user.id) query = query.filter( GithubUserInfo.id.in_(users_ids_list)) continue else: users_ids_list = [] subquery = (query.join( GithubUserInfo.repositories).distinct( GithubUserInfo.id).all()) for user in subquery: if len(user.repositories) == int(value): users_ids_list.append(user.id) query = query.filter( GithubUserInfo.id.in_(users_ids_list)) continue if param.startswith("date"): pass ## last_commit_date scraping functionality to be added to database ## try: ## value = datetime.strptime(value, "%d-%m-%Y").date() ## if param.endswith("[gt]"): ## query = query.filter(GithubUserInfo.date > value) ## continue ## elif param.endswith("[gte]"): ## query = query.filter(GithubUserInfo.date >= value) ## continue ## elif param.endswith("[lt]"): ## query = query.filter(GithubUserInfo.date < value) ## continue ## elif param.endswith("[lte]"): ## query = query.filter(GithubUserInfo.date <= value) ## continue ## else: ## query = query.filter(GithubUserInfo.date == value) ## continue ## except ValueError: ## continue if param.startswith("stars"): if param.endswith("[gt]"): users_ids_list = [] users_stars_dict = {} subquery = (query.join( GithubUserInfo.repositories).distinct( GithubUserInfo.id).all()) for user in subquery: users_stars_dict[user] = 0 for repo in user.repositories: users_stars_dict[user] = ( users_stars_dict[user] + repo.stars) if users_stars_dict[user] > int(value): users_ids_list.append(user.id) query = query.filter( GithubUserInfo.id.in_(users_ids_list)) continue elif param.endswith("[gte]"): users_ids_list = [] users_stars_dict = {} subquery = (query.join( GithubUserInfo.repositories).distinct( GithubUserInfo.id).all()) for user in subquery: users_stars_dict[user] = 0 for repo in user.repositories: users_stars_dict[user] = ( users_stars_dict[user] + repo.stars) if users_stars_dict[user] >= int(value): users_ids_list.append(user.id) query = query.filter( GithubUserInfo.id.in_(users_ids_list)) continue elif param.endswith("[lt]"): users_ids_list = [] users_stars_dict = {} subquery = (query.join( GithubUserInfo.repositories).distinct( GithubUserInfo.id).all()) for user in subquery: users_stars_dict[user] = 0 for repo in user.repositories: users_stars_dict[user] = ( users_stars_dict[user] + repo.stars) if users_stars_dict[user] < int(value): users_ids_list.append(user.id) query = query.filter( GithubUserInfo.id.in_(users_ids_list)) continue elif param.endswith("[lte]"): users_ids_list = [] users_stars_dict = {} subquery = (query.join( GithubUserInfo.repositories).distinct( GithubUserInfo.id).all()) for user in subquery: users_stars_dict[user] = 0 for repo in user.repositories: users_stars_dict[user] = ( users_stars_dict[user] + repo.stars) if users_stars_dict[user] <= int(value): users_ids_list.append(user.id) query = query.filter( GithubUserInfo.id.in_(users_ids_list)) continue else: users_ids_list = [] users_stars_dict = {} subquery = (query.join( GithubUserInfo.repositories).distinct( GithubUserInfo.id).all()) for user in subquery: users_stars_dict[user] = 0 for repo in user.repositories: users_stars_dict[user] = ( users_stars_dict[user] + repo.stars) if users_stars_dict[user] == int(value): users_ids_list.append(user.id) query = query.filter( GithubUserInfo.id.in_(users_ids_list)) continue return query.all() return query
def gather_muts_and_sites(self, mutations: BaseQuery, sites: BaseQuery, show_progress=True, occurrences_in: List[MutationSource] = None, intersection=None) -> MotifsData: """If occurrences_in is provided, the count of mutations will represent number of occurrences of mutations in provided sources, instead of number of distinct substitutions. """ if intersection: accepted_sites = sites.join(Mutation.affected_sites).filter( and_(*[Mutation.in_sources(source) for source in intersection])).all() else: accepted_sites = sites.all() mutations_affecting_sites = mutations.filter( Mutation.affected_sites.any(Site.types.contains(self.site_type))) muts_around_sites_with_motif = defaultdict(dict) muts_breaking_sites_motif = defaultdict(dict) sites_with_broken_motif = defaultdict(set) sites_with_motif = select_sites_with_motifs(accepted_sites, self.site_specific_motifs) if occurrences_in: def mutation_count(mut: Mutation): return sum([ mut.sources_map[source.name].get_value() if source.name in mut.sources_map else 0 for source in occurrences_in ]) else: def mutation_count(mut): return 1 is_affected = self.breaking_modes[self.mode] if show_progress: ptm_muts = mutations_affecting_sites.count() mutations_affecting_sites = tqdm(mutations_affecting_sites, total=ptm_muts) for mutation in mutations_affecting_sites: sites = mutation.affected_sites for site in sites: if site not in accepted_sites: continue for motif_name, motif in self.site_specific_motifs.items(): if site in sites_with_motif[motif_name]: count = mutation_count(mutation) muts_around_sites_with_motif[motif_name][ mutation] = count mutated_sequence = mutate_sequence(site, mutation, offset=7) if is_affected(mutated_sequence, motif): sites_with_broken_motif[motif_name].add(site) muts_breaking_sites_motif[motif_name][ mutation] = count return MotifsData( sites_with_motif=sites_with_motif, sites_with_broken_motif=sites_with_broken_motif, muts_around_sites_with_motif=muts_around_sites_with_motif, muts_breaking_sites_motif=muts_breaking_sites_motif)