def _topics_in_progress(self, lesson_topics: BaseQuery) -> Set[Topic]:
"""loop through given lesson topics, check for rows
that do not have is_finished in other rows -
these are the in progress topics.
"""
topics = (lt.topic for lt in lesson_topics.all())
in_progress_topics = itertools.filterfalse(
lambda topic: (LessonTopic.query.filter_by(topic_id=topic.id).
filter_by(is_finished=True).first()),
topics,
)
return set(list(in_progress_topics))
def calculate_job_query_stat(query_string: str, query: BaseQuery):
jobs = list(query.all())
result = {"count": len(jobs), "cpu_h": 0, "state": {}, "perf": {"avg": {}}}
states = current_app.app_config.cluster["JOB_STATES"]
sensors = current_app.app_config.monitoring["SENSOR_LIST"]
for state in states:
result["state"][state] = 0
for sensor in sensors:
result["perf"]["avg"]["avg_" + sensor] = 0
if len(jobs) == 0:
return result
perf_count = 0
for job, tag, perf in jobs:
result["cpu_h"] += (job.t_end - job.t_start) * job.num_cores / 3600
result["state"][job.state] += 1
has_perf_metric = False
for sensor in sensors:
try:
result["perf"]["avg"]["avg_" + sensor] += getattr(
perf, "avg_" + sensor)
except:
pass
else:
has_perf_metric = True
if has_perf_metric:
perf_count += 1
# except Exception as e:
# print("error calculating stat for {0}, skipped".format(job.id), file=sys.stderr)
# print(e)
if perf_count > 0:
for sensor in sensors:
result["perf"]["avg"]["avg_" + sensor] = result["perf"]["avg"].get(
"avg_" + sensor, 0) / perf_count
return result
class Summary():
"""list distinct instance on sa node between start_ts and end_ts """
query = None
def __init__(self, start_ts, end_ts):
"""Build a query to get a distinct list of instance_id bewteen sart and end ts."""
az_query = db.session.query(Hypervisor).join(AvailabilityZone).\
filter(Hypervisor.availability_zone_id == AvailabilityZone.id).\
filter(AvailabilityZone.name.like("sa%")).\
with_entities(Hypervisor.id).subquery()
self.query = BaseQuery([Snapshot, InstanceState], db.session()).\
filter(Snapshot.ts >= start_ts, Snapshot.ts < end_ts).\
filter(InstanceState.snapshot_id == Snapshot.id).\
with_entities(InstanceState.instance_id).\
distinct(InstanceState.instance_id).\
filter(InstanceState.hypervisor_id.in_(az_query))
def value(self):
return [item[0] for item in self.query.all()]
class Summary():
"""list distinct instance on sa node between start_ts and end_ts """
query = None
def __init__(self, start_ts, end_ts):
"""Build a query to get a distinct list of instance_id bewteen sart and end ts."""
az_query = db.session.query(Hypervisor).join(AvailabilityZone).\
filter(Hypervisor.availability_zone_id == AvailabilityZone.id).\
filter(AvailabilityZone.name.like("sa%")).\
with_entities(Hypervisor.id).subquery()
self.query = BaseQuery([Snapshot, InstanceState], db.session()).\
filter(Snapshot.ts >= start_ts, Snapshot.ts < end_ts).\
filter(InstanceState.snapshot_id == Snapshot.id).\
with_entities(InstanceState.instance_id).\
distinct(InstanceState.instance_id).\
filter(InstanceState.hypervisor_id.in_(az_query))
def value(self):
return [item[0] for item in self.query.all()]
def apply_filter(query: BaseQuery) -> BaseQuery:
for param, value in request.args.items():
if param not in {"fields", "sort", "page", "limit"}:
if param == "username":
query = query.filter(
GithubUserInfo.username.like(f"%{value}%"))
continue
if param == "language":
langs_list = value.split(" ")
for lang in langs_list:
query = query.join(GithubUserInfo.repositories).filter(
GithubRepositories.languages.contains(lang))
continue
if param.startswith("number_of_repositories"):
if param.endswith("[gt]"):
users_ids_list = []
subquery = (query.join(
GithubUserInfo.repositories).distinct(
GithubUserInfo.id).all())
for user in subquery:
if len(user.repositories) > int(value):
users_ids_list.append(user.id)
query = query.filter(
GithubUserInfo.id.in_(users_ids_list))
continue
elif param.endswith("[gte]"):
users_ids_list = []
subquery = (query.join(
GithubUserInfo.repositories).distinct(
GithubUserInfo.id).all())
for user in subquery:
if len(user.repositories) >= int(value):
users_ids_list.append(user.id)
query = query.filter(
GithubUserInfo.id.in_(users_ids_list))
continue
elif param.endswith("[lt]"):
users_ids_list = []
subquery = (query.join(
GithubUserInfo.repositories).distinct(
GithubUserInfo.id).all())
for user in subquery:
if len(user.repositories) < int(value):
users_ids_list.append(user.id)
query = query.filter(
GithubUserInfo.id.in_(users_ids_list))
continue
elif param.endswith("[lte]"):
users_ids_list = []
subquery = (query.join(
GithubUserInfo.repositories).distinct(
GithubUserInfo.id).all())
for user in subquery:
if len(user.repositories) <= int(value):
users_ids_list.append(user.id)
query = query.filter(
GithubUserInfo.id.in_(users_ids_list))
continue
else:
users_ids_list = []
subquery = (query.join(
GithubUserInfo.repositories).distinct(
GithubUserInfo.id).all())
for user in subquery:
if len(user.repositories) == int(value):
users_ids_list.append(user.id)
query = query.filter(
GithubUserInfo.id.in_(users_ids_list))
continue
if param.startswith("date"):
pass
## last_commit_date scraping functionality to be added to database
## try:
## value = datetime.strptime(value, "%d-%m-%Y").date()
## if param.endswith("[gt]"):
## query = query.filter(GithubUserInfo.date > value)
## continue
## elif param.endswith("[gte]"):
## query = query.filter(GithubUserInfo.date >= value)
## continue
## elif param.endswith("[lt]"):
## query = query.filter(GithubUserInfo.date < value)
## continue
## elif param.endswith("[lte]"):
## query = query.filter(GithubUserInfo.date <= value)
## continue
## else:
## query = query.filter(GithubUserInfo.date == value)
## continue
## except ValueError:
## continue
if param.startswith("stars"):
if param.endswith("[gt]"):
users_ids_list = []
users_stars_dict = {}
subquery = (query.join(
GithubUserInfo.repositories).distinct(
GithubUserInfo.id).all())
for user in subquery:
users_stars_dict[user] = 0
for repo in user.repositories:
users_stars_dict[user] = (
users_stars_dict[user] + repo.stars)
if users_stars_dict[user] > int(value):
users_ids_list.append(user.id)
query = query.filter(
GithubUserInfo.id.in_(users_ids_list))
continue
elif param.endswith("[gte]"):
users_ids_list = []
users_stars_dict = {}
subquery = (query.join(
GithubUserInfo.repositories).distinct(
GithubUserInfo.id).all())
for user in subquery:
users_stars_dict[user] = 0
for repo in user.repositories:
users_stars_dict[user] = (
users_stars_dict[user] + repo.stars)
if users_stars_dict[user] >= int(value):
users_ids_list.append(user.id)
query = query.filter(
GithubUserInfo.id.in_(users_ids_list))
continue
elif param.endswith("[lt]"):
users_ids_list = []
users_stars_dict = {}
subquery = (query.join(
GithubUserInfo.repositories).distinct(
GithubUserInfo.id).all())
for user in subquery:
users_stars_dict[user] = 0
for repo in user.repositories:
users_stars_dict[user] = (
users_stars_dict[user] + repo.stars)
if users_stars_dict[user] < int(value):
users_ids_list.append(user.id)
query = query.filter(
GithubUserInfo.id.in_(users_ids_list))
continue
elif param.endswith("[lte]"):
users_ids_list = []
users_stars_dict = {}
subquery = (query.join(
GithubUserInfo.repositories).distinct(
GithubUserInfo.id).all())
for user in subquery:
users_stars_dict[user] = 0
for repo in user.repositories:
users_stars_dict[user] = (
users_stars_dict[user] + repo.stars)
if users_stars_dict[user] <= int(value):
users_ids_list.append(user.id)
query = query.filter(
GithubUserInfo.id.in_(users_ids_list))
continue
else:
users_ids_list = []
users_stars_dict = {}
subquery = (query.join(
GithubUserInfo.repositories).distinct(
GithubUserInfo.id).all())
for user in subquery:
users_stars_dict[user] = 0
for repo in user.repositories:
users_stars_dict[user] = (
users_stars_dict[user] + repo.stars)
if users_stars_dict[user] == int(value):
users_ids_list.append(user.id)
query = query.filter(
GithubUserInfo.id.in_(users_ids_list))
continue
return query.all()
return query
def gather_muts_and_sites(self,
mutations: BaseQuery,
sites: BaseQuery,
show_progress=True,
occurrences_in: List[MutationSource] = None,
intersection=None) -> MotifsData:
"""If occurrences_in is provided, the count of mutations will
represent number of occurrences of mutations in provided
sources, instead of number of distinct substitutions.
"""
if intersection:
accepted_sites = sites.join(Mutation.affected_sites).filter(
and_(*[Mutation.in_sources(source)
for source in intersection])).all()
else:
accepted_sites = sites.all()
mutations_affecting_sites = mutations.filter(
Mutation.affected_sites.any(Site.types.contains(self.site_type)))
muts_around_sites_with_motif = defaultdict(dict)
muts_breaking_sites_motif = defaultdict(dict)
sites_with_broken_motif = defaultdict(set)
sites_with_motif = select_sites_with_motifs(accepted_sites,
self.site_specific_motifs)
if occurrences_in:
def mutation_count(mut: Mutation):
return sum([
mut.sources_map[source.name].get_value()
if source.name in mut.sources_map else 0
for source in occurrences_in
])
else:
def mutation_count(mut):
return 1
is_affected = self.breaking_modes[self.mode]
if show_progress:
ptm_muts = mutations_affecting_sites.count()
mutations_affecting_sites = tqdm(mutations_affecting_sites,
total=ptm_muts)
for mutation in mutations_affecting_sites:
sites = mutation.affected_sites
for site in sites:
if site not in accepted_sites:
continue
for motif_name, motif in self.site_specific_motifs.items():
if site in sites_with_motif[motif_name]:
count = mutation_count(mutation)
muts_around_sites_with_motif[motif_name][
mutation] = count
mutated_sequence = mutate_sequence(site,
mutation,
offset=7)
if is_affected(mutated_sequence, motif):
sites_with_broken_motif[motif_name].add(site)
muts_breaking_sites_motif[motif_name][
mutation] = count
return MotifsData(
sites_with_motif=sites_with_motif,
sites_with_broken_motif=sites_with_broken_motif,
muts_around_sites_with_motif=muts_around_sites_with_motif,
muts_breaking_sites_motif=muts_breaking_sites_motif)
