def generate_user_list():
"""Generate user objects with linkages."""
user_list = {}
user_collection = (Users.select().where(Users.deleted.is_null()))
project_collection = ProjectUser.select()
instrument_collection = InstrumentUser.select()
institution_collection = InstitutionUser.select()
users_expanded = prefetch(user_collection, project_collection,
instrument_collection,
institution_collection)
for user in users_expanded:
user_entry = UserQueryBase.format_user_block(user, 'simple')
user_entry['projects'] = [
proj.project.id for proj in user.projects
]
user_entry['instruments'] = [
inst.instrument.id for inst in user.instruments
]
user_entry['institutions'] = [
place.institution.id for place in user.institutions
]
user_list[user.id] = user_entry
return user_list
def generate_user_list():
"""Generate user objects with linkages."""
user_list = {}
user_collection = (Users.select().where(Users.deleted.is_null()))
proposal_collection = ProposalParticipant.select()
custodian_collection = InstrumentCustodian.select()
institution_collection = InstitutionPerson.select()
users_expanded = prefetch(user_collection, proposal_collection,
custodian_collection, institution_collection)
for user in users_expanded:
user_entry = UserQueryBase.format_user_block(user, 'simple')
user_entry['proposals'] = [
prop.proposal.id for prop in user.proposals
]
user_entry['custodian_list'] = [
inst.instrument.id for inst in user.instruments
]
user_entry['institutions'] = [
place.institution.id for place in user.institutions
]
user_list[user.id] = user_entry
return user_list
def question_list():
questions = prefetch(Question().select(), Answer)
if not len(questions):
flash('No question', 'kom')
return redirect(url_for('index'))
return render_template('list.html', questions=questions)
def selectFood():
"""Display caterering options"""
t = TenativeBooking.loadFromSession()
d = Dish.select().where(Dish.isDeleted == False)
c = Caterer.select().where(Caterer.isDeleted == False)
dishes = prefetch(d, c)
return render_template('event/caterer.html', dishes=dishes, booking=t)
def index(page=1, pagesize=10, sortby='id', sortdir='asc', search=None):
"""
/api/contacts
:return: list of contacts with paging information
"""
if search is not None:
return searchContacts(search)
contacts = Contact.select().order_by(
SQL(f"{sortby} COLLATE NOCASE {sortdir}")).paginate(page, pagesize)
emails = Email.select()
phones = Phone.select()
contacts_with_emails_iter = prefetch(contacts, emails, phones)
data = []
for contact in contacts_with_emails_iter:
data.append(
contact.serialize()
# {
# 'id': contact.id,
# 'fname': contact.fname,
# 'lname': contact.lname,
# 'dob': contact.dob,
# 'emails': [ {'id': em.id, 'email': em.email } for em in contact.emails ]
# }
)
return {
'data': data,
'numberRecords': Contact.select().count(),
'page': page,
'pagesize': pagesize
}
def donors(self):
"""Load donors from db and return it as a Donors class object."""
try:
# Get donor info from db and convert it into Donors class which
# a create_report() method to generate the report
dict_donors_gifts = {}
donors = Person.select().order_by(Person.person_name)
donations = Donation.select()
query = peewee.prefetch(donors, donations)
# Iterate over the query and collect all data into a dict
# with donors as keys and donations as lists of values
for donor in query:
dict_donors_gifts[donor.person_name] = []
for gift in donor.donations:
dict_donors_gifts[donor.person_name].append(
float(gift.donation))
# Convert the dict with donors as keys and donations as values into
# a Donors class object
return Donors([
SingleDonor(key, value)
for key, value in dict_donors_gifts.items()
])
except Exception as e:
print("Having a problems with loading the db because", e)
def get_sites(self):
sites = list()
try:
query = prefetch(Site.select(), Asset)
for site in query:
siteData = {
'siteId': site.siteId,
'siteName': site.siteName,
'siteDescr': site.siteDescr
}
assets = list()
for asset in site.refSite_prefetch:
assetData = {
'assetSerial': asset.assetSerial,
'assetConfigId': asset.assetConfigId,
'assetDescr': asset.assetDescr
}
assets.append(assetData)
siteData['assets'] = assets
sites.append(siteData)
return True, sites
except DoesNotExist as dne:
return False, dne.message
def generate_proposal_list():
"""Generate proposal objects with linkages."""
proposal_list = {}
prop_collection = (Proposals.select().order_by(Proposals.id).where(
Proposals.deleted.is_null()))
instrument_collection = (ProposalInstrument.select().order_by(
ProposalInstrument.instrument))
person_collection = (ProposalParticipant.select().order_by(
ProposalParticipant.person))
proposals_with_links = prefetch(prop_collection, instrument_collection,
person_collection)
for prop in proposals_with_links:
prop_entry = PropQueryBase.format_proposal_block(prop)
prop_entry['abstract'] = prop.abstract
prop_entry['instruments'] = [
inst.instrument.id for inst in prop.instruments
]
prop_entry['users'] = [
user_entry.person.id for user_entry in prop.users
]
proposal_list[prop.id] = prop_entry
return proposal_list
def get_scans_by_filter(ids: Optional[List[int]],
start_date: Optional[datetime],
end_date: Optional[datetime],
device_ids: Optional[List[int]],
owner_ids: Optional[List[int]], limit: Optional[int],
page: Optional[int]) -> List[dto.ScanSummary]:
# TODO Filter by device ids
# TODO Filter by owner ids
# TODO Limit / paging
ids_definite_list = ids if ids is not None else []
start_date_no_timezone = utils.remove_timezome(
start_date) if start_date is not None else None
end_date_no_timezone = utils.remove_timezome(
end_date) if end_date is not None else None
scans = models.Scan.select(
models.Scan
).where(
((ids is None) | (models.Scan.id.in_(ids_definite_list)))
& ((start_date is None) | (models.Scan.scan_time >= start_date_no_timezone))
& ((end_date is None) | (models.Scan.scan_time <= end_date_no_timezone))
) \
.join(models.Discovery, peewee.JOIN.LEFT_OUTER) \
.join(models.Device, peewee.JOIN.LEFT_OUTER) \
.join(models.Person, peewee.JOIN.LEFT_OUTER) \
.group_by(models.Scan.id) \
.order_by(models.Scan.scan_time.desc())
discoveries = models.Discovery.select(
models.Discovery, models.Device,
models.Person).join(models.Device).join(models.Person,
peewee.JOIN.LEFT_OUTER)
scans_with_discoveries = peewee.prefetch(scans, discoveries)
scan_dtos: List[dto.ScanSummary] = []
for scan in scans_with_discoveries:
discovered_devices_ids = []
discovered_devices_people_ids = []
primary_discovered_devices_ids = []
for discovery in scan.discoveries:
if discovery.device.id not in discovered_devices_ids:
discovered_devices_ids.append(discovery.device.id)
if discovery.device.owner is not None and discovery.device.owner.id not in discovered_devices_people_ids:
discovered_devices_people_ids.append(discovery.device.owner.id)
if discovery.device.is_primary and (
discovery.device.id not in primary_discovered_devices_ids):
primary_discovered_devices_ids.append(discovery.device.id)
scan_dtos.append(
dto.ScanSummary(
id=scan.id,
scan_time=utils.to_utc_datetime(scan.scan_time),
network_id=scan.network_id,
devices_discovered_count=len(discovered_devices_ids),
people_seen_count=len(discovered_devices_people_ids),
primary_devices_seen_count=len(primary_discovered_devices_ids),
))
return scan_dtos
def do_load(self):
logger.info('TIMESTAMP starting visit table load for domain %s %s' % (self.domain.name, datetime.datetime.now()))
users = User.select().where(User.domain == self.domain).order_by(User.user)
# dict with case event ids as keys, case_ids as values
cur1 = CaseEvent._meta.database.execute_sql('select case_event.id, cases.case_id from cases, case_event '
'where case_event.case_id = cases.id '
'and cases.domain_id = %d' % self.domain.id)
caseevent_caseid__dict = dict(set(cur1.fetchall()))
# dict with case event ids as keys, case parents as values
cur2 = CaseEvent._meta.database.execute_sql('select case_event.id, cases.parent_id from cases, case_event '
'where case_event.case_id = cases.id and cases.parent_id is not null '
'and cases.domain_id = %d' % self.domain.id)
caseevent_parent_dict = dict(set(cur2.fetchall()))
for usr in users:
self.delete_most_recent(usr)
logger.debug("getting visits for user %s" % usr.user)
# forms already in visit
prev_visited_forms = []
# cases and parents of cases in forms already in visit
prev_visited_case_ids = []
forms = usr.forms.select().where(~(Form.time_end >> None) & ~(Form.time_start >> None) & (Form.visit >> None)).order_by(Form.time_start)
ces = CaseEvent.select()
forms_prefetch = prefetch(forms, ces)
for frm in forms_prefetch:
case_events = frm.caseevents_prefetch
if len(case_events) > 0:
# cases updated in this form
form_case_ids = [caseevent_caseid__dict[cec.id] for cec in case_events if cec.id in caseevent_caseid__dict]
# parents of cases updated in this form
form_case_parents = [caseevent_parent_dict[cec.id] for cec in case_events if cec.id in caseevent_parent_dict]
# if cases in this form have parents or are parents of cases already in this visit, add this form to the visit
if len(set(form_case_ids) & set(prev_visited_case_ids)) > 0:
prev_visited_case_ids = prev_visited_case_ids + form_case_ids + form_case_parents
prev_visited_forms.append(frm)
# if parents of cases in this form have parents or are parents of cases already in this visit, add this form to the visit
elif len(set(prev_visited_case_ids) & set(form_case_parents)) > 0:
prev_visited_case_ids = prev_visited_case_ids + form_case_ids + form_case_parents
prev_visited_forms.append(frm)
# otherwise save the previous visit and create new lists of forms and cases for a new visit
else:
if prev_visited_forms:
previous_visit = self.create_visit(usr, prev_visited_forms)
prev_visited_case_ids = form_case_ids + form_case_parents
prev_visited_forms = [frm]
# save the last visit for this user
if prev_visited_forms:
previous_visit = self.create_visit(usr, prev_visited_forms)
def serialize_query(self, query, include_score=False):
documents = prefetch(query, Attachment, Metadata, IndexDocument, Index)
return [
self.serialize(document,
prefetched=True,
include_score=include_score)
for document in documents
]
def get_all_suppliers(cls): full_info = prefetch(Contragent.select().where(Contragent.IsSupplier == True), LegalEntity.select()) result = [] for c in full_info.order_by(+Contragent.Name): cont = c.todict() cont.update(Entities = [e.todict() for e in c.Entities.order_by(-LegalEntity.Inn)]) result.append(cont) return result
def show(username):
user = User.get_or_none(User.username == username)
if user:
user = pw.prefetch(user)
return render_template("users/show.html", user=user)
else:
flash(f"No {username} user found", "danger")
return redirect(url_for('home'))
def show(username):
user = User.select().where(User.username == username).limit(1)
if user:
user = pw.prefetch(user, Image, Donation)[0]
return render_template("users/show.html", user=user)
else:
flash("No user found")
return redirect(url_for("home"))
def test_many_to_one_prefetch(self, models, session, objects, calls,
lazy_listener):
users = pw.prefetch(
models.User.select(),
models.Address.select(),
)
list(users[0].addresses)
assert len(calls) == 0
def test_many_to_one_reverse_prefetch(self, models, session, objects,
calls):
addresses = pw.prefetch(
models.Address.select(),
models.User.select(),
)
addresses[0].user
assert len(calls) == 0
def list_packages_query(models_ref, namespace=None, media_type=None, search_query=None,
username=None):
""" List and filter repository by search query. """
Tag = models_ref.Tag
if username and not search_query:
repositories = model.repository.get_visible_repositories(username,
kind_filter='application',
include_public=True,
namespace=namespace,
limit=50)
if not repositories:
return []
repo_query = (Repository
.select(Repository, Namespace.username)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Repository.id << [repo.rid for repo in repositories]))
if namespace:
repo_query = (repo_query
.where(Namespace.username == namespace))
else:
if search_query is not None:
fields = [model.repository.SEARCH_FIELDS.name.name]
repositories = model.repository.get_app_search(search_query,
username=username,
search_fields=fields,
limit=50)
if not repositories:
return []
repo_query = (Repository
.select(Repository, Namespace.username)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Repository.id << [repo.id for repo in repositories]))
else:
repo_query = (Repository
.select(Repository, Namespace.username)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Repository.visibility == model.repository.get_public_repo_visibility(),
Repository.kind == Repository.kind.get_id('application')))
if namespace:
repo_query = (repo_query
.where(Namespace.username == namespace))
tag_query = (Tag
.select()
.where(Tag.tag_kind == Tag.tag_kind.get_id('release'))
.order_by(Tag.lifetime_start))
if media_type:
tag_query = tag_model.filter_tags_by_media_type(tag_query, media_type, models_ref)
tag_query = tag_model.tag_is_alive(tag_query, Tag)
query = prefetch(repo_query, tag_query)
return query
def newsfeed(username):
weeks_ago = datetime.date.today() - datetime.timedelta(days=7)
images = Image.select().where(Image.created_at >= weeks_ago)
users = User.select().join(
Follow,
on=(Follow.idol_id == User.id)).where(Follow.fan_id == current_user.id)
user_with_images = pw.prefetch(users, images)
return render_template('users/newsfeed.html',
user_with_images=user_with_images)
def fetch_mimetype_types(mtype):
mtype = MimetypeType.get_or_none(id=mtype)
if mtype is None:
raise ApiError('Unknown Type', status=404)
mimetypes = Mimetype.select().where(Mimetype.type == mtype)
extensions = MimetypeExtension.select()
group = prefetch(mimetypes, extensions)
return ApiResponse([x.to_dict() for x in group])
def get_persons(self) -> List[Person]:
"""
Returns all persons, known and unknown both
"""
logging.info("Getting all persons")
if not self.valid:
self.refresh()
return prefetch(self._persons_select, self._images_select,
self._encodings_select)
def get_documents_for_query(query):
if type(query) == str or type(query) == unicode:
query = u'%%%s%%' % query
documents = MOZDocument.select().where(MOZDocument.title.contains(query)).order_by(MOZDocument.title.asc())
categories = Category.select().order_by(Category.title.asc())
if not documents or len(documents) == 0 or not categories or len(categories) == 0:
return []
return prefetch(categories, documents)
return []
def run(self):
users = self.app.models.User.select().where(self.app.models.User.used_bytes >= self.app.models.User.max_bytes)
devices = self.app.models.Device.select()
users_with_devices = prefetch(users, devices)
for user in users_with_devices:
for device in user.devices:
self.app.ndsctl.deauth(device.mac_address)
device.delete_instance()
Log.info('Deauthed user #%d with %d >= %d' % (user.id, user.used_bytes, user.max_bytes))
def get(self):
snippets_query = (Snippet.select().where(Snippet.is_public).order_by(
Snippet.created_at.desc()).paginate(
int(request.args.get("page", 0)), 10))
files_query = File.select()
snippets = [
snippet for snippet in prefetch(snippets_query, files_query)
]
return snippets, 200
def paginate_query(page):
# Query objects by pagination
author_query = Author.select().paginate(page, 10)
query = prefetch(author_query, Album.select(), Track.select())
# Return serialized objects
authors = author_schema.dumps(query)
return authors
def index():
"""Lists all contacts"""
orgs = Organization.select().where(Organization.isDeleted == False)
contacts = Contact.select().where(Contact.isDeleted == False)
org_contacts = prefetch(orgs, contacts)
contacts = Contact.select().where(Contact.organization == None,
Contact.isDeleted == False)
return render_template('admin/contacts/index.html',
organizations=org_contacts,
contacts=contacts)
def getall(cls):
alli = prefetch(Ingredient.select(), CommonName.select(),
Measure.select())
return [{
"Id": i.id,
"Name": i.Name,
"ExpirationDays": i.ExpirationDays,
"CommonName": i.CommonNameId.Name,
"MeasureCaption": i.MeasureId.Caption
} for i in alli.order_by(+Ingredient.Name)]
def on_get(self, req, resp):
users_with_roles = prefetch(
models.User.select(),
models.Role.select()
)
users = []
for user in users_with_roles:
user.password = user.api_key = None
users.append(user)
req.context['result'] = {'users': users}
def serlialize_query(amount):
# Query given amount of objects to serialize
author_query = Author.select().limit(amount)
query = prefetch(author_query, Album.select())
# Return serialized objects
authors = author_schema.dumps(query)
return authors
def get_messages_from_channel(channel_id: str, date: datetime.date):
channel = TextChannel.get_by_id(channel_id)
users = User.select()
message_contents = MessageContent.select().order_by(
MessageContent.timestamp.desc())
messages = Message \
.select() \
.where((Message.channel == channel) & (Message.timestamp.day == date.day)) \
.order_by(Message.timestamp)
return prefetch(messages, users, message_contents)
def recuperer_tous_articles_par_categorie(filtres_article=[]):
logger.debug("recuperer_tous_articles_par_categorie")
les_articles = (Article.select().where(Article.lu == False).order_by(
Article.annee.desc()))
les_categories = Categorie.select().order_by(Categorie.preferee.desc(),
Categorie.nom)
c = [(x, x.articles) for x in prefetch(les_categories, les_articles)
if len(x.articles) > 0]
# filtrage des articles pour detecter certains sur le titre
logger.debug("nombre filtre %d", len(filtres_article))
for cat, articles in c:
cat.avec_filtre = False
for article in articles:
article.filtre = False
for filtre in filtres_article:
if article.title.find(filtre) != -1:
logger.debug("filtre article %s %s", filtre, article.title)
article.filtre = True
cat.avec_filtre = True
# si redis est dispo on va inserer les infos sur le statut de telechargement
status_sab = sabnzbd_util.get_status_sab_redis()
if len(status_sab) > 0:
nouvelle_liste_completed = []
nouvelle_liste_other = []
liste_id_termine = []
for z in c:
if z[0].nom == "Favoris":
z[0].ids_termine = []
nouvelle_liste_other = copy.copy(z[1])
for x in z[1]:
x.status_sabnzbd = ""
for y in x.recherche:
# logger.debug('index, title %s, id sab %s',
# x.title, y.id_sabnzbd)
if y.id_sabnzbd in status_sab:
x.status_sabnzbd = status_sab[y.id_sabnzbd]
logger.debug("index, trouve %s", x.status_sabnzbd)
if x.status_sabnzbd != "":
nouvelle_liste_other.remove(x)
if x.status_sabnzbd == "Completed":
nouvelle_liste_completed.insert(0, x)
liste_id_termine.append(x.id)
else:
nouvelle_liste_other.insert(0, x)
if ((len(nouvelle_liste_completed) > 0) or
(len(nouvelle_liste_other) > 0)) and (c[0][0].nom == "Favoris"):
c[0] = (c[0][0], nouvelle_liste_completed + nouvelle_liste_other)
c[0][0].ids_termine = liste_id_termine
return c
def index():
users = User.select()
user_images = Image.select()
user_with_images = prefetch(users, user_images)
follow = Follows.select()
# status= follow.where(Follows.fan_id==current_user.id, Follows.idol_id==user.id, Follows.approval==True)
return render_template('home.html',
users=user_with_images,
follow=follow,
Follows=Follows)
def get(self, service):
where = self.get_where(service)
if where is None:
return {
"err": "We don't understand that ID. Try an UID or a RID.",
}
# TODO pagination
schedules = Schedule.select().where(where)
callingpoints = CallingPoint.select().order_by(CallingPoint.id.asc())
results = prefetch(schedules, callingpoints)
res = []
for s in results:
res.append({
"uid": s.uid,
"rid": s.rid,
"date": str(s.start_date),
"toc": s.toc_code,
"status": s.status,
"category": s.category,
"headcode": s.headcode,
"active": s.active,
"delete": s.deleted,
"callingpoints": [
{
"id": c.id,
"station": c.tiploc,
"type": c.type,
"working_arrival": str(c.working_arrival),
"working_pass": str(c.working_pass),
"working_departure": str(c.working_departure),
"public_arrival": str(c.public_arrival),
"public_departure": str(c.public_departure),
"cancelled": c.cancelled,
} for c in s.callingpoint_set_prefetch
]
})
return res
def test_prefetch_regression(self):
a = UUIDData.create(id=uuid.uuid4(), data='a')
b = UUIDData.create(id=uuid.uuid4(), data='b')
for i in range(5):
for u in [a, b]:
UUIDRelatedModel.create(data=u, value=i)
with self.assertQueryCount(2):
query = prefetch(
UUIDData.select().order_by(UUIDData.data),
UUIDRelatedModel.select().where(UUIDRelatedModel.value < 3))
accum = []
for item in query:
accum.append((item.data, [
rel.value for rel in item.related_models_prefetch]))
self.assertEqual(accum, [
('a', [0, 1, 2]),
('b', [0, 1, 2]),
])
def GET(self):
questions = peewee.prefetch(models.Question.select(), models.Answer.select())
return self.template('site/questions.html', questions=questions)
def mod_turfs():
if hasattr(current_user, 'id'):
modid = int(current_user.id)
else:
modid = None
one_year_ago = datetime.datetime.now() - datetime.timedelta(days=365)
turfs_with_stuff = prefetch(
Turf.select(), Show.select(), Mod.select(), ShowTVDB.select())
show_info = {
s.id: (s, {
'n_mods': 0, 'mod_info': [], 'my_info': MyInfo(None, None),
'in_last_year': (
s.last_post is not None and s.last_post >= one_year_ago)
}) for s in Show.select().where(~Show.hidden)
.where(Show.deleted_at.is_null(True))
}
turfs_that_count = {TURF_LOOKUP['lead'], TURF_LOOKUP['backup']}
for turf in turfs_with_stuff:
show = turf.show
if show.hidden or show.deleted_at is not None:
continue
_, show_inf = show_info[show.id]
if turf.modid == modid:
show_inf['my_info'] = MyInfo(turf.state, turf.comments)
show_inf['mod_info'].append(ModInfo(
turf.mod.name,
turf.state,
turf.comments,
))
if turf.state in turfs_that_count:
show_inf['n_mods'] += 1
def get_name(show_and_info):
return strip_the(show_and_info[0].name).lower()
show_info = sorted(itervalues(show_info), key=get_name)
for show_id, info in show_info:
info['mod_info'] = sorted(
info['mod_info'],
key=lambda info: (TURF_ORDER.find(info.state), info.modname.lower()))
niceify = lambda c: c if c.isalpha() else '#'
firsts = [
(letter, next(iter(letter_show_infos))[0].id)
for letter, letter_show_infos
in groupby(show_info, key=lambda s: niceify(get_name(s)[0]))]
n_postses = sorted(show.n_posts() for show, info in show_info
if show.n_posts() != 'n/a')
hi_post_thresh = n_postses[int(len(n_postses) * .9)]
return render_template(
'mod_turfs.html',
shows=show_info, mods=Mod.select(), hi_post_thresh=hi_post_thresh,
now=datetime.datetime.now(), firsts=firsts,
TURF_LOOKUP=TURF_LOOKUP)
def test_many_to_one_reverse_prefetch(self, models, session, objects, calls):
addresses = models.Address.select()
users = models.User.select()
address = pw.prefetch(addresses, users).first()
address.user
assert len(calls) == 0
