def active_packages(self, load_eager=True):
query = Session.query(Package).\
filter_by(state=vdm.sqlalchemy.State.ACTIVE).\
join('groups').filter_by(id=self.id)
if load_eager:
query = query.options(eagerload_all('package_tags.tag'))
query = query.options(eagerload_all('package_resources_all'))
return query
def get(cls, reference):
'''Returns a package object referenced by its id or name.'''
query = Session.query(cls).filter(cls.id==reference)
query = query.options(eagerload_all('package_tags.tag'))
query = query.options(eagerload_all('resource_groups_all.resources_all'))
pkg = query.first()
if pkg == None:
pkg = cls.by_name(reference)
return pkg
def get(cls, reference):
'''Returns a package object referenced by its id or name.'''
query = Session.query(cls).filter(cls.id==reference)
query = query.options(eagerload_all('package_tags.tag'))
query = query.options(eagerload_all('resource_groups_all.resources_all'))
pkg = query.first()
if pkg == None:
pkg = cls.by_name(reference)
return pkg
def active_packages(self, load_eager=True):
query = Session.query(Package).\
filter_by(state=vdm.sqlalchemy.State.ACTIVE).\
join('package_group_all', 'group').filter_by(id=self.id)
if load_eager:
query = query.options(eagerload_all('package_tags.tag'))
query = query.options(
eagerload_all('resource_groups_all.resources_all'))
return query
def read(self, id):
query = model.Session.query(model.Tag)
query = query.filter(model.Tag.name==id)
query = query.options(eagerload_all('package_tags.package'))
query = query.options(eagerload_all('package_tags.package.package_tags.tag'))
query = query.options(eagerload_all('package_tags.package.resource_groups_all.resources_all'))
c.tag = query.first()
if c.tag is None:
abort(404)
return render('tag/read.html')
def read(self, id):
query = model.Session.query(model.Tag)
query = query.filter(model.Tag.name == id)
query = query.options(eagerload_all("package_tags.package"))
query = query.options(eagerload_all("package_tags.package.package_tags.tag"))
query = query.options(eagerload_all("package_tags.package.resource_groups_all.resources_all"))
c.tag = query.first()
if c.tag is None:
abort(404)
return render("tag/read.html")
def read(self, id):
query = model.Session.query(model.Tag)
query = query.filter(model.Tag.name==id)
query = query.options(eagerload_all('package_tags.package'))
query = query.options(eagerload_all('package_tags.package.package_tags.tag'))
query = query.options(eagerload_all('package_tags.package.package_resources_all'))
c.tag = query.first()
if c.tag is None:
abort(404)
return render('tag/read.html')
def active_packages(self, load_eager=True):
query = (
Session.query(Package)
.filter_by(state=vdm.sqlalchemy.State.ACTIVE)
.join("package_group_all", "group")
.filter_by(id=self.id)
)
if load_eager:
query = query.options(eagerload_all("package_tags.tag"))
query = query.options(eagerload_all("resource_groups_all.resources_all"))
return query
def _get_customer(self, session, customer_id):
customer = (
session.query(model.Customer).
options(
orm.eagerload_all('payments'),
orm.eagerload_all('archived_payments')
).
get(customer_id)
)
if not customer:
raise exceptions.NoSuchResource('no customer with id %s' % customer_id)
return customer
def _process_delete(self, session, spec):
query = (
session.query(model.Invoice).join('manifest', 'plane').join('customer').join('item').
filter(model.Manifest.manifest_id == spec['manifest_id']).
filter(model.Plane.plane_id == spec['plane_id'])
)
if spec.get('customer_id'):
query = query.filter(model.Customer.customer_id == spec['customer_id'])
if spec.get('item_id'):
query = query.filter(model.Item.item_id == spec['item_id'])
# Note that model.maintain_balances is invoked as a
# side-effect, which is why it's not just a delete-call here.
invoices = query.all()
for invoice in invoices:
session.delete(invoice)
# Delete the entire manifest if no customer was specified.
if not (spec.get('customer_id') or spec.get('item_id')):
session.execute(model.Manifest.__table__.delete().
where(model.Manifest.manifest_id == spec['manifest_id']).
where(model.Manifest.plane_id == spec['plane_id']))
session.commit()
if spec.get('customer_id'):
return (
session.query(model.Invoice).join('manifest').join('item').
options(
orm.contains_eager('manifest'),
orm.contains_eager('item')
).
filter(model.Manifest.manifest_id == spec['manifest_id']).
filter(model.Invoice.customer_id == spec['customer_id'])
).all()
else:
return (
session.query(model.Manifest).
options(
orm.eagerload_all('invoices.item'),
orm.eagerload_all('invoices.customer')
).
filter(model.Manifest.plane_id == spec['plane_id'])
).all()
def revision_detail(request, rev):
q = Revision.q.filter(Revision.svn_id <= rev)
revision, previous = q.order_by(Revision.svn_id.desc())[:2]
etag = check_cache(request, 'revision_detail', revision, previous)
fail_count = func.count(Result.fail)
failing = (TestFile.failing(revision).group_by(TestFile.id).
add_column(fail_count).order_by(fail_count.asc()))
q = Assertion.q.join(Result).filter_by(fail=True, revision=revision)
q = q.order_by(Assertion.test_id).options(eagerload_all('test.testfile'))
# Explicit JOIN; otherwise SA generates horrid LEFT OUTER JOINs.
q = q.join(Test).join(TestFile)
# We get TestFile.name from the `failing` query above, so we want
# to look up tests & assertions keyed by a testfile.
failures = {}
for test, assertions in itertools.groupby(q, attrgetter('test')):
fail_list = failures.setdefault(test.testfile.id, [])
fail_list.append((test, list(assertions)))
return Context({'revision': revision, 'diff': revision.diff,
'failing': failing, 'failures': failures,
'broken': revision.broken_tests},
headers={'Etag': etag})
def group_list(self, show_hidden=False, list_users=False):
"""Return a simple list of the available groups"""
groups = query(Group).options(eagerload_all('users_assoc.user')).all()
me = query(User).get(self.myself_id)
out = []
for grp in groups:
myug = [ug for ug in grp.users_assoc if ug.user_id == me.id]
res = {'id': grp.id,
'name': grp.name,
'member': me in grp.users,
'hidden': False,
'admin': False}
if grp.hidden:
# Only for global-admin or members, if we're asked to hide
# hidden groups
if not show_hidden and not me.is_admin and not myug:
continue
res['hidden'] = True
if len(myug):
res['cryptgroupkey'] = myug[0].cryptgroupkey
if myug[0].is_admin:
res['admin'] = True
res['members'] = [(u.user_id, u.user.username, u.is_admin)
for u in grp.users_assoc]
out.append(res)
return vaultMsg(True, 'Here is the list of groups', {'list': out})
def get_user(self, **kwargs):
id = kwargs["id"]
with self.session_scope() as s:
u = s.query(user.User).options(
eagerload_all('*')).filter(user.User.id == id).one()
s.expunge(u)
return u
def get_tests_query(filter_object=None):
# get the candidate tests to page through
query = tst.CandidateTest.query
# leave out alta tests
query = query.join('candidate_profile','account')
#query = query.filter(tst.Account.id!=352)
query = query.filter(tst.Account.id==957)
# leave out archived
query = query.join('test','test','type')
query = query.join('version')
query = query.filter(and_(tst.AccountTest.archived==False, tst.Test.archived==False, tst.TestVersion.archived==False))
# get only reading comprehension
query = query.filter(tst.TestType.id==6)
# ensure there are responses
query = query.join('sections','items')
query = query.filter(and_(tst.CandidateTestItem.response!=None, tst.CandidateTestItem.response!=''))
# order by test version
query = query.order_by(asc(tst.Test.id),asc(tst.TestVersion.id),asc(tst.CandidateTest.local_date))
# eagerload our responses
query = query.options(eagerload_all('sections.items'), eagerload('sections.items.item.type.has_response'))
return query.distinct()
def selectByUserAdresses(db_department, addresses):
s = db_department.session()
ret = (
s.query(Mail).options(eagerload_all("statehistory")).filter(Mail.idreceiveruseraddress.in_(addresses)).all()
)
s.close()
return ret
def get(cls, reference):
'''Returns a tag object referenced by its id or name.'''
query = Session.query(cls).filter(cls.id == reference)
query = query.options(eagerload_all('package_tags'))
tag = query.first()
if tag == None:
tag = cls.by_name(reference)
return tag
def get(cls, reference):
'''Returns a tag object referenced by its id or name.'''
query = Session.query(cls).filter(cls.id==reference)
query = query.options(eagerload_all('package_tags'))
tag = query.first()
if tag == None:
tag = cls.by_name(reference)
return tag
def _get_matching_planes_and_manifests(self, session, plane_id, manifest_id=None):
query = (
session.query(model.Plane).outerjoin(model.Manifest).
filter(model.Plane.plane_id > 0). # There's a "non-manifest" manifest for counter sales, etc.
filter(model.Plane.is_active == True).
options(
orm.eagerload_all('manifests.invoices.item'),
orm.eagerload_all('manifests.invoices.customer')
)
)
if plane_id:
query = query.filter(model.Plane.plane_id==plane_id)
# Not really needed?
if manifest_id:
query = query.filter(model.Manifest.manifest_id==manifest_id)
return query.all()
def index(self):
from ckan.lib.helpers import Page
query = ckan.authz.Authorizer().authorized_query(c.user, model.Group)
query = query.order_by(model.Group.name.asc())
query = query.order_by(model.Group.title.asc())
query = query.options(eagerload_all("packages"))
c.page = Page(collection=query, page=request.params.get("page", 1), items_per_page=20)
return render("group/index.html")
def active_folders(trans, folder):
# Stolen from galaxy.web.controllers.library_common (importing from which causes a circular issues).
# Much faster way of retrieving all active sub-folders within a given folder than the
# performance of the mapper. This query also eagerloads the permissions on each folder.
return trans.sa_session.query( trans.app.model.LibraryFolder ) \
.filter_by( parent=folder, deleted=False ) \
.options( eagerload_all( "actions" ) ) \
.order_by( trans.app.model.LibraryFolder.table.c.name ) \
.all()
def active_folders(trans, folder):
# Stolen from galaxy.web.controllers.library_common (importing from which causes a circular issues).
# Much faster way of retrieving all active sub-folders within a given folder than the
# performance of the mapper. This query also eagerloads the permissions on each folder.
return trans.sa_session.query(trans.app.model.LibraryFolder) \
.filter_by(parent=folder, deleted=False) \
.options(eagerload_all("actions")) \
.order_by(trans.app.model.LibraryFolder.table.c.name) \
.all()
def get_recent_pages():
# TODO needs date filter for features
return (
session.query(ComicPage)
.filter(~ ComicPage.is_queued)
.order_by(ComicPage.order.desc())
.options(
eagerload_all(ComicPage.chapter, ComicChapter.comic)
)
)
def get_recent_pages():
# TODO needs date filter for features
return (
session.query(ComicPage)
.filter(~ ComicPage.is_queued)
.order_by(ComicPage.order.desc())
.options(
eagerload_all(ComicPage.chapter, ComicChapter.comic)
)
)
def get_wiki(self, wiki=None, attrload=[], attrload_all=[]):
"""Get the Wiki instance identified by,
`wiki`, which can be,
`id` or `wikiurl` or `Wiki` instance.
Return
List of Wiki instances or
Wiki instance."""
msession = meta.Session()
if isinstance(wiki, Wiki) and attrload == [] and attrload == []:
return wiki
# Compose query based on `wiki` type
if isinstance(wiki, (int, long)):
q = msession.query(Wiki).filter_by(id=wiki)
elif isinstance(wiki, (str, unicode)):
q = msession.query(Wiki).filter_by(wikiurl=wiki)
elif isinstance(wiki, Wiki):
q = msession.query(Wiki).filter_by(id=wiki.id)
else:
q = None
# Compose eager-loading options
if q != None:
q = q.options(*[eagerload_all(e) for e in attrload_all])
q = q.options(*[eagerload(e) for e in attrload])
wiki = q.first()
elif wiki == None:
q = msession.query(Wiki)
q = q.options(*[eagerload_all(e) for e in attrload_all])
q = q.options(*[eagerload(e) for e in attrload])
wiki = q.all()
else:
wiki = None
return wiki
def subjects():
subjects = g.user.company.subjects.options(eagerload_all(
Subject.photos)).order_by(desc(Subject.id)).all()
ret = []
for subject in subjects:
item = subject.get_json(with_photos=True)
ret.append(item)
item['photo_ids'] = []
for photo in item['photos']:
item['photo_ids'].append(photo['id'])
return success_result(ret)
def index(self):
from ckan.lib.helpers import Page
query = ckan.authz.Authorizer().authorized_query(
c.user, model.AuthorizationGroup)
query = query.options(eagerload_all('users'))
c.page = Page(
collection=query,
page=request.params.get('page', 1),
items_per_page=20)
return render('authorization_group/index.html')
def all_q(cls, instance=None, include_deleted=False):
from membership import Membership
q = meta.Session.query(User)
if not include_deleted:
q = q.filter(or_(User.delete_time == None, User.delete_time > datetime.utcnow()))
if instance:
q = q.options(eagerload_all("memberships"))
q = q.join(Membership)
q = q.filter(or_(Membership.expire_time == None, Membership.expire_time > datetime.utcnow()))
q = q.filter(Membership.instance == instance)
return q
def get_history_datasets(self, trans, history):
"""
Returns history's datasets.
"""
query = (trans.sa_session.query(trans.model.HistoryDatasetAssociation)
.filter(trans.model.HistoryDatasetAssociation.history == history)
.join("dataset")
.options(eagerload_all("dataset.actions"))
.order_by(trans.model.HistoryDatasetAssociation.hid)
.filter(trans.model.HistoryDatasetAssociation.deleted == expression.false())
.filter(trans.model.Dataset.purged == expression.false()))
return query.all()
def get_history_datasets(self, trans, history):
"""
Returns history's datasets.
"""
query = (trans.sa_session.query(trans.model.HistoryDatasetAssociation)
.filter(trans.model.HistoryDatasetAssociation.history == history)
.join("dataset")
.options(eagerload_all("dataset.actions"))
.order_by(trans.model.HistoryDatasetAssociation.hid)
.filter(trans.model.HistoryDatasetAssociation.deleted == expression.false())
.filter(trans.model.Dataset.purged == expression.false()))
return query.all()
def get_wikicomment(self, wikicomment=None, attrload=[], attrload_all=[]):
"""Get the wiki comment identified by,
`wikicomment` which can be,
`id` or `WikiComment` instance.
if wikicomment==None,
return all the comments for `wiki`.
Return,
List of WikiComment instances or
WikiComment instance."""
if isinstance( wikicomment, WikiComment ) and attrload==[] and \
attrload_all==[] :
wikicmt = wikicomment
msession = meta.Session()
# Compose query based on `wikicmt` type
if isinstance(wikicomment, (int, long)):
q = msession.query(WikiComment).filter_by(id=wikicomment)
elif isinstance(wikicomment, WikiComment):
q = msession.query(WikiComment).filter_by(id=wikicomment.id)
else:
q = None
# Compose eager-loading options
if q != None:
q = q.options(*[eagerload_all(e) for e in attrload_all])
q = q.options(*[eagerload(e) for e in attrload])
wikicmt = q.first()
elif wikicomment == None:
q = msession.query(WikiComment)
q = q.options(*[eagerload_all(e) for e in attrload_all])
q = q.options(*[eagerload(e) for e in attrload])
wikicmt = q.all()
else:
wikicmt = None
return wikicmt
def index(self):
from ckan.lib.helpers import Page
try:
context = {"model": model, "user": c.user or c.author}
check_access("site_read", context)
except NotAuthorized:
abort(401, _("Not authorized to see this page"))
query = ckan.authz.Authorizer().authorized_query(c.user, model.AuthorizationGroup)
query = query.options(eagerload_all("users"))
c.page = Page(collection=query, page=request.params.get("page", 1), items_per_page=20)
return render("authorization_group/index.html")
def all_q(cls, instance=None, include_deleted=False):
from membership import Membership
q = meta.Session.query(User)
if not include_deleted:
q = q.filter(or_(User.delete_time == None,
User.delete_time > datetime.utcnow()))
if instance:
q = q.options(eagerload_all('memberships'))
q = q.join(Membership)
q = q.filter(or_(Membership.expire_time == None,
Membership.expire_time > datetime.utcnow()))
q = q.filter(Membership.instance == instance)
return q
def get_objects_list(objects_ids,
object_type,
eagerload_all_=None,
return_objects=True):
"""Get a list of objects by their IDs, either as int or str. Make
sure we return a list of integers as IDs.
object_type - the type of object to be returned. It must be one of
['groups', 'machines', 'customers']
return_objects - whether to return the actual objects or not.
"""
objects_types_assoc = {
'groups': Group,
'machines': Machine,
'customers': Customer
}
# Check if object_type is valid
if object_type not in objects_types_assoc:
raise ValueError("Invalid object type: %s" % (object_type))
# Get variables
if isinstance(objects_ids, str):
objects_ids = [int(objects_ids)]
elif isinstance(objects_ids, int):
objects_ids = [objects_ids]
elif isinstance(objects_ids, list):
objects_ids = [int(x) for x in objects_ids]
else:
raise ValueError("Invalid %s specification" % (object_type))
# Pull the objects/IDs from the DB
obj = objects_types_assoc[object_type]
if return_objects:
objects_q = query(obj).filter(obj.id.in_(objects_ids))
if eagerload_all_:
objects_q = objects_q.options(eagerload_all(eagerload_all_))
objects = objects_q.all()
else:
objects_q = sql.select([obj.id]).where(obj.id.in_(objects_ids))
objects = meta.Session.execute(objects_q).fetchall()
if len(objects) != len(objects_ids):
# Woah, you specified objects that didn't exist ?
valid_objects = [x.id for x in objects]
invalid_objects = [x for x in objects_ids if x not in valid_objects]
raise ValueError("Invalid %s: %s" % (object_type, invalid_objects))
return (objects if return_objects else None, objects_ids)
def build_query(cls,
session,
join=None,
where=None,
order_by=None,
limit=None,
offset=None,
count=None,
eagerload=None,
group=None):
aggregating = False
if count is not None:
aggregating = True
query = session.query(func.count(count)).select_from(cls)
else:
query = session.query(cls)
if join:
if isinstance(join, (list, tuple)):
for j in join:
query = query.join(j)
else:
query = query.join(join)
if where:
for filter in where:
query = query.filter(filter)
if eagerload and not aggregating:
for relname in eagerload:
query = query.options(eagerload_all(relname))
if order_by is not None:
if count is None:
if isinstance(order_by, (list, tuple)):
query = query.order_by(*order_by)
else:
query = query.order_by(order_by)
if limit:
query = query.limit(limit)
if offset:
query = query.offset(offset)
if group:
group_clause = []
for key in group:
group_clause.append(getattr(cls, key))
query = query.group_by(*group_clause)
return query
def index(self):
from ckan.lib.helpers import Page
if not self.authorizer.am_authorized(c, model.Action.SITE_READ, model.System):
abort(401, _('Not authorized to see this page'))
query = ckan.authz.Authorizer().authorized_query(c.user, model.AuthorizationGroup)
query = query.options(eagerload_all('users'))
c.page = Page(
collection=query,
page=request.params.get('page', 1),
items_per_page=20
)
return render('authorization_group/index.html')
def index(self):
from ckan.lib.helpers import Page
if not self.authorizer.am_authorized(c, model.Action.SITE_READ,
model.System):
abort(401, _('Not authorized to see this page'))
query = ckan.authz.Authorizer().authorized_query(
c.user, model.AuthorizationGroup)
query = query.options(eagerload_all('users'))
c.page = Page(collection=query,
page=request.params.get('page', 1),
items_per_page=20)
return render('authorization_group/index.html')
def index(self):
from ckan.lib.helpers import Page
try:
context = {'model': model, 'user': c.user or c.author}
check_access('site_read', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
query = ckan.authz.Authorizer().authorized_query(
c.user, model.AuthorizationGroup)
query = query.options(eagerload_all('users'))
c.page = Page(collection=query,
page=request.params.get('page', 1),
items_per_page=20)
return render('authorization_group/index.html')
def index(self):
from ckan.lib.helpers import Page
try:
context = {'model':model,'user': c.user or c.author}
check_access('site_read',context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
query = ckan.authz.Authorizer().authorized_query(c.user, model.AuthorizationGroup)
query = query.options(eagerload_all('users'))
c.page = Page(
collection=query,
page=request.params.get('page', 1),
items_per_page=20
)
return render('authorization_group/index.html')
def get_objects_list(objects_ids, object_type, eagerload_all_=None,
return_objects=True):
"""Get a list of objects by their IDs, either as int or str. Make
sure we return a list of integers as IDs.
object_type - the type of object to be returned. It must be one of
['groups', 'machines', 'customers']
return_objects - whether to return the actual objects or not.
"""
objects_types_assoc = {'groups': Group,
'machines': Machine,
'customers': Customer}
# Check if object_type is valid
if object_type not in objects_types_assoc:
raise ValueError("Invalid object type: %s" % (object_type))
# Get variables
if isinstance(objects_ids, str):
objects_ids = [int(objects_ids)]
elif isinstance(objects_ids, int):
objects_ids = [objects_ids]
elif isinstance(objects_ids, list):
objects_ids = [int(x) for x in objects_ids]
else:
raise ValueError("Invalid %s specification" % (object_type))
# Pull the objects/IDs from the DB
obj = objects_types_assoc[object_type]
if return_objects:
objects_q = query(obj).filter(obj.id.in_(objects_ids))
if eagerload_all_:
objects_q = objects_q.options(eagerload_all(eagerload_all_))
objects = objects_q.all()
else:
objects_q = sql.select([obj.id]).where(obj.id.in_(objects_ids))
objects = meta.Session.execute(objects_q).fetchall()
if len(objects) != len(objects_ids):
# Woah, you specified objects that didn't exist ?
valid_objects = [x.id for x in objects]
invalid_objects = [x for x in objects_ids if x not in valid_objects]
raise ValueError("Invalid %s: %s" % (object_type, invalid_objects))
return (objects if return_objects else None, objects_ids)
def get_user(user, eagerload_all_=None):
"""Get a user provided a username or an int(user_id), possibly eager
loading some relations.
"""
if isinstance(user, int):
uq = query(User).filter_by(id=user)
else:
uq = query(User).filter_by(username=user)
if eagerload_all_:
uq = uq.options(eagerload_all(eagerload_all_))
usr = uq.first()
if not usr:
raise LookupError("Invalid user: %s" % user)
return usr
def get_user(user, eagerload_all_=None):
"""Get a user provided a username or an int(user_id), possibly eager
loading some relations.
"""
if isinstance(user, int):
uq = query(User).filter_by(id=user)
else:
uq = query(User).filter_by(username=user)
if eagerload_all_:
uq = uq.options(eagerload_all(eagerload_all_))
usr = uq.first()
if not usr:
raise LookupError("Invalid user: %s" % user)
return usr
def build_query(cls, session, join=None, where=None, order_by=None,
limit=None, offset=None, count=None, eagerload=None,
group=None):
aggregating = False
if count is not None:
aggregating = True
query = session.query(func.count(count)).select_from(cls)
else:
query = session.query(cls)
if join:
if isinstance(join, (list, tuple)):
for j in join:
query = query.join(j)
else:
query = query.join(join)
if where:
for filter in where:
query = query.filter(filter)
if eagerload and not aggregating:
for relname in eagerload:
query = query.options(eagerload_all(relname))
if order_by is not None:
if count is None:
if isinstance(order_by, (list, tuple)):
query = query.order_by(*order_by)
else:
query = query.order_by(order_by)
if limit:
query = query.limit(limit)
if offset:
query = query.offset(offset)
if group:
group_clause = []
for key in group:
group_clause.append(getattr(cls, key))
query = query.group_by(*group_clause)
return query
def user_list(self, groups=False):
"""Return a simple list of the users
groups - return the list of groups for each user, or not
"""
req = query(User)
if groups:
req = req.options(eagerload_all('groups_assoc.group'))
lst = query(User).all()
out = []
for x in lst:
# perhaps add the pubkey ?
if x.created_time:
stmp = xmlrpclib.DateTime(x.created_time)
else:
stmp = 0
nx = {
'id': x.id,
'username': x.username,
'created_stamp': stmp,
'is_admin': x.is_admin,
'setup_expired': x.setup_expired(),
'waiting_setup': bool(x.waiting_setup)
}
if groups:
# Loop group_assocs for that user..
nx['groups'] = []
for ug in x.groups_assoc:
nxgrp = {
'is_admin': ug.is_admin,
'name': ug.group.name,
'id': ug.group_id
}
nx['groups'].append(nxgrp)
out.append(nx)
# Can use: datetime.fromtimestamp(x.created_stamp)
# to get a datetime object back from the x.created_time
return vaultMsg(True, "Here is the user list", {'list': out})
def _compute_stats_relation(stats_clause):
l_session = tc_session()
for rs in RouterStats.query.\
filter(stats_clause).\
options(eagerload_all('router.circuits.extensions')).\
all():
rs.circ_fail_to = 0
rs.circ_try_to = 0
rs.circ_fail_from = 0
rs.circ_try_from = 0
tot_extend_time = 0
tot_extends = 0
for c in rs.router.circuits:
for e in c.extensions:
if e.to_node == r:
rs.circ_try_to += 1
if isinstance(e, FailedExtension):
rs.circ_fail_to += 1
elif e.hop == 0:
tot_extend_time += e.delta
tot_extends += 1
elif e.from_node == r:
rs.circ_try_from += 1
if isinstance(e, FailedExtension):
rs.circ_fail_from += 1
if isinstance(c, FailedCircuit):
pass # TODO: Also count timeouts against earlier nodes?
elif isinstance(c, DestroyedCircuit):
pass # TODO: Count these somehow..
if tot_extends > 0: rs.avg_first_ext = (1.0*tot_extend_time)/tot_extends
else: rs.avg_first_ext = 0
if rs.circ_try_from > 0:
rs.circ_from_rate = (1.0*rs.circ_fail_from/rs.circ_try_from)
if rs.circ_try_to > 0:
rs.circ_to_rate = (1.0*rs.circ_fail_to/rs.circ_try_to)
if rs.circ_try_to+rs.circ_try_from > 0:
rs.circ_bi_rate = (1.0*rs.circ_fail_to+rs.circ_fail_from)/(rs.circ_try_to+rs.circ_try_from)
l_session.add(rs)
l_session.commit()
tc_session.remove()
def _compute_stats_relation(stats_clause):
for rs in RouterStats.query.\
filter(stats_clause).\
options(eagerload_all('router.circuits.extensions')).\
all():
rs.circ_fail_to = 0
rs.circ_try_to = 0
rs.circ_fail_from = 0
rs.circ_try_from = 0
tot_extend_time = 0
tot_extends = 0
for c in rs.router.circuits:
for e in c.extensions:
if e.to_node == r:
rs.circ_try_to += 1
if isinstance(e, FailedExtension):
rs.circ_fail_to += 1
elif e.hop == 0:
tot_extend_time += e.delta
tot_extends += 1
elif e.from_node == r:
rs.circ_try_from += 1
if isinstance(e, FailedExtension):
rs.circ_fail_from += 1
if isinstance(c, FailedCircuit):
pass # TODO: Also count timeouts against earlier nodes?
elif isinstance(c, DestroyedCircuit):
pass # TODO: Count these somehow..
if tot_extends > 0: rs.avg_first_ext = (1.0*tot_extend_time)/tot_extends
else: rs.avg_first_ext = 0
if rs.circ_try_from > 0:
rs.circ_from_rate = (1.0*rs.circ_fail_from/rs.circ_try_from)
if rs.circ_try_to > 0:
rs.circ_to_rate = (1.0*rs.circ_fail_to/rs.circ_try_to)
if rs.circ_try_to+rs.circ_try_from > 0:
rs.circ_bi_rate = (1.0*rs.circ_fail_to+rs.circ_fail_from)/(rs.circ_try_to+rs.circ_try_from)
tc_session.add(rs)
tc_session.commit()
def index_zitate():
_solr = solr()
log.info("indexing transcripts...")
datas = []
for zitat in Zitat.query.options(
eagerload_all(Zitat.person, Zitat.sitzung,
Zitat.debatten_zitate)):
data = zitat.to_dict()
data = flatten(data)
data.update(type_info(zitat))
data = convert_dates(data)
data = convert_text(data)
datas.append(data)
if len(datas) % 1000 == 0:
sys.stdout.write(".")
sys.stdout.flush()
_solr.add_many(datas)
_solr.commit()
datas = []
_solr.add_many(datas)
_solr.commit()
def _compute_filtered_relational(min_ratio, stats_clause, filter_clause):
l_session = tc_session()
badrouters = RouterStats.query.filter(stats_clause).filter(filter_clause).\
filter(RouterStats.sbw_ratio < min_ratio).all()
# TODO: Turn this into a single query....
for rs in RouterStats.query.filter(stats_clause).\
options(eagerload_all('router.streams.circuit.routers')).all():
tot_sbw = 0
sbw_cnt = 0
for s in rs.router.streams:
if isinstance(s, ClosedStream):
skip = False
#for br in badrouters:
# if br != rs:
# if br.router in s.circuit.routers:
# skip = True
if not skip:
# Throw out outliers < mean
# (too much variance for stddev to filter much)
if rs.strm_closed == 1 or s.bandwidth() >= rs.sbw:
tot_sbw += s.bandwidth()
sbw_cnt += 1
if sbw_cnt: rs.filt_sbw = tot_sbw/sbw_cnt
else: rs.filt_sbw = None
l_session.add(rs)
if sqlalchemy.__version__ < "0.5.0":
avg_sbw = RouterStats.query.filter(stats_clause).avg(RouterStats.filt_sbw)
else:
avg_sbw = l_session.query(func.avg(RouterStats.filt_sbw)).filter(stats_clause).scalar()
for rs in RouterStats.query.filter(stats_clause).all():
if type(rs.filt_sbw) == float and avg_sbw:
rs.filt_sbw_ratio = rs.filt_sbw/avg_sbw
else:
rs.filt_sbw_ratio = None
l_session.add(rs)
l_session.commit()
tc_session.remove()
def user_list(self, groups=False):
"""Return a simple list of the users
groups - return the list of groups for each user, or not
"""
req = query(User)
if groups:
req = req.options(eagerload_all('groups_assoc.group'))
lst = query(User).all()
out = []
for x in lst:
# perhaps add the pubkey ?
if x.created_time:
stmp = xmlrpclib.DateTime(x.created_time)
else:
stmp = 0
nx = {'id': x.id, 'username': x.username,
'created_stamp': stmp,
'is_admin': x.is_admin,
'setup_expired': x.setup_expired(),
'waiting_setup': bool(x.waiting_setup)}
if groups:
# Loop group_assocs for that user..
nx['groups'] = []
for ug in x.groups_assoc:
nxgrp = {'is_admin': ug.is_admin,
'name': ug.group.name,
'id': ug.group_id}
nx['groups'].append(nxgrp)
out.append(nx)
# Can use: datetime.fromtimestamp(x.created_stamp)
# to get a datetime object back from the x.created_time
return vaultMsg(True, "Here is the user list", {'list': out})
def group_list(self, show_hidden=False, list_users=False):
"""Return a simple list of the available groups"""
# FIXME: list_users is not used
# FIXME: show_hidden is not used
groups = query(Group).options(eagerload_all('users_assoc.user')).all()
me = query(User).get(self.myself_id)
out = []
for grp in groups:
myug = [ug for ug in grp.users_assoc if ug.user_id == me.id]
res = {
'id': grp.id,
'name': grp.name,
'member': me in grp.users,
'hidden': False,
'admin': False
}
if grp.hidden:
# Only for global-admin or members, if we're asked to hide
# hidden groups
if not show_hidden and not me.is_admin and not myug:
continue
res['hidden'] = True
if len(myug):
res['cryptgroupkey'] = myug[0].cryptgroupkey
if myug[0].is_admin:
res['admin'] = True
res['members'] = [(u.user_id, u.user.username, u.is_admin)
for u in grp.users_assoc]
out.append(res)
return vaultMsg(True, 'Here is the list of groups', {'list': out})
def _compute_filtered_relational(min_ratio, stats_clause, filter_clause):
badrouters = RouterStats.query.filter(stats_clause).filter(filter_clause).\
filter(RouterStats.sbw_ratio < min_ratio).all()
# TODO: Turn this into a single query....
for rs in RouterStats.query.filter(stats_clause).\
options(eagerload_all('router.streams.circuit.routers')).all():
tot_sbw = 0
sbw_cnt = 0
for s in rs.router.streams:
if isinstance(s, ClosedStream):
skip = False
#for br in badrouters:
# if br != rs:
# if br.router in s.circuit.routers:
# skip = True
if not skip:
# Throw out outliers < mean
# (too much variance for stddev to filter much)
if rs.strm_closed == 1 or s.bandwidth() >= rs.sbw:
tot_sbw += s.bandwidth()
sbw_cnt += 1
if sbw_cnt: rs.filt_sbw = tot_sbw/sbw_cnt
else: rs.filt_sbw = None
tc_session.add(rs)
if sqlalchemy.__version__ < "0.5.0":
avg_sbw = RouterStats.query.filter(stats_clause).avg(RouterStats.filt_sbw)
else:
avg_sbw = tc_session.query(func.avg(RouterStats.filt_sbw)).filter(stats_clause).scalar()
for rs in RouterStats.query.filter(stats_clause).all():
if type(rs.filt_sbw) == float and avg_sbw:
rs.filt_sbw_ratio = rs.filt_sbw/avg_sbw
else:
rs.filt_sbw_ratio = None
tc_session.add(rs)
tc_session.commit()
from sqlalchemy.sql import exists, func
from sqlalchemy.orm import lazyload, eagerload, eagerload_all
from sqlalchemy import and_, or_, not_
from pokedex.db import connect, tables, load
session = connect()
session.query(tables.PokemonFormGeneration).delete()
generations = list(
session.query(tables.Generation).order_by(tables.Generation.id))
q = session.query(tables.PokemonForm)
q = q.options(eagerload_all('pokemon', 'species'))
q = q.order_by(tables.PokemonForm.order)
form_orders = dict(
unown=list('abcdefghijklmnopqrstuvwxyz') + ['exclamation', 'question'],
deoxys=['normal', 'attack', 'defense', 'speed'],
burmy=['plant', 'sandy', 'trash'],
wormadam=['plant', 'sandy', 'trash'],
shellos=['west', 'east'],
gastrodon=['west', 'east'],
rotom=[None, 'heat', 'wash', 'frost', 'fan', 'mow'],
giratina=['altered', 'origin'],
shaymin=['land', 'sky'],
castform=[None, 'sunny', 'rainy', 'snowy'],
basculin=['red-striped', 'blue-striped'],
darmanitan=['standard', 'zen'],
def get_all_gpx_points(self):
with self.session_scope() as s:
ret = s.query(map_point.GpxPoint).options(
eagerload_all('gpx_point.*')).all()
return ret
from sqlalchemy.sql import exists, func
from sqlalchemy.orm import lazyload, eagerload, eagerload_all
from sqlalchemy import and_, or_, not_
from pokedex.db import connect, tables, load
session = connect()
session.query(tables.PokemonFormGeneration).delete()
generations = list(session.query(tables.Generation).order_by(
tables.Generation.id))
q = session.query(tables.PokemonForm)
q = q.options(eagerload_all('pokemon', 'species'))
q = q.order_by(tables.PokemonForm.order)
form_orders = dict(
unown=list('abcdefghijklmnopqrstuvwxyz') + ['exclamation', 'question'],
deoxys=['normal', 'attack', 'defense', 'speed'],
burmy=['plant', 'sandy', 'trash'],
wormadam=['plant', 'sandy', 'trash'],
shellos=['west', 'east'],
gastrodon=['west', 'east'],
rotom=[None, 'heat', 'wash', 'frost', 'fan', 'mow'],
def f(**kwargs) -> user.User:
id = kwargs["id"]
u = session.query(user.User).options(
eagerload_all('*')).filter(user.User.id == id).one()
session.expunge(u)
return u
def __default_query_options(self, query):
for j in self.join:
query=query.options(eagerload_all(j))
return query
gen_process = Process(target=generate, args=(url,))
gen_process.start()
gen_process.join()
# import generated classes
import classes
import inspect
import pickle
# open a session to execute queries
session = create_session(bind=engine)
# get all classes from the generated module
for name, obj in inspect.getmembers(classes):
# get a list of generated classes only, without other classes like 'Integer','DateTime'...etc
if inspect.isclass(obj) and obj.__module__ == 'classes':
print("\nRetrieving data from Table \"" + obj.__tablename__ + "\" ...")
# similar to "select * from Table"
data = session.query(obj).options(eagerload_all("*")).all()
# make directory if doesn't exist
if not os.path.exists('data'):
os.mkdir('data')
# save the object list in a binary file (serialization)
with open("data/" + obj.__tablename__ + '.bin', 'wb') as output:
pickle.dump(data, output)
print("Finished successfully [OK]")
