def delete_rows_with_missing_fkey(fkey, delete_missing=True):
fkey = _as_fkey(fkey)
if fkey.parent.nullable:
return True
session = get_session_maker()()
source = fkey.parent.table
target = fkey.column.table
if source == target:
target = alias(source)
source_primary_key = primary_key_col(source)
q = session.query(source_primary_key).outerjoin(
target, fkey.parent == fkey.column).filter(
target.c.id == None)
count = q.count()
if count:
if delete_missing:
with transaction.manager:
#session.execute(source.delete(source.c.id.in_(q)))
for (id,) in q:
delete_row(session, source, id)
mark_changed(session)
else:
print "There are %d ids in %s with dangling %s:" % (
count, source.name, fk_as_str(fkey))
print q.all()
return False
return True
def copy_discussion(source_config, dest_config, source_slug, dest_slug,
delete=False, debug=False, permissions=None):
if (session_maker_is_initialized() and abspath(source_config) == get_config()["__file__"]):
# not running from script
dest_session = get_session_maker()()
dest_metadata = get_metadata()
else:
dest_metadata, dest_session = engine_from_settings(
dest_config, True)
dest_tables = dest_metadata.sorted_tables
if source_config != dest_config:
from assembl.lib.sqla import _session_maker
temp = _session_maker
assert temp == dest_session
source_metadata, source_session = engine_from_settings(
source_config, False)
source_tables_by_name = {
table.name: table.tometadata(source_metadata, source_metadata.schema)
for table in dest_tables
}
_session_maker = dest_session
else:
source_metadata, source_session = dest_metadata, dest_session
try:
init_key_for_classes(dest_session)
from assembl.models import Discussion
discussion = source_session.query(Discussion).filter_by(
slug=source_slug).one()
assert discussion, "No discussion named " + source_slug
permissions = [x.split('+') for x in permissions or ()]
for (role, permission) in permissions:
assert role in SYSTEM_ROLES
assert permission in ASSEMBL_PERMISSIONS
existing = dest_session.query(Discussion).filter_by(slug=dest_slug).first()
if existing:
if delete:
print "deleting", dest_slug
with transaction.manager:
delete_discussion(dest_session, existing.id)
else:
print "Discussion", dest_slug,
print "already exists! Add -d to delete it."
exit(0)
from assembl.models import Role, Permission, DiscussionPermission
with dest_session.no_autoflush:
copy = clone_discussion(
source_session, discussion.id, dest_session, dest_slug)
for (role, permission) in permissions:
role = dest_session.query(Role).filter_by(name=role).one()
permission = dest_session.query(Permission).filter_by(
name=permission).one()
# assumption: Not already defined.
dest_session.add(DiscussionPermission(
discussion=copy, role=role, permission=permission))
except Exception:
traceback.print_exc()
if debug:
pdb.post_mortem()
capture_exception()
return dest_session
def ensure_inheritance_of(cls):
# Do not bother with tableless classes
if not '__tablename__' in cls.__dict__:
return
base = cls
first = None
table = cls.__table__
for c in cls.mro():
if c == cls:
continue
if '__tablename__' in c.__dict__:
if first is None:
first = c
base = c
if base == cls:
return
basetable = base.__table__
db = get_session_maker()()
poly_col = base.__mapper_args__['polymorphic_on']
if not isinstance(poly_col, Column):
poly_col = basetable.c[poly_col]
poly_id = cls.__mapper_args__['polymorphic_identity']
sub_poly_id = first.__mapper_args__['polymorphic_identity']
query = db.query(basetable.c.id).outerjoin(table, basetable.c.id==table.c.id).filter((poly_col==poly_id) & (table.c.id==None))
if query.count() > 0:
with transaction.manager:
db.execute(basetable.update().where(basetable.c.id.in_(query)).values(**{poly_col.name: sub_poly_id}))
mark_changed(db)
def test_app_no_login_real_policy(request, test_app_no_perm):
"""A configured Assembl fixture with permissions
and no user logged in"""
config = testing.setUp(
registry=test_app_no_perm.app.registry,
settings=get_config(),
)
from ...auth.util import authentication_callback
from pyramid.authorization import ACLAuthorizationPolicy
from pyramid.path import DottedNameResolver
resolver = DottedNameResolver(__package__)
auth_policy_name = "assembl.auth.util.UpgradingSessionAuthenticationPolicy"
auth_policy = resolver.resolve(auth_policy_name)(
callback=authentication_callback)
config.set_authorization_policy(ACLAuthorizationPolicy())
config.set_authentication_policy(auth_policy)
import transaction
# ensure default roles and permissions at startup
from ...models import get_session_maker
with transaction.manager:
session = get_session_maker()
from ...lib.migration import bootstrap_db_data
bootstrap_db_data(session, False)
return test_app_no_perm
def engine_from_settings(config, full_config=False):
settings = get_appsettings(config, 'assembl')
if settings['sqlalchemy.url'].startswith('virtuoso:'):
db_schema = '.'.join((settings['db_schema'], settings['db_user']))
else:
db_schema = settings['db_schema']
set_config(settings, True)
session = None
if full_config:
env = bootstrap(config)
configure_zmq(settings['changes_socket'], False)
configure_indexing()
configure_model_watcher(env['registry'], 'assembl')
logging.config.fileConfig(config)
session = get_session_maker()
metadata = get_metadata()
else:
session = make_session_maker(zope_tr=True)
import assembl.models
from assembl.lib.sqla import class_registry
engine = configure_engine(settings, session_maker=session)
metadata = get_metadata()
metadata.bind = engine
session = sessionmaker(engine)()
return (metadata, session)
def upgrade(pyramid_env):
with context.begin_transaction():
op.create_table(
'permission',
sa.Column('id', sa.Integer(), primary_key=True),
sa.Column('name', sa.String(20), nullable=False))
op.create_table(
'discussion_permission',
sa.Column('id', sa.Integer(), primary_key=True),
sa.Column(
'discussion_id', sa.Integer,
sa.ForeignKey('discussion.id', ondelete='CASCADE')),
sa.Column(
'role_id', sa.Integer,
sa.ForeignKey('role.id', ondelete='CASCADE')),
sa.Column(
'permission_id', sa.Integer,
sa.ForeignKey('permission.id', ondelete='CASCADE')))
# Do stuff with the app's models here.
from assembl.auth.models import (populate_default_roles,
populate_default_permissions, create_default_permissions)
from assembl.models import Discussion
SQLAlchemyBaseModel.metadata.bind = op.get_bind()
db = get_session_maker()()
with transaction.manager:
populate_default_roles(db)
populate_default_permissions(db)
for d in db.query(Discussion).all():
create_default_permissions(db, d)
def session_factory(request):
session_factory = get_session_maker()
def fin():
print "finalizer session_factory"
session_factory.remove()
request.addfinalizer(fin)
return session_factory
def rebuild_table(table, delete_missing=False):
from virtuoso.alchemy import AddForeignKey, DropForeignKey
print "rebuilding", table
session = get_session_maker()()
incoming = set(get_incoming_fks(table))
outgoing = set(table.foreign_keys)
all_fkeys = incoming | outgoing
self_ref = incoming & outgoing
try:
for fk in all_fkeys:
if not delete_rows_with_missing_fkey(fk, delete_missing):
print "There are missing keys, will not rebuild " + table.name
return
except Exception as e:
traceback.print_exc()
print "Could not delete missing keys"
raise e
# Booleans with NULL values
for col in table.c:
if isinstance(col.type, Boolean):
session.execute(table.update().where(col == None).values(**{col.name:0}))
# Drop all keys
for fk in all_fkeys:
try:
session.execute(DropForeignKey(fk))
except Exception as e:
print "Could not drop fkey %s, maybe does not exist." % (fk_as_str(fk),)
print e
clone = clone_table(table, table.name+"_temp", False, False)
clone.create(session.bind)
column_names = [c.name for c in table.columns]
sel = select([getattr(table.c, cname) for cname in column_names])
with transaction.manager:
session.execute(clone.insert().from_select(column_names, sel))
mark_changed(session)
session.execute(DropTable(table))
# Should we create it without outgoing first?
table.create(session.bind)
# self ref will make the insert fail.
for fk in self_ref:
try:
session.execute(DropForeignKey(fk))
except Exception as e:
print "Could not drop fkey %s, maybe does not exist." % (fk_as_str(fk),)
print e
sel = select([getattr(clone.c, cname) for cname in column_names])
with transaction.manager:
session.execute(table.insert().from_select(column_names, sel))
mark_changed(session)
session.execute(DropTable(clone))
if delete_missing:
# Delete a second time, in case.
for fk in outgoing:
assert delete_rows_with_missing_fkey(fk, True), "OUCH"
for fk in incoming: # includes self_ref
session.execute(AddForeignKey(fk))
def session_factory(request):
"""An SQLAlchemy Session Maker fixture"""
# Get the zopeless session maker,
# while the Webtest server will use the
# default session maker, which is zopish.
session_factory = get_session_maker()
def fin():
print "finalizer session_factory"
session_factory.remove()
request.addfinalizer(fin)
return session_factory
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
engine = get_session_maker(False).bind
connection = engine.connect()
context.configure(connection=connection,
target_metadata=get_metadata())
try:
context.run_migrations(pyramid_env=pyramid_env)
finally:
connection.close()
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(url=url)
with context.begin_transaction():
context.run_migrations(pyramid_env=pyramid_env)
session_maker = get_session_maker()
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"configuration",
help="configuration file with destination database configuration")
args = parser.parse_args()
env = bootstrap(args.configuration)
settings = get_appsettings(args.configuration, 'assembl')
set_config(settings)
logging.config.fileConfig(args.configuration)
configure_zmq(settings['changes_socket'], False)
configure_indexing()
configure_engine(settings, True)
session = get_session_maker()()
try:
reindex_all_contents(session)
transaction.commit()
except Exception as e:
traceback.print_exc()
pdb.post_mortem()
def search_endpoint(context, request):
if not indexing_active():
return HTTPServiceUnavailable("Indexing inactive")
query = request.json_body
# u'query': {u'bool': {u'filter': [{u'term': {u'discussion_id': u'23'}}]}}
filters = [fil for fil in query['query']['bool']['filter']]
discussion_id = [f.values()[0].values()[0]
for f in filters if 'discussion_id' in f.values()[0].keys()][0]
discussion = models.Discussion.get_instance(discussion_id)
if discussion is None:
raise HTTPUnauthorized()
user_id = request.authenticated_userid or Everyone
permissions = get_permissions(user_id, discussion_id)
if not discussion.user_can(user_id, CrudPermissions.READ, permissions):
raise HTTPUnauthorized()
es = connect()
index_name = get_index_settings(config)['index_name']
# print get_curl_query(query)
result = es.search(index=index_name, body=query)
# add creator_name in each hit
creator_ids = set([hit['_source']['creator_id']
for hit in result['hits']['hits']
if hit['_source'].get('creator_id', None) is not None])
session = get_session_maker()
creators = session.query(models.AgentProfile.id, models.AgentProfile.name
).filter(models.AgentProfile.id.in_(creator_ids)).all()
creators_by_id = dict(creators)
for hit in result['hits']['hits']:
source = hit['_source']
creator_id = source.get('creator_id', None)
# Remove inner_hits key to not leak posts from private discussion.
# You can easily craft a query to get the participants of a public
# discussion and do a has_child filter with inner_hits on a private discussion.
if 'inner_hits' in hit:
del hit['inner_hits']
if creator_id is not None:
source['creator_name'] = creators_by_id.get(creator_id)
if hit['_type'] == 'idea':
idea = models.Idea.get_instance(source['id'])
# The check is not really necessary because it's the same
# 'read' permission as the discussion, but it doesn't cost anything
# to check it and the READ permission may change in the future.
if not idea.user_can(user_id, CrudPermissions.READ, permissions):
raise HTTPUnauthorized
source['num_posts'] = idea.num_posts
source['num_contributors'] = idea.num_contributors
elif hit['_type'] == 'user':
agent_profile = models.AgentProfile.get_instance(source['id'])
if not agent_profile.user_can(user_id, CrudPermissions.READ, permissions):
raise HTTPUnauthorized
source['num_posts'] = agent_profile.count_posts_in_discussion(discussion_id)
# Don't do an extra request to verify the CrudPermissions.READ permission
# for post or synthesis.
# It's currently the same 'read' permission as the discussion.
# elif hit['_type'] in ('synthesis', 'post'):
# post = models.Post.get_instance(source['id'])
# if not post.user_can(user_id, CrudPermissions.READ, permissions):
# raise HTTPUnauthorized
return result
from assembl.lib.sqla import get_session_maker, get_metadata
from assembl.lib.zmqlib import configure_zmq
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
pyramid_env = bootstrap(config.config_file_name)
configure_zmq(pyramid_env['registry'].settings['changes.socket'], False)
get_session_maker(False)
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
engine = get_session_maker(False).bind
connection = engine.connect()
context.configure(connection=connection,
target_metadata=get_metadata())
try:
parser.add_argument("--rebuild_all_tables", action="store_true", default=False,
help="All tables will be rebuilt.")
parser.add_argument("--ensure_inheritance", action="store_true", default=False,
help="Make sure no class has a missing subclass row.")
parser.add_argument("-d", "--delete_missing", action="store_true", default=False,
help="Delete rows with missing corresponding values. (otherwise abort rebuild.)")
parser.add_argument("--reset_extract_discussion", action="store_true", default=False,
help="Special case: rebuild a dependent foreign key on extract table")
args = parser.parse_args()
settings = get_appsettings(args.configuration, 'assembl')
set_config(settings)
logging.config.fileConfig(args.configuration)
configure_zmq(settings['changes_socket'], False)
configure_indexing()
configure_engine(settings, True)
session = get_session_maker()()
import assembl.models
try:
if (args.reset_extract_discussion and
session.query(assembl.models.Extract).filter_by(discussion_id=-1).count()):
session.execute("""UPDATE "extract" SET discussion_id = (
SELECT content.discussion_id FROM content
JOIN idea_content_link on (content_id=content.id)
JOIN "extract" on ("extract".id = idea_content_link.id)
WHERE "extract".discussion_id=-1)""")
if args.rebuild_all_tables:
rebuild_all_tables(session, args.delete_missing)
elif args.rebuild_all_fkeys:
rebuild_all_tables_fkeys(session, args.rebuild_table, args.delete_missing)
else:
tables = assembl.models.get_metadata().sorted_tables
def committing_session_tween(request):
get_session_maker().commit()
resp = handler(request)
get_session_maker().flush()
return resp
def committing_session_tween(request):
get_session_maker().commit()
return handler(request)
