def upgrade(pyramid_env):
with context.begin_transaction():
op.execute("""INSERT INTO social_auth_account
(id, provider_id, username, uid, extra_data, picture_url,
provider_domain)
SELECT idprovider_agent_account.id, provider_id, username,
userid AS uid, profile_info AS extra_data, picture_url,
facebook_account.app_id AS provider_domain
FROM idprovider_agent_account
LEFT JOIN facebook_account
ON facebook_account.id=idprovider_agent_account.id""")
op.execute(
"""UPDATE identity_provider
SET provider_type = 'google-oauth2'
WHERE provider_type = 'google_oauth2'""")
op.execute("""UPDATE abstract_agent_account
SET type='social_auth_account'
WHERE type IN ('idprovider_agent_account','facebook_account')""")
with context.begin_transaction():
op.drop_constraint(
"facebook_access_token_facebook_account_fb_account_id_id",
"facebook_access_token")
op.drop_constraint(
"facebook_source_facebook_account_creator_id_id",
"facebook_source")
op.create_foreign_key(
"facebook_access_token_social_auth_account_fb_account_id_id",
"facebook_access_token", "social_auth_account",
["fb_account_id"], ["id"])
op.create_foreign_key(
"facebook_source_social_auth_account_creator_id_id",
"facebook_source", "social_auth_account",
["creator_id"], ["id"])
op.drop_table("facebook_account")
op.drop_table("idprovider_agent_account")
def downgrade(pyramid_env):
with context.begin_transaction():
op.add_column(
'thematic',
sa.Column('title_id', sa.Integer,
sa.ForeignKey('langstring.id')))
op.add_column(
'thematic',
sa.Column('description_id', sa.Integer,
sa.ForeignKey('langstring.id')))
op.add_column(
'question',
sa.Column('title_id', sa.Integer,
sa.ForeignKey('langstring.id')))
# Do stuff with the app's models here.
from assembl import models as m
db = m.get_session_maker()()
with transaction.manager:
db.execute("UPDATE thematic SET title_id = (SELECT title_id FROM idea WHERE id=thematic.id AND sqla_type='thematic')")
db.execute("UPDATE thematic SET description_id = (SELECT description_id FROM idea WHERE id=thematic.id AND sqla_type='thematic')")
db.execute("UPDATE question SET title_id = (SELECT title_id FROM idea WHERE id=question.id AND sqla_type='question')")
mark_changed()
with context.begin_transaction():
op.drop_column('idea', 'title_id')
op.drop_column('idea', 'description_id')
def downgrade(pyramid_env):
with context.begin_transaction():
op.add_column(
'agent_email_account',
sa.Column("preferred", sa.SmallInteger,
default=False, server_default='0'))
# Do stuff with the app's models here.
from assembl import models as m
db = m.get_session_maker()()
with transaction.manager:
# get from previous values
db.execute("""UPDATE agent_email_account SET preferred=(
SELECT abstract_agent_account.preferred
FROM abstract_agent_account
WHERE abstract_agent_account.id = agent_email_account.id
AND abstract_agent_account."type" = 'agent_email_account')""")
# Force update, transaction manager saw nothing
aaa = db.query(m.Role).first()
flag_modified(aaa, 'name')
with context.begin_transaction():
db.execute('ALTER TABLE agent_email_account ADD CHECK (preferred IN (0, 1))')
op.drop_column(
'abstract_agent_account', "preferred")
def upgrade(pyramid_env):
with context.begin_transaction():
op.drop_column('posts', u'headers')
op.add_column('posts', sa.Column('message_id', sa.String(),
nullable=True))
op.add_column('posts', sa.Column('parent_id', sa.Integer(),
nullable=True))
op.create_table('emails',
sa.Column('ins_date', sa.DateTime(), nullable=False),
sa.Column('mod_date', sa.DateTime(), nullable=False),
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('headers', sa.Text(), nullable=False),
sa.Column('body', sa.Text(), nullable=False),
sa.Column('post_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['post_id'], ['posts.id'], ),
sa.PrimaryKeyConstraint('id'))
with transaction.manager:
for post in db.query(m.Post).all():
if not post.message_id:
post.message_id = msg_id()
with context.begin_transaction():
op.alter_column('posts', u'message_id',
existing_type=sa.VARCHAR(),
nullable=False)
def downgrade(pyramid_env):
if using_virtuoso():
with context.begin_transaction():
op.create_table('social_auth_account_temp',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('username', sa.String(200)))
# Do stuff with the app's models here.
from assembl import models as m
db = m.get_session_maker()()
with transaction.manager:
db.execute("""INSERT INTO social_auth_account_temp
SELECT id, username FROM social_auth_account
WHERE username IS NOT NULL""")
mark_changed()
with context.begin_transaction():
op.drop_column('social_auth_account', 'username')
op.add_column(
'social_auth_account', sa.Column('username', sa.String(200)))
with transaction.manager:
db.execute("""UPDATE social_auth_account
SET username = (
SELECT username FROM social_auth_account_temp
WHERE social_auth_account_temp.id = social_auth_account.id)""")
mark_changed()
with context.begin_transaction():
op.drop_table('social_auth_account_temp')
else:
with context.begin_transaction():
op.alter_column('social_auth_account', 'username', type_=sa.Unicode(200))
def upgrade(pyramid_env):
with context.begin_transaction():
op.add_column('file', sa.Column('file_identity', sa.String(64), index=True))
# Do stuff with the app's models here.
from assembl import models as m
db = m.get_session_maker()()
hash_fs = get_hashfs()
with transaction.manager:
# document.creation_date?
for fid, title, creation_date, data in db.execute(
"""SELECT file.id, document.title, document.creation_date, file.data
FROM file JOIN document using(id)"""):
data = BytesIO(data)
data.seek(0)
parts = title.split('.')
extension = parts[-1] if len(parts) > 1 else None
address = hash_fs.put(data, extension)
creation_date = creation_date or datetime.now()
creation_date = timegm(creation_date.timetuple())
if address.is_duplicate:
creation_date = min(creation_date, path.getmtime(address.abspath))
utime(address.abspath, (creation_date, creation_date))
db.execute("UPDATE file SET file_identity='%s' WHERE id=%d" % (address.id, fid))
mark_changed()
with context.begin_transaction():
op.drop_column('file', 'data')
op.execute('vacuum full', {'isolation_level':'AUTOCOMMIT'})
def downgrade(pyramid_env):
from assembl import models as m
db = m.get_session_maker()()
with context.begin_transaction():
op.create_table(
"user_language_preference_temp",
sa.Column("id", sa.Integer, primary_key=True),
sa.Column("user_id", sa.Integer),
sa.Column("lang_code", sa.String),
sa.Column("preferred_order", sa.Integer),
sa.Column("locale_id", sa.Integer),
sa.Column("explicitly_defined", sa.Boolean, server_default="0"))
with transaction.manager:
from assembl.models.auth import LanguagePreferenceOrder
op.execute("""INSERT INTO user_language_preference_temp
(id, user_id, locale_id, preferred_order)
SELECT id, user_id, locale_id, source_of_evidence
FROM user_language_preference""")
locale_ids = db.execute(
"""SELECT DISTINCT locale_id, locale.code
FROM user_language_preference
JOIN locale ON (locale.id=locale_id)""")
for locale_id, locale_name in locale_ids:
op.execute("UPDATE user_language_preference_temp SET lang_code = '%s' WHERE locale_id = %d" % (
locale_name, locale_id))
op.execute("""UPDATE user_language_preference_temp
SET explicitly_defined = 1 WHERE preferred_order = %d""" % (LanguagePreferenceOrder.Explicit,))
op.execute("DELETE FROM user_language_preference")
mark_changed()
with context.begin_transaction():
op.add_column(
'user_language_preference', sa.Column(
'explicitly_defined', sa.Boolean, nullable=False, server_default=TextClause("0")))
op.add_column(
'user_language_preference', sa.Column(
'lang_code', sa.String(), nullable=False, server_default=""))
op.drop_index(
'%s_%s_user_language_preference_UNQC_user_id_locale_id' % (
config.get('db_schema'), config.get('db_user')))
op.create_index(
'%s_%s_user_language_preference_UNQC_user_id_lang_code' % (
config.get('db_schema'), config.get('db_user')),
'user_language_preference', ['user_id', 'lang_code'], unique=True)
op.drop_column('user_language_preference', 'source_of_evidence')
op.drop_column('user_language_preference', 'translate_to')
op.drop_column('user_language_preference', 'locale_id')
with transaction.manager:
op.execute("""INSERT INTO user_language_preference
(id, user_id, lang_code, preferred_order, explicitly_defined)
SELECT id, user_id, lang_code, preferred_order, explicitly_defined
FROM user_language_preference_temp""")
mark_changed()
with context.begin_transaction():
op.drop_table("user_language_preference_temp")
def upgrade(pyramid_env):
with context.begin_transaction():
# Virtuoso will corrupt the database if the body is
# not nulled before dropping. Setting it to '' is not enough.
# (Does not matter for subject column, for some reason. LONG NVARCHAR?)
op.execute("update content set subject=null, body=null")
with context.begin_transaction():
op.drop_column("content", "body")
op.drop_column("content", "subject")
def upgrade(pyramid_env):
with context.begin_transaction():
op.add_column('announce', sa.Column('title_id',
sa.Integer(), sa.ForeignKey('langstring.id')))
op.add_column('announce', sa.Column('body_id',
sa.Integer(), sa.ForeignKey('langstring.id')))
# Do stuff with the app's models here.
from assembl import models as m
from assembl.nlp.translation_service import LanguageIdentificationService
db = m.get_session_maker()()
# Disable idea reindexation
from assembl.lib.sqla import BaseOps, orm_update_listener
if sa.event.contains(BaseOps, 'after_update', orm_update_listener):
sa.event.remove(BaseOps, 'after_update', orm_update_listener)
with transaction.manager:
ds = db.query(m.Discussion).all()
locales_of_discussion = {d.id: d.discussion_locales for d in ds}
langid_services = {d.id: LanguageIdentificationService(d) for d in ds
if len(locales_of_discussion[d.id]) > 1}
announcement_strings = db.execute(
"SELECT id, title, body FROM announce")
announcement_strings = {id: (title, body)
for (id, title, body) in announcement_strings}
for announcement in db.query(m.Announcement):
candidate_langs = locales_of_discussion[announcement.discussion_id]
(title, body) = announcement_strings[announcement.id]
if len(candidate_langs) == 1:
lang = candidate_langs[0]
else:
text = ' '.join(filter(None, (
title or '',
sanitize_text(body or ''))))
lang = None
if text:
# Use idea language for priors?
lang, data = langid_services[announcement.discussion_id].identify(text)
if not lang:
print "***** Could not identify for announcement %d: %s" % (announcement.id, text)
lang = candidate_langs[0]
def as_lang_string(text):
ls = m.LangString.create(text, lang)
return ls
if title:
announcement.title = as_lang_string(title)
if body:
announcement.body = as_lang_string(body)
with context.begin_transaction():
op.drop_column('announce', 'title')
op.drop_column('announce', 'body')
def upgrade(pyramid_env):
with context.begin_transaction():
op.add_column(
'abstract_agent_account',
sa.Column("verified", sa.SmallInteger,
default=False, server_default='0'))
op.add_column(
'abstract_agent_account',
sa.Column('email', sa.String(100), index=True))
# Do stuff with the app's models here.
from assembl import models as m
db = m.get_session_maker()()
with transaction.manager:
# Start with a blanket 0
db.execute("UPDATE abstract_agent_account SET verified=0")
# get from previous values
db.execute("""UPDATE abstract_agent_account SET email=(
SELECT agent_email_account.email
FROM agent_email_account
WHERE abstract_agent_account.id = agent_email_account.id)
WHERE abstract_agent_account."type" = 'agent_email_account'""")
db.execute("""UPDATE abstract_agent_account SET verified=(
SELECT agent_email_account.verified
FROM agent_email_account
WHERE abstract_agent_account.id = agent_email_account.id)
WHERE abstract_agent_account."type" = 'agent_email_account'""")
db.execute("""UPDATE abstract_agent_account SET verified=(
SELECT identity_provider.trust_emails
FROM identity_provider
JOIN idprovider_agent_account ON (
idprovider_agent_account.provider_id = identity_provider.id)
WHERE abstract_agent_account.id = idprovider_agent_account.id)
WHERE abstract_agent_account."type" = 'idprovider_agent_account'""")
db.flush()
ipaccounts = db.query(m.IdentityProviderAccount).all()
for ipaccount in ipaccounts:
ipaccount.interpret_profile()
if ipaccount.email:
email_accounts = db.query(m.EmailAccount).filter_by(
email=ipaccount.email).all()
for email_account in email_accounts:
if email_account.profile == ipaccount.profile:
ipaccount.verified |= email_account.verified
db.delete(email_account)
elif ipaccount.verified and not email_account.verified:
db.delete(email_account)
else:
# I checked that this case did not happen
# in our existing databases
ipaccount.profile.merge(email_account.profile)
with context.begin_transaction():
db.execute('ALTER TABLE abstract_agent_account ADD CHECK (verified IN (0, 1))')
op.drop_table('agent_email_account')
def downgrade(pyramid_env):
with context.begin_transaction():
op.add_column('thematic', sa.Column('video_description_id',
sa.Integer, sa.ForeignKey('langstring.id')))
with transaction.manager:
op.execute("UPDATE thematic SET video_description_id = video_description_top_id")
mark_changed()
with context.begin_transaction():
op.drop_column('thematic', 'video_description_bottom_id')
op.drop_column('thematic', 'video_description_top_id')
def downgrade(pyramid_env):
with context.begin_transaction():
op.execute("delete from user_language_preference where source_of_evidence > 0")
with context.begin_transaction():
op.drop_constraint(
"%s_%s_user_language_preference_UNQC_user_id_locale_id_source_of_evidence" % (
config.get('db_schema'), config.get('db_user')),
"user_language_preference")
op.create_unique_constraint(
"%s_%s_user_language_preference_UNQC_user_id_locale_id" % (
config.get('db_schema'), config.get('db_user')),
"user_language_preference", ["user_id", "locale_id"])
def downgrade(pyramid_env):
with context.begin_transaction():
op.create_table(
'agent_email_account',
sa.Column('id', sa.Integer, sa.ForeignKey(
'abstract_agent_account.id', ondelete='CASCADE',
onupdate='CASCADE'),
primary_key=True),
sa.Column('email', sa.String(100), nullable=False, index=True),
sa.Column('verified', sa.SmallInteger(), server_default='0'),
sa.Column('active', sa.Boolean(), server_default='1'))
# Do stuff with the app's models here.
from assembl import models as m
db = m.get_session_maker()()
with transaction.manager:
# get from previous values
db.execute("""INSERT INTO agent_email_account (id, email, verified)
SELECT abstract_agent_account.id,
abstract_agent_account.email,
abstract_agent_account.verified
FROM abstract_agent_account
WHERE abstract_agent_account.email IS NOT NULL
AND abstract_agent_account."type" = 'agent_email_account'""")
ipaccounts = db.query(m.IdentityProviderAccount).all()
for ipaccount in ipaccounts:
ipaccount.interpret_profile()
if ipaccount.email:
db.add(m.EmailAccount(
email=ipaccount.email, profile_id=ipaccount.profile_id,
verified=ipaccount.verified, preferred=ipaccount.preferred))
email_accounts = db.query(m.EmailAccount).filter_by(
email=ipaccount.email).all()
for email_account in email_accounts:
if email_account.profile == ipaccount.profile:
ipaccount.verified |= email_account.verified
db.delete(email_account)
elif ipaccount.verified and not email_account.verified:
db.delete(email_account)
else:
# I checked that this case did not happen
# in our existing databases
ipaccount.profile.merge(email_account.profile)
with context.begin_transaction():
db.execute(
"ALTER TABLE agent_email_account ADD CHECK (verified IN (0, 1))")
op.drop_column(
'abstract_agent_account', "verified")
op.drop_index('ix_abstract_agent_account_email')
op.drop_column('abstract_agent_account', "email")
def downgrade(pyramid_env):
with context.begin_transaction():
op.drop_constraint("idea_content_widget_link_idea_content_positive_link_id_id", "idea_content_widget_link")
op.create_foreign_key(
"idea_content_widget_link_idea_content_link_id_id",
"idea_content_widget_link",
"idea_content_link",
["id"],
["id"],
)
with context.begin_transaction():
op.execute(
"""DELETE FROM idea_content_positive_link WHERE id IN
(SELECT id FROM idea_content_widget_link)"""
)
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
# for the --sql use case, run migrations for each URL into
# individual files.
engines = {'': {'url': context.config.get_main_option('sqlalchemy.url')}}
for name in bind_names:
engines[name] = rec = {}
rec['url'] = context.config.get_section_option(name,
"sqlalchemy.url")
for name, rec in engines.items():
logger.info("Migrating database %s" % (name or '<default>'))
file_ = "%s.sql" % name
logger.info("Writing output to %s" % file_)
with open(file_, 'w') as buffer:
context.configure(url=rec['url'], output_buffer=buffer,
target_metadata=get_metadata(name),
literal_binds=True)
with context.begin_transaction():
context.run_migrations(engine_name=name)
def upgrade(pyramid_env):
# Do stuff with the app's models here.
from assembl import models as m
db = m.get_session_maker()()
with transaction.manager:
# take the first sysadmin as creator
sysadmin_role = db.query(m.Role).filter(m.Role.name == R_SYSADMIN).first()
creator_id = m.User.default_db.query(m.User).join(
m.User.roles).filter(m.Role.id == sysadmin_role.id)[0:1][0].id
columns_headers = dict(list(db.execute(
"SELECT id, header_id FROM idea_message_column")))
columns = db.query(m.IdeaMessageColumn).all()
for column in columns:
synthesis = column.get_column_synthesis()
header_id = columns_headers.get(column.id, None)
if header_id is not None and synthesis is None:
name_en = column.name.closest_entry('en') or column.name.first_original()
name_fr = column.name.closest_entry('fr') or column.name.first_original()
subject_ls = m.LangString.create(u"Synthesis: {}".format(name_en.value), 'en')
subject_ls.add_value(u"Synthèse : {}".format(name_fr.value), 'fr')
body_ls = m.LangString.get(header_id) # don't clone, reuse the same langstring
column.create_column_synthesis(
subject=subject_ls,
body=body_ls,
creator_id=creator_id)
with context.begin_transaction():
op.drop_column('idea_message_column', 'header_id')
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
# Override sqlalchemy.url value to application's value
alembic_config = config.get_section(config.config_ini_section)
alembic_config['sqlalchemy.url'] = app.config['SQLALCHEMY_DATABASE_URI']
engine = engine_from_config(
alembic_config,
prefix='sqlalchemy.',
poolclass=pool.NullPool)
connection = engine.connect()
context.configure(
connection=connection,
target_metadata=db.metadata,
compare_type=True
)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
# this callback is used to prevent an auto-migration from being generated
# when there are no changes to the schema
# reference: http://alembic.readthedocs.org/en/latest/cookbook.html
def process_revision_directives(context, revision, directives):
if getattr(config.cmd_opts, 'autogenerate', False):
script = directives[0]
if script.upgrade_ops.is_empty():
directives[:] = []
logger.info('No changes in schema detected.')
engine = engine_from_config(config.get_section(config.config_ini_section),
prefix='sqlalchemy.',
poolclass=pool.NullPool)
connection = engine.connect()
context.configure(connection=connection,
target_metadata=target_metadata,
process_revision_directives=process_revision_directives,
**current_app.extensions['migrate'].configure_args)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
alembic_config = config.get_section(config.config_ini_section)
# if there is a .env at the root, add them to the system variables.
abs_file_path = os.path.abspath('../../../.env')
load_environment_variables(abs_file_path)
alembic_config['sqlalchemy.url'] = os.environ["DATABASE_URL"]
connectable = engine_from_config(
alembic_config,
prefix='sqlalchemy.',
poolclass=pool.NullPool)
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
dbcfg = config.get_section(config.config_ini_section)
if 'DATABASE_URL' in os.environ:
dbcfg['sqlalchemy.url'] = os.environ['DATABASE_URL']
engine = engine_from_config(
dbcfg,
prefix='sqlalchemy.',
poolclass=pool.NullPool)
connection = engine.connect()
context.configure(
connection=connection,
target_metadata=target_metadata
)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
alembic_config = config.get_section(config.config_ini_section)
alembic_config['sqlalchemy.url'] = seplis_config['api']['database']
engine = engine_from_config(
alembic_config,
prefix='sqlalchemy.',
poolclass=pool.NullPool
)
connection = engine.connect()
context.configure(
connection=connection,
target_metadata=target_metadata
)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
def upgrade(pyramid_env):
with context.begin_transaction():
op.create_table(
'edgesense_drupal_source',
sa.Column('id', sa.Integer, sa.ForeignKey(
'post_source.id',
onupdate='CASCADE',
ondelete='CASCADE'), primary_key=True),
sa.Column('node_source', sa.String(1024), nullable=False),
sa.Column('node_root', sa.String(200)),
sa.Column('user_source', sa.String(1024), nullable=False),
sa.Column('comment_source', sa.String(1024), nullable=False),
sa.Column('post_id_prepend', sa.String(100), nullable=False)
)
op.create_table(
'source_specific_account',
sa.Column('id', sa.Integer, sa.ForeignKey(
'abstract_agent_account.id',
onupdate='CASCADE',
ondelete='CASCADE'), primary_key=True),
sa.Column('user_info', sa.Text),
sa.Column('user_link', sa.String(1024)),
sa.Column('user_id', sa.String(15), nullable=False),
sa.Column('source_id', sa.Integer, sa.ForeignKey(
'edgesense_drupal_source.id', onupdate='CASCADE',
ondelete='CASCADE'), nullable=False),
)
# Do stuff with the app's models here.
from assembl import models as m
db = m.get_session_maker()()
with transaction.manager:
pass
def downgrade(pyramid_env):
with context.begin_transaction():
op.add_column('idea_message_column', sa.Column('header_id',
sa.Integer(), sa.ForeignKey('langstring.id')))
from assembl import models as m
db = m.get_session_maker()()
with transaction.manager:
columns = db.query(m.IdeaMessageColumn).all()
for column in columns:
synthesis = column.get_column_synthesis()
if synthesis is not None:
header = synthesis.body.clone()
# we need to clone here, otherwise the langstring is deleted with db.delete(synthesis)
# because of the delete-orphan on the relationship and result to an Integrity error
# because the langstring is still referenced from idea_message_column table.
db.add(header)
db.flush()
# we can't use here: column.header_id = header.id
# the mapper doesn't now about header_id and the change
# will not be committed
db.execute("""update idea_message_column set header_id = %d
where id = %d""" % (header.id, column.id))
mark_changed()
db.delete(synthesis)
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
if isinstance(engine, Engine):
connection = engine.connect()
else:
raise Exception(
'Expected engine instance got %s instead' % type(engine)
)
context.configure(
connection=connection,
target_metadata=target_metadata
)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
options = config.get_section(config.config_ini_section)
options['sqlalchemy.url'] = db_url
engine = engine_from_config(options,
prefix='sqlalchemy.',
poolclass=pool.NullPool)
# url="sqlalchemy.url" + "postgresql://" + environ['DB_1_PORT'][7:]
connection = engine.connect()
context.configure(
connection=connection,
target_metadata=target_metadata
)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix='sqlalchemy.',
poolclass=pool.NullPool)
# we can get the table prefix via the tag object
tag = context.get_tag_argument()
if tag and isinstance(tag, dict):
table_prefix = tag.get('table_prefix', '')
else:
table_prefix = ''
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata,
version_table=table_prefix + 'alembic_version'
)
with context.begin_transaction():
context.run_migrations(table_prefix=table_prefix)
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
core_configs = config.get_section(config.config_ini_section)
core_configs['sqlalchemy.url'] = settings.SQLALCHEMY_DATABASE_URI
engine = engine_from_config(
core_configs,
prefix='sqlalchemy.',
poolclass=pool.NullPool)
connection = engine.connect()
context.configure(
connection=connection,
target_metadata=target_metadata
)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
from uliweb.manage import make_simple_application
from uliweb import orm, settings
# engine = engine_from_config(
# config.get_section(config.config_ini_section),
# prefix='sqlalchemy.',
# poolclass=pool.NullPool)
name = config.get_main_option("engine_name")
make_simple_application(project_dir='.')
target_metadata = orm.get_metadata(name)
connection = orm.get_connection(engine_name=name).connect()
# connection = engine.connect()
context.configure(
connection=connection,
target_metadata=target_metadata,
compare_server_default=True,
include_object=uliweb_include_object,
# compare_server_default=uliweb_compare_server_default,
)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
if 'DB_URL' in os.environ:
engine = engine_from_config(
config.get_section("app:pyramidapp"),
prefix='sqlalchemy.',
url=os.environ['DB_URL'],
poolclass=pool.NullPool)
else:
engine = engine_from_config(
config.get_section("app:pyramidapp"),
prefix='sqlalchemy.',
poolclass=pool.NullPool)
connection = engine.connect()
context.configure(
connection=connection,
target_metadata=target_metadata
)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
cfg = config.get_section(config.config_ini_section)
if 'use_flask_db_url' in cfg and cfg['use_flask_db_url'] == 'true':
cfg['sqlalchemy.url'] = get_app_config('SQLALCHEMY_DATABASE_URI')
engine = engine_from_config(
cfg,
prefix='sqlalchemy.',
poolclass=pool.NullPool
)
connection = engine.connect()
context.configure(
connection=connection,
target_metadata=target_metadata
)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url,
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
connectable = engine_from_config(config.get_section(
config.config_ini_section),
prefix='sqlalchemy.',
poolclass=pool.NullPool)
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata,
compare_type=True,
compare_server_default=True,
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
connectable = config.attributes.get('connection', None)
if connectable is None:
# only create Engine if we don't have a Connection
# from the outside
connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix='sqlalchemy.',
poolclass=pool.NullPool)
# when connectable is already a Connection object, calling
# connect() gives us a *branched connection*.
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
def process_revision_directives(context, revision, directives):
script = directives[0]
if script.upgrade_ops.is_empty():
directives[:] = []
log.info("No changes found skipping revision creation.")
connectable = engine_from_config(
config.get_section(config.config_ini_section), prefix="sqlalchemy.", poolclass=pool.NullPool
)
with connectable.connect() as connection:
# get the schema names
for schema in get_tenant_schemas(connection):
log.info(f"Migrating {schema}...")
connection.execute(f'set search_path to "{schema}"')
connection.dialect.default_schema_name = schema
context.configure(
connection=connection,
target_metadata=target_metadata,
include_schemas=True,
include_object=include_object,
process_revision_directives=process_revision_directives,
)
with context.begin_transaction():
context.run_migrations()
if context.config.cmd_opts:
if context.config.cmd_opts.cmd == "revision":
break
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = get_url()
context.configure(url=url,
target_metadata=target_metadata,
literal_binds=True,
compare_type=True)
with context.begin_transaction():
context.run_migrations()
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
with open(join(dirname(__file__), '../config.cfg')) as f:
config.read_file(f)
url = config['db']['engine']
context.configure(url=url, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
def runMigrationsOnline():
"""Run migrations in 'online' mode.
See Alembic documentation for more details on these functions.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
CONFIG.set_main_option("sqlalchemy.url", repository.getDbDSN())
engine = engine_from_config(CONFIG.get_section(CONFIG.config_ini_section),
prefix="sqlalchemy.",
poolclass=pool.NullPool)
connection = engine.connect()
context.configure(connection=connection, target_metadata=TARGET_METADATA)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
def run_migrations_online(connection=None):
"""
Run migrations in 'online' mode.
In this scenario we need to create an Engine and associate a connection with the context.
"""
if not config_connection:
url = get_db_url()
else:
url = config_connection.engine.url
def connect(c=None):
if isinstance(c, Connection) and not c.closed:
return c
if not isinstance(c, Connectable):
c = create_engine(url, convert_unicode=True, echo=False)
return c.connect()
if not database_exists(url):
db_name = get_constant("MAGPIE_POSTGRES_DB")
LOGGER.warning("Database [{}] not found, attempting creation...".format(db_name))
connection = create_database(url, encoding="utf8", template="template0")
# retry connection and run migration
with connect(connection) as migrate_conn:
try:
context.configure(
connection=migrate_conn,
target_metadata=target_metadata,
version_table="alembic_version",
transaction_per_migration=True,
render_as_batch=True
)
with context.begin_transaction():
context.run_migrations()
finally:
# close the connection only if not given argument
if migrate_conn is not connection:
migrate_conn.close()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
config = {'sqlalchemy.url': DB_CONNECTION, 'sqlalchemy.echo': DEBUG}
connectable = engine_from_config(
config,
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(connection=connection,
target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
alembic_config = config.get_section(config.config_ini_section)
alembic_config['sqlalchemy.url'] = engine_url
connectible = engine_from_config(
alembic_config,
prefix='sqlalchemy.',
poolclass=pool.NullPool)
with connectible.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with either a URL
or an Engine.
Calls to context.execute() here emit the given string to the
script output.
"""
set_mysql_engine()
kwargs = dict()
if neutron_config.database.connection:
kwargs['url'] = neutron_config.database.connection
else:
kwargs['dialect_name'] = neutron_config.database.engine
kwargs['version_table'] = P4_VERSION_TABLE
context.configure(**kwargs)
with context.begin_transaction():
context.run_migrations()
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option('sqlalchemy.url')
context.configure(url=url,
target_metadata=target_metadata,
include_schemas=True,
version_table_schema='public',
include_object=_include_object,
render_item=_render_item,
template_args={'toplevel_code': set()})
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
engine = engine_from_config(
config.get_section(config.config_ini_section),
prefix='sqlalchemy.',
poolclass=pool.NullPool)
connection = engine.connect()
context.configure(
connection=connection,
target_metadata=target_metadata)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata,
version_table="rie_alembic_version",
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
db_uri = os.environ.get('DATABASE_URL')
# try:
# create_database(db_uri)
# except Exception:
# logging.info('Failed to create database. Probably already exists.')
connectable = create_engine(db_uri)
with connectable.connect() as connection:
context.configure(connection=connection,
target_metadata=target_metadata,
compare_type=True)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
configuration = config.get_section(config.config_ini_section)
configuration['sqlalchemy.url'] = settings.SQLALCHEMY_DATABASE_URI
print(settings.SQLALCHEMY_DATABASE_URI)
connectable = engine_from_config(
configuration,
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(connection=connection,
target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix='sqlalchemy.',
poolclass=pool.NullPool)
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata,
render_as_batch=(config.get_main_option('sqlalchemy.url')
.startswith('sqlite:///'))
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
configuration = config.get_section(config.config_ini_section)
configuration["sqlalchemy.url"] = get_url()
connectable = engine_from_config(
configuration,
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(connection=connection,
target_metadata=target_metadata,
render_as_batch=True)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
connectible = None
print(f"run_migrations_online")
print(f"environment:{api_config.environment}")
print(f"connection string:{connection_string}")
connectible = create_engine(connection_string, poolclass=pool.NullPool)
SCHEMA_NAME = "NOT_test_fktdb"
def include_object(object, name, type_, reflected, compare_to):
if False: # (type_ == "table"):
return object.schema == 'Common'
else:
return True
if connectible is not None:
# Create schema; if it already exists, skip this
try:
connectible.execute(CreateSchema("Common"))
except sqlalchemy.exc.ProgrammingError:
pass
with connectible.connect() as connection:
context.configure(connection=connection,
target_metadata=target_metadata,
compare_type=True,
compare_server_default=True,
include_schemas=True,
version_table='AlembicVersion',
version_table_schema='Common',
include_object=include_object)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
# this callback is used to prevent an auto-migration from being generated
# when there are no changes to the schema
# reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
def process_revision_directives(context, revision, directives):
if getattr(config.cmd_opts, "autogenerate", False):
script = directives[0]
if script.upgrade_ops.is_empty():
directives[:] = []
LOGGER.info("No changes in schema detected.")
# TODO: Enable postgres version 7/23/2019 # configuration = config.get_section(config.config_ini_section)
# TODO: Enable postgres version 7/23/2019 # configuration['sqlalchemy.url'] = get_url()
connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata,
process_revision_directives=process_revision_directives,
)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
# this callback is used to prevent an auto-migration from being generated
# when there are no changes to the schema
# reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
def process_revision_directives(context, revision, directives):
if getattr(config.cmd_opts, "autogenerate", False):
script = directives[0]
if script.upgrade_ops.is_empty():
directives[:] = []
logger.info("No changes in schema detected.")
engine = engine_from_config(
config.get_section(config.config_ini_section),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
connection = engine.connect()
context.configure(
connection=connection,
target_metadata=target_metadata,
include_object=include_object,
process_revision_directives=process_revision_directives,
**current_app.extensions["migrate"].configure_args,
)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
# this callback is used to prevent an auto-migration from being generated
# when there are no changes to the schema
# reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
def process_revision_directives(context, revision, directives):
if getattr(config.cmd_opts, 'autogenerate', False):
script = directives[0]
if script.upgrade_ops.is_empty():
directives[:] = []
logger.info('No changes in schema detected.')
connectable = engine_from_config(config.get_section(
config.config_ini_section),
prefix='sqlalchemy.',
poolclass=pool.NullPool,
pool_pre_ping=True)
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata,
process_revision_directives=process_revision_directives,
version_table_schema=target_metadata.schema,
include_schemas=True,
compare_type=True,
dialect_name="sqlite",
**current_app.extensions['migrate'].configure_args)
with context.begin_transaction():
# context.execute('SET search_path TO public')
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
from clscraper.models import url
_config = config.get_section(config.config_ini_section)
_config["sqlalchemy.url"] = url
connectable = engine_from_config(
_config,
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(
connection=connection, target_metadata=target_metadata
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
if os.environ.get('DATABASE_URL') is None:
connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix='sqlalchemy.',
poolclass=pool.NullPool)
else:
connectable = create_engine(os.environ.get('DATABASE_URL'))
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = os.getenv("DATABASE_URI")
if url is None:
print('Environment variable DATABASE_URI is not set. Exiting...')
sys.exit(1)
context.configure(url=url,
target_metadata=target_metadata,
literal_binds=True,
compare_type=True)
with context.begin_transaction():
context.run_migrations()
def upgrade(pyramid_env):
with context.begin_transaction():
op.add_column(
"user",
sa.Column("last_idealoom_login", sa.DateTime))
op.add_column(
"social_auth_account",
sa.Column("last_checked", sa.DateTime))
op.execute('UPDATE public.user SET last_idealoom_login = last_login')
# set last_checked from last_login when single account
op.execute('''
UPDATE social_auth_account SET last_checked = (
SELECT last_login FROM public.user
JOIN abstract_agent_account ON public.user.id = abstract_agent_account.profile_id
WHERE abstract_agent_account.id = social_auth_account.id)
WHERE id IN (
SELECT min(abstract_agent_account.id)
FROM abstract_agent_account
JOIN public.user ON (public.user.id = abstract_agent_account.profile_id)
WHERE abstract_agent_account.verified
GROUP BY profile_id
HAVING count(abstract_agent_account.id) = 1
AND min(abstract_agent_account.type)='social_auth_account')''')
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
# connectable = engine_from_config(
# config.get_section(config.config_ini_section),
# prefix="sqlalchemy.",
# poolclass=pool.NullPool,
# )
connectable = create_engine(get_url_db(config_gl['postgres_db']))
with connectable.connect() as connection:
context.configure(
connection=connection, target_metadata=target_metadata,
compare_type=True,
)
with context.begin_transaction():
context.run_migrations()
def run_migrations(db):
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
url = "postgres://{user}:{password}@{host}/{database}".format(
user=db.user,
password=db.password,
host=db.host,
database=db.database)
connectable = create_engine(url, poolclass=pool.NullPool)
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
cfg = config.get_section(config.config_ini_section)
if 'use_flask_db_url' in cfg and cfg['use_flask_db_url'] == 'true':
cfg['sqlalchemy.url'] = get_app_config('SQLALCHEMY_DATABASE_URI')
engine = engine_from_config(cfg,
prefix='sqlalchemy.',
poolclass=pool.NullPool)
connection = engine.connect()
context.configure(connection=connection, target_metadata=target_metadata)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
settings = config.get_section(config.config_ini_section)
settings['sqlalchemy.url'] = os.environ.get('PG_DATABASE_URL', settings['sqlalchemy.url'])
connectable = engine_from_config(
settings,
prefix='sqlalchemy.',
poolclass=pool.NullPool)
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata
)
with context.begin_transaction():
context.run_migrations()
