def main(argv=sys.argv):
import logging
logging.basicConfig(level=logging.DEBUG)
# if len(argv) != 2:
# usage(argv)
# config_uri = argv[1]
# setup_logging(config_uri)
# settings = get_appsettings(config_uri)
# engine = engine_from_config(settings, 'sqlalchemy.')
logging.debug("Initialise Engine")
engine = sqlalchemy.create_engine(DBFILE, echo=False)
meta.Session.configure(bind=engine)
Base.metadata.bind = engine
Base.metadata.create_all(engine)
DBSession = meta.Session()
#DBSession.configure(bind=engine)
logging.debug("Populating Data")
populateData.init_data(DBSession)
#populateUser()
logging.debug("Populated")
DBSession.flush()
#DBSession.commit()
#We also want any alembic scripts to be executed (or not if we build the DB
#properly)
alembic_cfg = Config("cogent/alembic.ini") #TODO: WARNING RELATIVE PATH
command.stamp(alembic_cfg,"head")
DBSession.close()
def create_schema(self, set_alembic_version=True):
"""Creates the 'mediatum' schema.
:param set_alembic_version: Stamp database with current alembic revision information. Defaults to True.
Can be disabled if a schema for testing is going to be created.
"""
s = self.session
logg.info("creating DB schema...")
s.execute("CREATE SCHEMA mediatum")
s.commit()
try:
self.create_all()
if set_alembic_version:
# create alembic version table and set current alembic version to head
from alembic.config import Config
from alembic import command
alembic_cfg = Config(os.path.join(config.basedir, "alembic.ini"))
alembic_cfg.attributes["running_in_mediatum"] = True
command.stamp(alembic_cfg, "head")
s.commit()
logg.info("commited database structure")
except:
# I tried to use a transaction to enclose everything, but sqlalchemy (?) fails when the schema is created within the transaction
# solution: just drop the schema it if something fails after schema creation
s.execute("DROP SCHEMA mediatum CASCADE")
raise
def place_database_under_alembic_control():
a_config = get_alembic_config()
if not is_database_under_migrate_control():
return
if not is_database_under_alembic_control():
print(_("Database is currently not under Alembic's migration "
"control."))
head = get_current_legacy_head()
if head == 42:
alembic_version = 'liberty'
elif head == 43:
alembic_version = 'mitaka01'
elif head == 44:
alembic_version = 'mitaka02'
elif head == 45:
alembic_version = 'ocata01'
elif head in range(1, 42):
print("Legacy head: ", head)
sys.exit(_("The current database version is not supported any "
"more. Please upgrade to Liberty release first."))
else:
sys.exit(_("Unable to place database under Alembic's migration "
"control. Unknown database state, can't proceed "
"further."))
print(_("Placing database under Alembic's migration control at "
"revision:"), alembic_version)
alembic_command.stamp(a_config, alembic_version)
def db_create():
'''Create the db'''
db.create_all()
# then, load the Alembic configuration and generate the
# version table, "stamping" it with the most recent rev:
alembic_cfg = Config("alembic.ini")
command.stamp(alembic_cfg,"head")
def bootstrap_db(config_uri=None, engine=None, with_migration=True):
"""Bring a blank database to a functional state."""
if engine is None:
engine = create_engine(config_uri)
db.configure(bind=engine)
if with_migration:
context = MigrationContext.configure(engine.connect())
db_version = context.get_current_revision()
if db_version:
sys.stderr.write('Database already initialized. Bailing out.\n')
sys.exit(2)
config = Config(config_uri)
script_dir = ScriptDirectory.from_config(config)
heads = script_dir.get_heads()
if len(heads) > 1:
sys.stderr.write('Error: migration scripts have more than one '
'head.\nPlease resolve the situation before '
'attempting to bootstrap the database.\n')
sys.exit(2)
metadata.create_all(engine)
with transaction.manager:
model = MyModel(name='one', value=1)
db.add(model)
# Clean up the sccoped session to allow a later app instantiation.
db.remove()
if with_migration and heads:
command.stamp(config, 'head')
def create_tables(db_url, alembic_ini=None, debug=False):
""" Create the tables in the database using the information from the
url obtained.
:arg db_url, URL used to connect to the database. The URL contains
information with regards to the database engine, the host to
connect to, the user and password and the database name.
ie: <engine>://<user>:<password>@<host>/<dbname>
:kwarg alembic_ini, path to the alembic ini file. This is necessary to
be able to use alembic correctly, but not for the unit-tests.
:kwarg debug, a boolean specifying wether we should have the verbose
output of sqlalchemy or not.
:return a session that can be used to query the database.
"""
engine = sa.create_engine(db_url, echo=debug)
BASE.metadata.create_all(engine)
if alembic_ini is not None: # pragma: no cover
# then, load the Alembic configuration and generate the
# version table, "stamping" it with the most recent rev:
from alembic.config import Config
from alembic import command
alembic_cfg = Config(alembic_ini)
command.stamp(alembic_cfg, "head")
scopedsession = scoped_session(sessionmaker(bind=engine))
return scopedsession
def test_history_current_to_head_as_b(self):
command.stamp(self.cfg, self.b)
self.cfg.stdout = buf = StringIO()
command.history(self.cfg, "current:")
self._eq_cmd_output(buf, [
'%(revb)s -> %(revc)s (head), Rev C',
])
def setup_db():
"""
Either initialize the database if none yet exists, or migrate as needed
"""
from alembic.config import Config
from alembic import command
with current_app.app_context():
# Alembic config used by migration or stamping
alembic_cfg = Config(
os.path.join(current_app.config["PROJECT_PATH"], "alembic.ini")
)
# Database connections
g.db_session = create_db_session()
con = g.db_session.connection()
# Query list of existing tables
tables = con.execute("show tables").fetchall()
alembic_table = ('alembic_version',)
if alembic_table in tables:
# Latest version has been stamped or we have been upgrading
logging.info("Database: Migrating")
command.upgrade(alembic_cfg, "head")
else:
# The database needs to be initialized
logging.info("Database: Initializing")
init_db()
command.stamp(alembic_cfg, "head")
def setup_tables(cls, engine):
with engine.connect() as conn:
trans = conn.begin()
_Model.metadata.create_all(engine)
# Now stamp the latest alembic version
alembic_cfg = Config()
alembic_cfg.set_section_option(
'alembic', 'script_location', 'alembic')
alembic_cfg.set_section_option(
'alembic', 'sqlalchemy.url', str(engine.url))
alembic_cfg.set_section_option(
'alembic', 'sourceless', 'true')
command.stamp(alembic_cfg, 'head')
# always add a test API key
conn.execute(ApiKey.__table__.delete())
key1 = ApiKey.__table__.insert().values(
valid_key='test', allow_fallback=False, allow_locate=True)
conn.execute(key1)
key2 = ApiKey.__table__.insert().values(
valid_key='export', allow_fallback=False, allow_locate=False)
conn.execute(key2)
trans.commit()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--recreate',
help='Drop and re-create all tables.',
action='store_true')
parser.add_argument('--dog-csv',
help='The csv file containing the Dog_info table.')
parser.add_argument('--user-csv',
help='The csv file containing the auth table.')
parser.add_argument('--app-csv',
help='The csv file containing the Apps table.')
args = parser.parse_args()
if args.recreate:
print "Dropping all tables."
db.drop_all()
print "Creating all tables."
db.create_all()
print "Stamping alembic @ HEAD."
with app.app_context():
Migrate(app, db, directory=pkg_resources.resource_filename('crrr', 'migrations'))
command.stamp(_get_config(None), 'head')
if args.dog_csv:
import_dogs(args.dog_csv)
if args.user_csv:
import_users(args.user_csv)
if args.app_csv:
import_apps(args.app_csv)
def db_create():
"""Create the tables and do alembic stuff"""
print 'db at: {0}'.format(
app.config['SQLALCHEMY_DATABASE_URI'])
try:
db.engine.execute('select * from project')
print 'Database already exists with tables.'
return
except (OperationalError, ProgrammingError):
# An operational error here means that the "project" table
# doesn't exist so we should create things!
pass
print 'Creating {0}....'.format(
app.config['SQLALCHEMY_DATABASE_URI'])
db.create_all()
from alembic.config import Config
from alembic import command
alembic_cfg = Config('alembic.ini')
alembic_cfg.set_main_option("sqlalchemy.url",
app.config["SQLALCHEMY_DATABASE_URI"])
command.stamp(alembic_cfg, 'head')
print 'Done.'
def reset_db():
"""Reset database"""
print "WARNING: This will reset the database and may cause data loss."
response = raw_input("Are you sure you want to continue? (Yes/No) ")
if not response == "Yes":
print "Aborted."
sys.exit()
db.drop_all()
db.create_all()
admin = User(
name=u'admin',
email=u'*****@*****.**',
password=u'123456',
role_code=ADMIN,
status_code=ACTIVE,
user_detail=UserDetail(
gender_code=MALE,
age=25,
url=u'http://example.com',
location=u'Kampala',
bio=u''))
db.session.add(admin)
english = Language(name="English",iso639_1="en",iso639_2="eng",locale_code="en_UG")
db.session.add(english)
luganda = Language(name="Luganda",iso639_1="lg",iso639_2="lug",locale_code="lg_UG")
db.session.add(luganda)
db.session.commit()
alembic_cfg = Config("alembic.ini")
command.stamp(alembic_cfg, "head")
def test_noerr_transaction_opened_externally(self):
a, b, c = self._opened_transaction_fixture()
env_file_fixture("""
from sqlalchemy import engine_from_config, pool
def run_migrations_online():
connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix='sqlalchemy.',
poolclass=pool.NullPool)
with connectable.connect() as connection:
with connection.begin() as real_trans:
context.configure(
connection=connection,
transactional_ddl=False,
transaction_per_migration=False
)
with context.begin_transaction():
context.run_migrations()
run_migrations_online()
""")
command.stamp(self.cfg, c)
def test_destination_rev_pre_context(self):
env_file_fixture("""
assert context.get_revision_argument() == '%s'
""" % b)
command.upgrade(self.cfg, b, sql=True)
command.stamp(self.cfg, b, sql=True)
command.downgrade(self.cfg, "%s:%s" % (c, b), sql=True)
def test_stamp_existing_downgrade(self):
command.stamp(self.cfg, self.b)
command.stamp(self.cfg, self.a)
eq_(
self.bind.scalar("select version_num from alembic_version"),
self.a
)
def main(argv, _db_master=None):
parser = argparse.ArgumentParser(
prog=argv[0], description='Initialize Ichnaea database')
parser.add_argument('--initdb', action='store_true',
help='Initialize database')
args = parser.parse_args(argv[1:])
if args.initdb:
from ichnaea import config
# make sure content models are imported
from ichnaea.content import models # NOQA
conf = config()
db_master = Database(conf.get('ichnaea', 'db_master'))
engine = db_master.engine
with engine.connect() as conn:
trans = conn.begin()
_Model.metadata.create_all(engine)
trans.commit()
# Now stamp the latest alembic version
from alembic.config import Config
from alembic import command
import os
ini = os.environ.get('ICHNAEA_CFG', 'ichnaea.ini')
alembic_ini = os.path.join(os.path.split(ini)[0], 'alembic.ini')
alembic_cfg = Config(alembic_ini)
command.stamp(alembic_cfg, "head")
command.current(alembic_cfg)
else:
parser.print_help()
def init():
parser = argparse.ArgumentParser(description='Initialise a database')
parser.add_argument('dburl', help='Database URL for SQLAlchemy')
args = parser.parse_args()
repositories.sa_pixort_data(url=args.dburl, create_schema=True)
command.stamp(_get_config(args.dburl), "head")
def main(argv=sys.argv):
if len(argv) < 2:
usage(argv)
config_uri = argv[1]
options = parse_vars(argv[2:])
setup_logging(config_uri)
settings = get_appsettings(config_uri, options=options)
engine = engine_from_config(settings, 'sqlalchemy.')
DBSession.configure(bind=engine)
Base.metadata.create_all(engine)
with transaction.manager:
user = User(email='*****@*****.**', password='******', admin=True)
DBSession.add(user)
site = Site(name='asd', key='80d621df066348e5938a469730ae0cab')
DBSession.add(site)
site.api_keys.append(SiteAPIKey(key='GIKfxIcIHPbM0uX9PrQ1To29Pb2on0pa'))
site.users.append(user)
aspect_ratio_1_1 = SiteAspectRatio(width=1, height=1)
aspect_ratio_3_1 = SiteAspectRatio(width=3, height=1)
site.aspect_ratios.append(aspect_ratio_1_1)
site.aspect_ratios.append(aspect_ratio_3_1)
from alembic.config import Config
from alembic import command
alembic_cfg = Config("alembic.ini")
command.stamp(alembic_cfg, "head")
def test_history_indicate_current(self):
command.stamp(self.cfg, (self.b,))
self.cfg.stdout = buf = self._buf_fixture()
command.history(self.cfg, indicate_current=True, verbose=True)
self._eq_cmd_output(
buf, [self.c, self.b, self.a], currents=(self.b,), env_token=True
)
def main(argv=sys.argv):
if len(argv) != 2:
usage(argv)
config_uri = argv[1]
if config_uri not in ["jenkins.ini"]:
print "You are not initialising using a testing URI"
print "Edit init_testingdb.py if you really want to do this"
sys.exit(0)
setup_logging(config_uri)
settings = get_appsettings(config_uri)
engine = engine_from_config(settings, "sqlalchemy.")
meta.Session.configure(bind=engine)
Base.metadata.bind = engine
# Create Everythin
print "Dropping Tables"
Base.metadata.drop_all(engine)
print "Creating Tables"
Base.metadata.create_all(engine)
# We also want any alembic scripts to be executed
alembic_cfg = Config(config_uri) # TODO: WARNING RELATIVE PATH
command.stamp(alembic_cfg, "head")
DBSession = meta.Session()
# DBSession.configure(bind=engine)
populateData.init_data(DBSession)
populateUser()
populatedata()
def run(self, args):
pysite.models.create_all()
alembic_cfg = Config(args.alembic_config)
command.stamp(alembic_cfg, "head")
with transaction.manager:
self._setup_users()
def initdb(args, settings):
print('Creating DB structure...')
models.Base.metadata.create_all(DBSession.bind)
from alembic.config import Config
from alembic import command
alembic_cfg = Config("./alembic.ini")
command.stamp(alembic_cfg, "head")
print('Add default data...')
def try_add(obj):
try:
DBSession.add(obj)
DBSession.commit()
except sqlalchemy.exc.IntegrityError:
DBSession.rollback()
pass
hosted_group = models.Group(name='hosted', description='Hosted users')
support_group = models.Group(name='support', description='Support')
admin_group = models.Group(name='admin', description='Administrator')
try_add(hosted_group)
try_add(support_group)
try_add(admin_group)
admin_user = models.User(username='******', groups=[admin_group])
admin_user.set_password('admin')
try_add(admin_user)
def init(uri=None, alembic_ini=None, engine=None, create=False):
""" Initialize a connection. Create tables if requested."""
if uri and engine:
raise ValueError("uri and engine cannot both be specified")
if uri is None and not engine:
uri = 'sqlite:////tmp/datanommer.db'
log.warning("No db uri given. Using %r" % uri)
if uri and not engine:
engine = create_engine(uri)
# We need to hang our own attribute on the sqlalchemy session to stop
# ourselves from initializing twice. That is only a problem is the code
# calling us isn't consistent.
if getattr(session, '_datanommer_initialized', None):
log.warning("Session already initialized. Bailing")
return
session._datanommer_initialized = True
session.configure(bind=engine)
DeclarativeBase.query = session.query_property()
# Loads the alembic configuration and generates the version table, with
# the most recent revision stamped as head
if alembic_ini is not None:
from alembic.config import Config
from alembic import command
alembic_cfg = Config(alembic_ini)
command.stamp(alembic_cfg, "head")
if create:
DeclarativeBase.metadata.create_all(engine)
def main(argv=sys.argv):
args = parser.parse_args(argv[1:])
setup_logging(args.config)
app_settings = get_appsettings(args.config)
alembic_cfg = Config(args.config)
blame = alembic_cfg.get_main_option('blame')
engine = create_engine(alembic_cfg.get_main_option('sqlalchemy.url'))
apps = aslist(app_settings['occams.apps'])
errors = []
assert blame, 'Need to blame someone!'
with engine.begin() as connection:
connection.info['blame'] = blame
for app in apps:
try:
module = import_module(app)
except ImportError:
errors.append('{}: Unable to import'.format(app))
continue
else:
if hasattr(module, 'initdb'):
module.initdb(connection)
else:
errors.append('{}: Does not have "initdb"'.format(app))
for error in errors:
print(error)
alembic_cfg.attributes['connection'] = connection
command.stamp(alembic_cfg, 'heads')
def stamp(context, directory='migrations', revision='head', sql=False, tag=None):
"""'stamp' the revision table with the given revision; don't run any
migrations"""
from app import create_app
with create_app().app_context():
config = _get_config(directory)
command.stamp(config, revision, sql=sql, tag=tag)
def reset_all_dbs(config_hint):
"""
모든 데이터 베이스를 리셋합니다. 만약에 대비해 패스워드를 확인합니다.
전체 리셋 패스워드를 지정하지 않았다면 사용할 수 없습니다.
"""
config_file_path = pm.smart_find_file_path(
config_hint, base_dir_path=CONFIG_DIR_PATH)
app = create_application(config_file_path)
print "#### reset all databases"
print "* database uri: %s" % app.config['SQLALCHEMY_DATABASE_URI']
for key, value in sorted(app.config['SQLALCHEMY_BINDS'].iteritems()):
print " * bind_key: %s uri:%s" % (key, value)
print "* reset_all_password:"******"head")
def install_or_upgrade_db(self, skip_backup=False):
from domogik.common import sql_schema
from domogik.common import database
from sqlalchemy import create_engine, MetaData, Table
from alembic.config import Config
from alembic import command
info("Installing or upgrading the db")
if not sql_schema.Device.__table__.exists(bind=self._engine):
sql_schema.metadata.drop_all(self._engine)
ok("Droping all existing tables...")
sql_schema.metadata.create_all(self._engine)
ok("Creating all tables...")
with self._db.session_scope():
self._db.add_default_user_account()
ok("Creating admin user...")
command.stamp(self.alembic_cfg, "head")
ok("Setting db version to head")
else:
if not skip_backup:
ok("Creating backup")
self.backup_existing_database()
ok("Upgrading")
command.upgrade(self.alembic_cfg, "head")
return
def upgrade(self, nocreate=False, create_legacy_resource_types=False):
from alembic import command
from alembic import migration
cfg = self._get_alembic_config()
cfg.conf = self.conf
if nocreate:
command.upgrade(cfg, "head")
else:
with self.facade.writer_connection() as connection:
ctxt = migration.MigrationContext.configure(connection)
current_version = ctxt.get_current_revision()
if current_version is None:
Base.metadata.create_all(connection)
command.stamp(cfg, "head")
else:
command.upgrade(cfg, "head")
# TODO(sileht): generic shouldn't be a particular case
# we must create a rt_generic and rt_generic_history table
# like other type
for rt in base.get_legacy_resource_types():
if not (rt.name == "generic" or create_legacy_resource_types):
continue
try:
with self.facade.writer() as session:
session.add(rt)
except exception.DBDuplicateEntry:
pass
with self.facade.writer_connection() as connection:
self._RESOURCE_TYPE_MANAGER.map_and_create_tables(
rt, connection)
def _create_tables(self, conn_string):
print('Creating tables')
from server.models import db
current_app.config['SQLALCHEMY_DATABASE_URI'] = conn_string
try:
db.create_all()
except OperationalError as ex:
if 'could not connect to server' in ex.message:
print('ERROR: {red}PostgreSQL service{white} is not running. Please verify that it is running in port 5432 before executing setup script.'.format(red=Fore.RED, white=Fore.WHITE))
sys.exit(1)
elif 'password authentication failed' in ex.message:
print('ERROR: ')
sys.exit(1)
else:
raise
except ProgrammingError as ex:
print(ex)
print('Please check postgres user permissions.')
sys.exit(1)
except ImportError as ex:
if 'psycopg2' in ex:
print(
'ERROR: Missing python depency {red}psycopg2{white}. Please install it with {blue}pip install psycopg2'.format(red=Fore.RED, white=Fore.WHITE, blue=Fore.BLUE))
sys.exit(1)
else:
raise
else:
from alembic.config import Config
from alembic import command
alembic_cfg = Config(os.path.join(os.getcwd(), 'alembic.ini'))
command.stamp(alembic_cfg, "head")
def create_schema(engine, alembic_cfg, location_cfg): # pragma: no cover
old_version = False
with engine.connect() as conn:
trans = conn.begin()
stmt = text('select version_num from alembic_version')
try:
result = conn.execute(stmt).fetchall()
if len(result):
old_version = True
except ProgrammingError:
pass
if not old_version:
_Model.metadata.create_all(engine)
add_api_key(conn)
add_export_config(conn)
add_users(conn, location_cfg)
trans.commit()
# Now stamp the latest alembic version
if not old_version:
command.stamp(alembic_cfg, 'head')
command.current(alembic_cfg)
def stamp(config, revision='head', sql=False, tag=None):
"""'stamp' the revision table with the given revision; don't run any
migrations"""
command.stamp(config, revision, sql=sql, tag=tag)
def stamp(directory=None, revision='head', sql=False, tag=None):
"""'stamp' the revision table with the given revision; don't run any
migrations"""
config = current_app.extensions['migrate'].migrate.get_config(directory)
command.stamp(config, revision, sql=sql, tag=tag)
def init():
engine = create_engine(SQLALCHEMY_DATABASE_URL)
Base.metadata.create_all(engine)
alembic_cfg = Config("/app/alembic.ini")
command.stamp(alembic_cfg, "head")
print()
sys.path.append(
os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))
import config
from pynab.db import Base, engine, Session, User, Group, Category, TvShow, Movie
import pynab.util
from scripts import nzedb_pre_import
db = Session()
start = time.time()
print('Building tables...')
Base.metadata.drop_all(engine)
Base.metadata.create_all(engine)
from alembic.config import Config
from alembic import command
alembic_cfg = Config("alembic.ini")
command.stamp(alembic_cfg, "head")
print('Installing admin user...')
with open('db/initial/users.json', encoding='utf-8', errors='ignore') as f:
data = json.load(f)
try:
engine.execute(User.__table__.insert(), data)
except Exception as e:
print('Problem inserting data into database: {}'.format(e))
sys.exit(0)
print('Copying groups into db...')
with open('db/initial/groups.json', encoding='utf-8',
errors='ignore') as f:
data = json.load(f)
try:
engine.execute(Group.__table__.insert(), data)
except Exception as e:
def stamp_alembic_rev(alembic_config, conn, rev='head', quiet=True):
with quieten(quiet):
alembic_config.attributes['connection'] = conn
stamp(alembic_config, rev)
def main():
sys.stderr.write("create_tables.main: starting\n")
start_time = time.time()
from docassemble.webapp.database import dbprefix
if dbprefix.startswith('postgresql') and not daconfig.get(
'force text to varchar upgrade', False):
do_varchar_upgrade = False
else:
do_varchar_upgrade = True
with app.app_context():
sys.stderr.write("create_tables.main: inside app context after " +
str(time.time() - start_time) + "\n")
if daconfig.get('use alembic', True):
sys.stderr.write("create_tables.main: running alembic after " +
str(time.time() - start_time) + "\n")
insp = inspect(db.engine)
if do_varchar_upgrade:
changed = False
if insp.has_table(dbtableprefix + 'userdict'):
db.session.execute(
delete(UserDict).where(
db.func.length(UserDict.filename) > 255).
execution_options(synchronize_session=False))
changed = True
if insp.has_table(dbtableprefix + 'userdictkeys'):
db.session.execute(
delete(UserDictKeys).where(
db.func.length(UserDictKeys.filename) > 255).
execution_options(synchronize_session=False))
changed = True
if insp.has_table(dbtableprefix + 'chatlog'):
db.session.execute(
delete(ChatLog).where(
db.func.length(ChatLog.filename) > 255).
execution_options(synchronize_session=False))
changed = True
if insp.has_table(dbtableprefix + 'uploads'):
db.session.execute(
delete(Uploads).where(
db.func.length(Uploads.filename) > 255).
execution_options(synchronize_session=False))
db.session.execute(
delete(Uploads).where(
db.func.length(Uploads.yamlfile) > 255).
execution_options(synchronize_session=False))
changed = True
if insp.has_table(dbtableprefix + 'objectstorage'):
db.session.execute(
delete(ObjectStorage).where(
db.func.length(ObjectStorage.key) > 1024).
execution_options(synchronize_session=False))
changed = True
if insp.has_table(dbtableprefix + 'speaklist'):
db.session.execute(
delete(SpeakList).where(
db.func.length(SpeakList.filename) > 255).
execution_options(synchronize_session=False))
changed = True
if insp.has_table(dbtableprefix + 'shortener'):
db.session.execute(
delete(Shortener).where(
db.func.length(Shortener.filename) > 255).
execution_options(synchronize_session=False))
db.session.execute(
delete(Shortener).where(
db.func.length(Shortener.key) > 255).
execution_options(synchronize_session=False))
changed = True
if insp.has_table(dbtableprefix + 'machinelearning'):
db.session.execute(
delete(MachineLearning).where(
db.func.length(MachineLearning.key) > 1024).
execution_options(synchronize_session=False))
db.session.execute(
delete(MachineLearning).where(
db.func.length(MachineLearning.group_id) > 1024).
execution_options(synchronize_session=False))
changed = True
if insp.has_table(dbtableprefix + 'globalobjectstorage'):
db.session.execute(
delete(GlobalObjectStorage).where(
db.func.length(GlobalObjectStorage.key) > 1024).
execution_options(synchronize_session=False))
changed = True
if changed:
db.session.commit()
packagedir = pkg_resources.resource_filename(
pkg_resources.Requirement.parse('docassemble.webapp'),
'docassemble/webapp')
if not os.path.isdir(packagedir):
sys.exit("path for running alembic could not be found")
from alembic.config import Config
from alembic import command
alembic_cfg = Config(os.path.join(packagedir, 'alembic.ini'))
alembic_cfg.set_main_option("sqlalchemy.url",
alchemy_connection_string())
alembic_cfg.set_main_option("script_location",
os.path.join(packagedir, 'alembic'))
if not insp.has_table(dbtableprefix + 'alembic_version'):
sys.stderr.write(
"create_tables.main: creating alembic stamp\n")
command.stamp(alembic_cfg, "head")
sys.stderr.write(
"create_tables.main: done creating alembic stamp after " +
str(time.time() - start_time) + " seconds\n")
if insp.has_table(dbtableprefix + 'user'):
sys.stderr.write(
"create_tables.main: creating alembic stamp\n")
sys.stderr.write(
"create_tables.main: running alembic upgrade\n")
command.upgrade(alembic_cfg, "head")
sys.stderr.write(
"create_tables.main: done running alembic upgrade after " +
str(time.time() - start_time) + " seconds\n")
#db.drop_all()
try:
sys.stderr.write("create_tables.main: trying to create tables\n")
db.create_all()
except:
sys.stderr.write(
"create_tables.main: error trying to create tables; trying a second time.\n"
)
try:
db.create_all()
except:
sys.stderr.write(
"create_tables.main: error trying to create tables; trying a third time.\n"
)
db.create_all()
sys.stderr.write(
"create_tables.main: finished creating tables after " +
str(time.time() - start_time) + " seconds.\n")
if dbprefix.startswith('postgresql'):
try:
test_for_errors(start_time=start_time)
except:
sys.stderr.write(
"create_tables.main: unable to test for errors after " +
str(time.time() - start_time) + " seconds.\n")
populate_tables(start_time=start_time)
db.engine.dispose()
def stamp_alembic_rev(alembic_config, conn, rev="head", quiet=True):
with _alembic_lock, quieten(quiet):
alembic_config.attributes["connection"] = conn
stamp(alembic_config, rev)
def stamp_db(self):
if self.rev:
with self.db.engine.connect() as conn:
trans = conn.begin()
alembic_command.stamp(self.alembic_config(), self.rev)
trans.commit()
def stamp(revision):
"""Fake a migration to a particular revision"""
alembic_cfg = Config("alembic.ini")
command.stamp(alembic_cfg, revision)
def initialize(settings,options):
engine = engine_from_config(settings, 'sqlalchemy.')
config = Configurator(settings=settings)
pyramid_dogpile_cache.includeme(config)
from gengine.metadata import (
init_session,
init_declarative_base,
init_db
)
init_caches()
init_session()
init_declarative_base()
init_db(engine)
from gengine.metadata import (
Base,
DBSession
)
if options.get("reset_db",False):
Base.metadata.drop_all(engine)
engine.execute("DROP SCHEMA IF EXISTS public CASCADE")
engine.execute("CREATE SCHEMA IF NOT EXISTS public")
from alembic.config import Config
from alembic import command
alembic_cfg = Config(attributes={
'engine' : engine,
'schema' : 'public'
})
script_location = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
'app/alembic'
)
alembic_cfg.set_main_option("script_location", script_location)
do_upgrade = options.get("upgrade",False)
if not do_upgrade:
#init
from gengine.app import model
tables = [t for name, t in model.__dict__.items() if isinstance(t, Table)]
Base.metadata.create_all(engine, tables=tables)
command.stamp(alembic_cfg, "head")
if options.get("populate_demo", False):
populate_demo(DBSession)
else:
#upgrade
command.upgrade(alembic_cfg,'head')
admin_user = options.get("admin_user", False)
admin_password = options.get("admin_password", False)
if admin_user and admin_password:
create_user(DBSession = DBSession, user=admin_user,password=admin_password)
engine.dispose()
def tearDown(self):
app = self.create_app()
with app.app_context():
command.stamp(self.alembic_configuration, "base")
db_session.remove()
Base.metadata.drop_all(bind=engine)
def main(self):
database_name = self.config.database_name
if not database_name:
self.config.logger.error(
'"database_name" cannot be an empty string')
return 1
# superuser credentials for overall database
superuser_pg_url = self.create_connection_url(
'postgres', self.config.database_superusername,
self.config.database_superuserpassword)
# superuser credentials for working database
superuser_normaldb_pg_url = self.create_connection_url(
database_name, self.config.database_superusername,
self.config.database_superuserpassword)
# normal user credentials
normal_user_pg_url = self.create_connection_url(
database_name, self.config.database_username,
self.config.database_password)
# ensure that if on Heroku the the normal_user_pg_url and the
# superuser_pg_url are the same
if self.config.on_heroku and (normal_user_pg_url != superuser_pg_url):
self.config.logger.error(
'there is no superuser (%s) when using Heroku',
self.config.database_superusername)
return 1
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# table logging section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if self.config.unlogged:
@compiles(CreateTable)
def create_table(element, compiler, **kw):
text = compiler.visit_create_table(element, **kw)
text = re.sub("^\sCREATE(.*TABLE)",
lambda m: "CREATE UNLOGGED %s" % m.group(1),
text)
return text
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Postgres version check section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
self.config.logger.info('Postgres version check section with %s',
superuser_pg_url)
with PostgreSQLAlchemyManager(superuser_pg_url,
self.config.logger,
autocommit=False) as db:
if not db.min_ver_check(90200):
self.config.logger.error('unrecognized PostgreSQL version: %s',
db.version_string())
self.config.logger.error('Only 9.2+ is supported at this time')
return 1
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# drop database section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# We can only do the following if the DB is not Heroku
# XXX Might add the heroku commands for resetting a DB here
if self.config.dropdb and not self.config.on_heroku:
self.config.logger.info('drop database section with %s',
superuser_pg_url)
with PostgreSQLAlchemyManager(superuser_pg_url,
self.config.logger,
autocommit=False) as db:
if 'test' not in database_name and not self.config.force:
confirm = raw_input('drop database %s [y/N]: ' %
database_name)
if not confirm == "y":
self.config.logger.warn('NOT dropping table')
return 2
db.drop_database(database_name)
db.commit()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# create database section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if self.config.createdb:
self.config.logger.info('create database section with %s',
superuser_pg_url)
with PostgreSQLAlchemyManager(superuser_pg_url,
self.config.logger,
autocommit=False) as db:
db.create_database(database_name)
if self.config.no_roles:
self.config.logger.info("Skipping role creation")
else:
db.create_roles(self.config)
db.commit()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# database extensions section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
self.config.logger.info('database extensions section with %s',
superuser_normaldb_pg_url)
with PostgreSQLAlchemyManager(superuser_normaldb_pg_url,
self.config.logger,
autocommit=False,
on_heroku=self.config.on_heroku) as db:
db.setup_extensions()
db.grant_public_schema_ownership(self.config.database_username)
db.commit()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# database schema section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if self.config.no_schema:
self.config.logger.info("not adding a schema")
return 0
alembic_cfg = Config(self.config.alembic_config)
alembic_cfg.set_main_option('sqlalchemy.url', normal_user_pg_url)
self.config.logger.info('database schema section with %s',
normal_user_pg_url)
with PostgreSQLAlchemyManager(normal_user_pg_url,
self.config.logger,
autocommit=False,
on_heroku=self.config.on_heroku) as db:
# Order matters below
db.turn_function_body_checks_off()
db.load_raw_sql('types')
db.load_raw_sql('procs')
# We need to commit to make a type visible for table creation
db.commit()
db.create_tables()
db.load_raw_sql('views')
db.commit()
if not self.config.get('no_staticdata'):
self.import_staticdata(db)
if self.config['fakedata']:
self.generate_fakedata(db, self.config['fakedata_days'])
db.commit()
command.stamp(alembic_cfg, "heads")
db.session.close()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# database owner section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
self.config.logger.info('database extensions section with %s',
superuser_normaldb_pg_url)
with PostgreSQLAlchemyManager(superuser_normaldb_pg_url,
self.config.logger,
autocommit=False,
on_heroku=self.config.on_heroku) as db:
db.set_table_owner(self.config.database_username)
db.set_default_owner(database_name, self.config.database_username)
db.set_grants(self.config) # config has user lists
return 0
def setUp(self):
command.stamp(self.cfg, "base")
def test_stamp():
with capture_context_buffer() as buf:
command.stamp(cfg, "head", sql=True)
assert "UPDATE alembic_version SET version_num='%s';" % c in buf.getvalue()
def test_stamp_creates_table(self):
command.stamp(self.cfg, "head")
eq_(
self.bind.scalar("select version_num from alembic_version"), self.b
)
def test_stamp_existing_downgrade(self):
command.stamp(self.cfg, self.b)
command.stamp(self.cfg, self.a)
eq_(
self.bind.scalar("select version_num from alembic_version"), self.a
)
def test_heads_one_is_dependent(self):
command.stamp(self.cfg, ())
command.stamp(self.cfg, (self.b2.revision,))
with self._assert_lines(["a2", "b2"]):
command.current(self.cfg)
def test_heads_upg(self):
command.stamp(self.cfg, (self.b2.revision,))
command.upgrade(self.cfg, (self.b3.revision))
with self._assert_lines(["a2", "b3"]):
command.current(self.cfg)
def test_plain_current(self):
command.stamp(self.cfg, ())
command.stamp(self.cfg, self.a3.revision)
with self._assert_lines(["a3"]):
command.current(self.cfg)
def test_two_heads(self):
command.stamp(self.cfg, ())
command.stamp(self.cfg, (self.a1.revision, self.b1.revision))
with self._assert_lines(["a1", "b1"]):
command.current(self.cfg)
def test_history_current_to_head_as_base(self):
command.stamp(self.cfg, "base")
self.cfg.stdout = buf = self._buf_fixture()
command.history(self.cfg, "current:", verbose=True)
self._eq_cmd_output(buf, [self.c, self.b, self.a], env_token=True)
def test_no_current(self):
command.stamp(self.cfg, ())
with self._assert_lines([]):
command.current(self.cfg)
def stamp_db(revision, dburl):
command.stamp(alembic_config(dburl=dburl), revision)
def initialize_database(engine: Engine):
import_all_modules()
Base.metadata.create_all(engine, checkfirst=False)
alembic_cfg = get_alembic_config(engine)
stamp(alembic_cfg, 'head')
def stamp_metadata_database(self):
alembic_cfg = self.get_alembic_config()
if self.engine is not None:
alembic_cfg.attributes["connection"] = self.engine
command.stamp(alembic_cfg, "23dd1cc88eb2")
def initialize_db(alembic_config) -> None:
log.info('[•] Initializing DB...')
Base.metadata.create_all(bind=engine, checkfirst=True)
command.stamp(alembic_config, 'head')
log.info('[✔] DB created ({path})'.format(path=DB.SQLALCHEMY_DATABASE_URI))
def setup_alembic_config():
alembic_cfg = Config("alembic.ini")
command.stamp(alembic_cfg, "head")
def initialize(args):
"""
This command exists to:
- Prevent the user having to type more than one thing
- Prevent the user seeing internals like 'manage.py' which we would
rather people were not messing with on production systems.
"""
log.info("Loading configuration..")
config = CalamariConfig()
# Generate django's SECRET_KEY setting
# Do this first, otherwise subsequent django ops will raise ImproperlyConfigured.
# Write into a file instead of directly, so that package upgrades etc won't spuriously
# prompt for modified config unless it really is modified.
if not os.path.exists(config.get('calamari_web', 'secret_key_path')):
chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
open(config.get('calamari_web', 'secret_key_path'),
'w').write(get_random_string(50, chars))
run_local_salt(sls=RELAX_SALT_PERMS_SLS, message='salt')
run_local_salt(sls=POSTGRES_SLS, message='postgres')
# Cthulhu's database
db_path = config.get('cthulhu', 'db_path')
engine = create_engine(db_path)
Base.metadata.reflect(engine)
alembic_config = AlembicConfig()
if ALEMBIC_TABLE in Base.metadata.tables:
log.info("Updating database...")
# Database already populated, migrate forward
command.upgrade(alembic_config, "head")
else:
log.info("Initializing database...")
# Blank database, do initial population
Base.metadata.create_all(engine)
command.stamp(alembic_config, "head")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "calamari_web.settings")
# Django's database
with quiet():
execute_from_command_line(["", "syncdb", "--noinput"])
create_default_roles()
create_admin_users(args)
log.info("Initializing web interface...")
# Django's static files
with quiet():
execute_from_command_line(["", "collectstatic", "--noinput"])
# Because we've loaded Django, it will have written log files as
# this user (probably root). Fix it so that apache can write them later.
apache_user = pwd.getpwnam(config.get('calamari_web', 'username'))
os.chown(config.get('calamari_web', 'log_path'), apache_user.pw_uid,
apache_user.pw_gid)
# Handle SQLite case, otherwise no chown is needed
if config.get('calamari_web', 'db_engine').endswith("sqlite3"):
os.chown(config.get('calamari_web', 'db_name'), apache_user.pw_uid,
apache_user.pw_gid)
# Start services, configure to run on boot
run_local_salt(sls=SERVICES_SLS, message='services')
# During an upgrade: update minions that were connected previously
update_connected_minions()
# Signal supervisor to restart cthulhu as we have created its database
log.info("Restarting services...")
subprocess.call(['supervisorctl', 'restart', 'cthulhu'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
subprocess.call(['supervisorctl', 'restart', 'carbon-cache'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# TODO: optionally generate or install HTTPS certs + hand to apache
log.info("Complete.")
def stamp(to_version='head', sql_url=None):
"""Stamp the specified version, with no migration performed."""
alembic_cfg = init_config(sql_url)
if alembic_cfg:
alembic_command.stamp(alembic_cfg, to_version)
def install(self, req, sess, allow_upgrades=False):
"""
Run module's installation hooks and register the module in DB.
"""
from alembic import command
from netprofile_core.models import NPModule
if not isinstance(req, Requirement):
req = Requirement(req)
moddef = req.name
modspec = req.specifier
if ('core' not in self.loaded) and (moddef != 'core'):
raise ModuleError(
'Unable to install anything prior to loading core module.')
if self.is_installed(req, sess):
if allow_upgrades and self.is_installed(Requirement(moddef), sess):
return self.upgrade(req, sess)
return False
ep = self._find_ep(moddef)
try:
modcls = ep.load()
except ImportError as e:
raise ModuleError(
'Can\'t locate ModuleBase class for module \'%s\'.' %
(moddef, )) from e
modversion = self._cls_version(modcls)
if modversion not in modspec:
raise ModuleError(
'Module \'%s\' version %s can\'t satisfy requirements: %s.' %
(moddef, str(modversion), str(modspec)))
vpair = VersionPair(None, modversion)
get_deps = getattr(modcls, 'get_deps', None)
if callable(get_deps):
for dep in get_deps():
depreq = Requirement(dep)
if not self.is_installed(depreq, sess):
self.install(depreq, sess, allow_upgrades=allow_upgrades)
modprep = getattr(modcls, 'prepare', None)
if callable(modprep):
modprep()
if moddef != 'core':
self.preload(moddef)
get_models = getattr(modcls, 'get_models', None)
if callable(get_models):
tables = [model.__table__ for model in get_models()]
Base.metadata.create_all(sess.bind, tables)
get_sql_functions = getattr(modcls, 'get_sql_functions', None)
if callable(get_sql_functions):
for func in get_sql_functions():
sess.execute(func.create(moddef))
get_sql_views = getattr(modcls, 'get_sql_views', None)
if callable(get_sql_views):
for view in get_sql_views():
sess.execute(view.create())
modobj = NPModule(id=None)
modobj.name = moddef
modobj.current_version = str(modversion)
if moddef == 'core':
modobj.enabled = True
sess.add(modobj)
sess.flush()
get_sql_data = getattr(modcls, 'get_sql_data', None)
if callable(get_sql_data):
get_sql_data(modobj, vpair, sess)
mod_install = getattr(modcls, 'install', None)
if callable(mod_install):
mod_install(sess)
if self.installed is None:
self.installed = dict()
self.installed[moddef] = modversion
transaction.commit()
get_sql_events = getattr(modcls, 'get_sql_events', None)
if callable(get_sql_events):
for evt in get_sql_events():
sess.execute(evt.create(moddef))
transaction.commit()
alembic_cfg = get_alembic_config(self, stdout=self.stdout)
command.stamp(alembic_cfg, moddef + '@head')
transaction.commit()
if moddef == 'core':
self.load(moddef)
return True