def async_db(request):
"""
Фикстура для асинхронного взаимодействия с базой PostgreSQL через PeeWee ORM
"""
# Создание базы
process = Popen(['createdb', TEST_DB])
process.communicate()
db.allow_sync = True
# Миграции
migrator = Router(db)
migrator.migrate_dir = os.path.join(base_dir, 'application', 'migrations')
migrator.run()
db.allow_sync = False
def teardown():
terminate_sql = ("SELECT pg_terminate_backend(pg_stat_activity.pid) "
"FROM pg_stat_activity "
"WHERE pg_stat_activity.datname = '%s' "
"AND pid <> pg_backend_pid();" % TEST_DB)
process = Popen(['psql', '-c', terminate_sql])
process.communicate()
process = Popen(['dropdb', TEST_DB])
process.communicate()
request.addfinalizer(teardown)
return db
def migrate(name=None):
"""
Runs migrations
"""
migrator = Router(db, logger=LOGGER)
migrator.migrate_dir = os.path.join(base_dir, 'application', 'migrations')
migrator.run(name=name)
class Migrations(object):
"""
Migrations
Handle all migrations during application start.
"""
def __init__(self, config):
# Based on configuration, select database to connect to.
if config['TYPE'] == 'SQLITE':
# Create SQLite directory if not exists
db_file_directory = os.path.dirname(config['FILE'])
if not os.path.exists(db_file_directory):
os.makedirs(db_file_directory)
database = SqliteDatabase(config['FILE'])
self.router = Router(database=database,
migrate_dir=config['MIGRATIONS_DIR'])
def run_all(self):
"""
Run all new migrations.
Migrations that have already been run will be ignored.
:return:
"""
self.router.run()
async def main():
load_modules(os.getcwd())
db.connect()
router = Router(db)
router.run()
mailer = Mailer(token=BOT_API_TOKEN, update_frequency=UPDATE_FREQUENCY)
await start_bot_jobs(mailer)
def test_migrations(app_before_init_db):
"""The database migrations complete successfully."""
app, conf_obj = app_before_init_db
dbp.connect()
if conf_obj.database.engine == "PostgresqlDatabase":
dbp.execute_sql("DROP SCHEMA public CASCADE;")
dbp.execute_sql("CREATE SCHEMA public;")
dbp.execute_sql("GRANT ALL ON SCHEMA public TO public;")
router = Router(dbp, migrate_dir="migrations", ignore=["basemodel"])
router.run()
applied_migrations = list(router.done)
applied_migrations.reverse()
# Shut up a warning in rollback that we can't do anything about.
logging.getLogger("peewee_migrate").warn = Mock()
# Make sure new rollbacks work. The existing ones are what they are.
for m in applied_migrations:
if m == "029_message_read":
break
router.rollback(m)
dbp.close()
def migrate():
database = load_plugin("chain.plugins.database")
migrate_dir = os.path.join(os.getcwd(), "chain", "plugins", "database",
"migrations")
router = Router(database.db, migrate_dir=migrate_dir)
router.run()
def migrate(self):
from peewee_migrate import Router
with self._db() as db, db.atomic():
router = Router(db,
migrate_table=self.migrate_table,
migrate_dir=self.migrate_dir)
router.run()
def app():
flask_app = create_app(configs['testing'])
router = Router(flask_app.db.database)
router.run()
yield flask_app
flask_app.db.close_db('')
current_path = os.path.dirname(__file__)
os.remove('{}/../test.db'.format(current_path))
def run_migrations(collection_old_db_path):
# TODO: write tests for this
migrations_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
"database/migrations")
directory = os.path.dirname(collection_old_db_path)
router = Router(
SqliteDatabase(os.path.join(directory, 'collection_new.sqlite3')),
migrations_dir)
router.run()
def apply_migrations():
# type: () -> None
logger.info('Applying migrations')
# Run all unapplied migrations
db = Database.get_db()
gateway_src = os.path.abspath(os.path.join(__file__, '..'))
router = Router(db,
migrate_dir=os.path.join(gateway_src, 'migrations/orm'))
router.run()
def migrate(self, _):
"""Migrates database schema."""
from peewee_migrate import Router
with self._db('cardinal') as db, db.atomic():
router = Router(db,
migrate_table=self.migrate_table,
migrate_dir=self.migrate_dir)
router.run()
def migrate_tables():
from peewee_migrate import Router
router = Router(db)
# Create migration
router.create('migration_name', auto=True)
# Run migration/migrations
router.run('migration_name')
# Run all unapplied migrations
router.run()
def setup_db():
# init db
models.database.init(host=settings.DB_HOST,
database=DB_NAME,
user=settings.DB_USER,
password=settings.DB_PASSWORD)
# run migrations
router = Router(models.database)
router.run()
def create():
database = db.database
database.connect()
router = Router(database)
# Create migration
# router.create(name=BaseModel.sub_models())
router.create(auto=db)
# Run all unapplied migrations
router.run()
def migrate():
database = PostgresqlDatabase(
database='postgres',
user='******',
host='127.0.0.1',
port='5432',
# password='******'
)
router = Router(database)
router.run()
def run(path):
items.db.connect()
router = Router(items.db,
ignore="basemodel",
migrate_dir=path,
migrate_table='toolsmigrate')
router.create(auto=items)
router.run()
items.db.close()
def makemigrate(model, ignore=['basemodel'], name='auto'):
if models.db.is_closed():
models.db.connect()
router = Router(model.db,
ignore=['basemodel'],
migrate_dir=os.path.join(
os.path.dirname(
os.path.dirname(os.path.realpath(__file__))),
'migrations'))
router.create(name='auto', auto=model)
router.run()
if not models.db.is_closed():
models.db.close()
def run(path):
db_items_module.db.connect()
# migrate_table: 迁移表的名称
router = Router(db_items_module.db,
ignore="basemodel",
migrate_dir=path,
migrate_table='toolsmigrate')
router.create(auto=db_items_module)
router.run()
db_items_module.db.close()
def init_database(self):
migrate_db = SqliteExtDatabase(self.config.database.path)
# Run migrations
del (logging.getLogger('peewee_migrate').handlers[:])
router = Router(migrate_db)
router.run()
migrate_db.close()
self.db = SqliteQueueDatabase(self.config.database.path)
models = [Event]
self.db.bind(models)
def migrate(name):
router = Router(models.database, ignore=[models.MyModel])
if name is None:
router.create(auto=models)
name = router.todo[-1]
try:
backup(name)
router.run()
except:
logging.exception("migrate失败")
clean(name, backup_py=True)
sys.exit(-1)
def migrate(name=None, fake=False):
"""makemigrate
:param name: 迁移文件名称
:param fake: fake=True, 如果你的数据库已经有了数据, 且不想删除它
"""
router = Router(db, ignore='basemodel')
names = [mm.name for mm in router.model.select().order_by(router.model.id)]
if name:
router.run(name, fake)
else:
path = pathlib.Path(__file__).parent
migrations_path = path.joinpath('migrations')
for single in migrations_path.rglob('*_auto.py'):
if single.stem not in names:
router.run(single.stem, fake)
def app():
config = configs['testing']
flask_app = create_app(config)
router = Router(
flask_app.db.database,
migrate_dir=f'{DEFAULT_MIGRATE_DIR}',
)
# log_router = Router(
# flask_app.logdb.database,
# migrate_dir=f'{DEFAULT_MIGRATE_DIR}/logs',
# )
router.run()
# log_router.run()
yield flask_app
flask_app.db.close_db('')
# flask_app.logdb.close_db('')
current_path = os.path.dirname(__file__)
def init_database(self):
# Migrate DB location
old_db_path = os.path.join(CLIPS_DIR, "frigate.db")
if not os.path.isfile(
self.config.database.path) and os.path.isfile(old_db_path):
os.rename(old_db_path, self.config.database.path)
# Migrate DB schema
migrate_db = SqliteExtDatabase(self.config.database.path)
# Run migrations
del logging.getLogger("peewee_migrate").handlers[:]
router = Router(migrate_db)
router.run()
migrate_db.close()
self.db = SqliteQueueDatabase(self.config.database.path)
models = [Event, Recordings]
self.db.bind(models)
def init_db():
try:
if SETTINGS_DATABASE.get('sql_log_enabled', False):
import logging
logger = logging.getLogger('peewee')
logger.addHandler(logging.StreamHandler())
logger.setLevel(logging.DEBUG)
db.connect()
router = Router(database=db,
migrate_dir=os.path.join(CURDIR,
'server/db/migrations'))
router.run()
data = SETTINGS_DATABASE.get('data', False)
if data:
create_db_data(data)
finally:
db.close()
def _runmigration(migration_name):
router = Router(DB)
router.run(migration_name)
def pytest_runtest_setup():
colibris.setup()
if persist.is_enabled():
router = Router(persist.get_database(),
migrate_dir=persist.get_migrations_dir())
router.run()
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from peewee_migrate import Router
from playhouse.pool import PooledMySQLDatabase
from arch.task_manager.settings import DATABASE
data_base_config = DATABASE.copy()
# TODO: create instance according to the engine
engine = data_base_config.pop("engine")
db_name = data_base_config.pop("name")
DB = PooledMySQLDatabase(db_name, **data_base_config)
router = Router(DB)
# Create migration
router.create('task_manager')
# Run migration/migrations
router.run('task_manager')
# Run all unapplied migrations
router.run()
router = Router(database,
migrate_dir='../migrations'
if os.getcwd().endswith('scripts') else 'migrations',
ignore=['basemodel'])
parser = argparse.ArgumentParser(
description='Apply or manage database migrations.')
parser.add_argument('-c',
'--create',
metavar='NAME',
help='Creates a new migration')
parser.add_argument('-a',
'--auto',
metavar='NAME',
help='Creates a new migration (automatic)')
parser.add_argument('-r',
'--rollback',
metavar='NAME',
help='Rolls back a migration')
args = parser.parse_args()
if args.create:
router.create(args.create)
elif args.auto:
router.create(args.auto, 'app')
elif args.rollback:
router.rollback(args.rollback)
else:
router.run()
from tqdm import tqdm
import pandas as pd
from config import PURCHASES_CSV_PATH
def insert_denorm_data_from_temp_table(target_model, value_list, database):
with database as db:
sql = f"""INSERT INTO {target_model._meta.table_name}({', '.join(target_model.fields_to_copy())})
VALUES {', '.join(map(str, value_list))}
ON CONFLICT DO NOTHING"""
db.execute_sql(sql, commit=True)
def insert_denorm(models, database):
for df in tqdm(pd.read_csv(PURCHASES_CSV_PATH, chunksize=100000)):
for model in models:
df_model_fields = df[model.get_fields()]
tuples = [tuple(x) for x in df_model_fields.to_numpy()]
insert_denorm_data_from_temp_table(model, tuples, database)
if __name__ == '__main__':
from models import db, TransactionProduct, Transaction
from peewee_migrate import Router
router = Router(db)
router.run('fill_initial_models')
insert_denorm([Transaction, TransactionProduct], db)
def migrate(self):
router = Router(database=self.database, migrate_dir=MIGRATE_DIR)
router.run()
