def main() -> None:
"""
Command-line entry point.
"""
parser = argparse.ArgumentParser(
description="Tool to initialize the database used by CRATE's "
"implementation of an NLPRP server.")
parser.add_argument(
"config_uri",
type=str,
help="Config file to read (e.g. 'development.ini'); URL of database "
"is found here.")
args = parser.parse_args()
main_only_quicksetup_rootlogger()
config_file = args.config_uri
log.debug(f"Settings file: {config_file}")
settings = get_appsettings(config_file)
engine = engine_from_config(settings,
NlpServerConfigKeys._SQLALCHEMY_PREFIX)
sqla_url = get_safe_url_from_engine(engine)
log.info(f"Using database {sqla_url!r}")
dbsession.configure(bind=engine)
log.info("Creating database structure...")
Base.metadata.create_all(engine)
log.info("... done.")
def export_whole_database(req: "CamcopsRequest",
recipient: ExportRecipient,
via_index: bool = True) -> None:
"""
Exports to a database.
Holds a recipient-specific file lock in the process.
Args:
req: a :class:`camcops_server.cc_modules.cc_request.CamcopsRequest`
recipient: an :class:`camcops_server.cc_modules.cc_exportmodels.ExportRecipient`
via_index: use the task index (faster)?
""" # noqa
cfg = req.config
lockfilename = cfg.get_export_lockfilename_db(
recipient_name=recipient.recipient_name)
try:
with lockfile.FileLock(lockfilename, timeout=0): # doesn't wait
collection = get_collection_for_export(req,
recipient,
via_index=via_index)
dst_engine = create_engine(recipient.db_url,
echo=recipient.db_echo)
log.info("Exporting to database: {}",
get_safe_url_from_engine(dst_engine))
dst_session = sessionmaker(bind=dst_engine)() # type: SqlASession
task_generator = gen_tasks_having_exportedtasks(collection)
export_options = TaskExportOptions(
include_blobs=recipient.db_include_blobs,
# *** todo: other options, specifically DB_PATIENT_ID_PER_ROW
)
copy_tasks_and_summaries(
tasks=task_generator,
dst_engine=dst_engine,
dst_session=dst_session,
export_options=export_options,
req=req,
)
dst_session.commit()
except lockfile.AlreadyLocked:
log.warning(
"Export logfile {!r} already locked by another process; "
"aborting", lockfilename)
def make_wsgi_app(global_config: Dict[Any, Any], **settings) -> Router:
# Logging
main_only_quicksetup_rootlogger()
logging.getLogger('sqlalchemy').setLevel(logging.WARNING)
# log.debug(f"global_config: {global_config!r}")
# log.debug(f"settings: {settings!r}")
# Database
engine = engine_from_config(settings,
NlpServerConfigKeys._SQLALCHEMY_PREFIX,
**SQLALCHEMY_COMMON_OPTIONS)
# ... add to config - pool_recycle is set to create new sessions every 7h
sqla_url = get_safe_url_from_engine(engine)
log.info(f"Using database {sqla_url!r}")
dbsession.configure(bind=engine)
Base.metadata.bind = engine
# Pyramid
config = Configurator(settings=settings)
# Security policies
authn_policy = AuthTktAuthenticationPolicy(
settings[NlpServerConfigKeys.NLP_WEBSERVER_SECRET],
secure=True, # only allow requests over HTTPS
hashalg='sha512')
authz_policy = ACLAuthorizationPolicy()
config.set_authentication_policy(authn_policy)
config.set_authorization_policy(authz_policy)
# Compression
config.add_tween("cardinal_pythonlib.pyramid.compression.CompressionTweenFactory") # noqa
# Routes
config.add_route('index', '/')
config.scan('.views')
# Create WSGI app
return config.make_wsgi_app()
def create_table_from_orm_class(engine: Engine,
ormclass: DeclarativeMeta,
without_constraints: bool = False) -> None:
"""
From an SQLAlchemy ORM class, creates the database table via the specified
engine, using a ``CREATE TABLE`` SQL (DDL) statement.
Args:
engine: SQLAlchemy :class:`Engine` object
ormclass: SQLAlchemy ORM class
without_constraints: don't add foreign key constraints
"""
table = ormclass.__table__ # type: Table
log.info("Creating table {} on engine {}{}", table.name,
get_safe_url_from_engine(engine),
" (omitting constraints)" if without_constraints else "")
# https://stackoverflow.com/questions/19175311/how-to-create-only-one-table-with-sqlalchemy # noqa
if without_constraints:
include_foreign_key_constraints = []
else:
include_foreign_key_constraints = None # the default
creator = CreateTable(
table, include_foreign_key_constraints=include_foreign_key_constraints)
creator.execute(bind=engine)
def merge_camcops_db(
src: str,
echo: bool,
report_every: int,
dummy_run: bool,
info_only: bool,
default_group_id: Optional[int],
default_group_name: Optional[str],
groupnum_map: Dict[int, int],
whichidnum_map: Dict[int, int],
skip_export_logs: bool = True,
skip_audit_logs: bool = True,
) -> None:
"""
Merge an existing database (with a pre-v2 or later structure) into a
comtemporary CamCOPS database.
Args:
src:
source database SQLAlchemy URL
echo:
echo the SQL that is produced?
report_every:
provide a progress report every *n* records
dummy_run:
don't alter the destination database
info_only:
show info, then stop
default_group_id:
integer group ID (in the destination database) to use for source
records that have no group (because they come from a very old
source database) but need one
default_group_name:
group name (in the destination database) to use for source
records that have no group (because they come from a very old
source database) but need one
groupnum_map:
dictionary mapping group ID values from the source database to
the destination database
whichidnum_map:
dictionary mapping ``which_idnum`` values from the source database
to the destination database
skip_export_logs:
skip export log tables
skip_audit_logs:
skip audit log table
"""
req = get_command_line_request() # requires manual COMMIT; see below
src_engine = create_engine(src, echo=echo, pool_pre_ping=True)
log.info("SOURCE: " + get_safe_url_from_engine(src_engine))
log.info("DESTINATION: " + get_safe_url_from_engine(req.engine))
log.info(
"Destination ID number type map (source:destination) is: {!r}",
whichidnum_map,
)
log.info("Group number type map (source:destination) is {!r}",
groupnum_map)
# Delay the slow import until we've checked our syntax
log.info("Loading all models...")
# noinspection PyUnresolvedReferences
import camcops_server.cc_modules.cc_all_models # delayed import # import side effects (ensure all models registered) # noqa
log.info("Models loaded.")
# Now, any special dependencies?
# From the point of view of translating any tablet-related fields, the
# actual (server) PK values are irrelevant; all relationships will be
# identical if you change any PK (not standard database practice, but
# convenient here).
# The dependencies that do matter are server-side things, like user_id
# variables.
# For debugging only, some junk:
# test_dependencies = [
# TableDependency(parent_tablename="patient",
# child_tablename="_dirty_tables")
# ]
# -------------------------------------------------------------------------
# Tables to skip
# -------------------------------------------------------------------------
skip_tables = [
# Transient stuff we don't want to copy across, or wouldn't want to
# overwrite the destination with, or where the PK structure has
# changed and we don't care about old data:
TableIdentity(tablename=x) for x in (
CamcopsSession.__tablename__,
DirtyTable.__tablename__,
ServerSettings.__tablename__,
SecurityAccountLockout.__tablename__,
SecurityLoginFailure.__tablename__,
UserGroupMembership.__tablename__,
group_group_table.name,
)
]
# Tedious and bulky stuff the user may want to skip:
if skip_export_logs:
skip_tables.extend([
TableIdentity(tablename=x) for x in (
Email.__tablename__,
ExportRecipient.__tablename__,
ExportedTask.__tablename__,
ExportedTaskEmail.__tablename__,
ExportedTaskFileGroup.__tablename__,
ExportedTaskHL7Message.__tablename__,
)
])
if skip_audit_logs:
skip_tables.append(TableIdentity(tablename=AuditEntry.__tablename__))
# -------------------------------------------------------------------------
# Initial operations on SOURCE database
# -------------------------------------------------------------------------
src_tables = get_table_names(src_engine)
skip_tables += get_skip_tables(src_tables=src_tables)
src_iddefs = get_src_iddefs(src_engine, src_tables)
log.info("Source ID number definitions: {!r}", src_iddefs)
# -------------------------------------------------------------------------
# Initial operations on DESTINATION database
# -------------------------------------------------------------------------
dst_session = req.dbsession
# So that system users get the first ID (cosmetic!):
_ = User.get_system_user(dbsession=dst_session)
_ = Device.get_server_device(dbsession=dst_session)
# -------------------------------------------------------------------------
# Set up source-to-destination mappings
# -------------------------------------------------------------------------
# Map source to destination ID number types
for src_which_idnum, dest_which_idnum in whichidnum_map.items():
assert isinstance(src_which_idnum, int)
assert isinstance(dest_which_idnum, int)
src_iddef = src_iddefs[src_which_idnum]
dst_iddef = ensure_dest_iddef_exists(dest_which_idnum, dst_session)
ensure_no_iddef_clash(src_iddef, dst_iddef)
# Map source to destination group numbers
for src_groupnum, dest_groupnum in groupnum_map.items():
assert isinstance(src_groupnum, int)
assert isinstance(dest_groupnum, int)
_ = get_dst_group(dest_groupnum, dst_session)
# -------------------------------------------------------------------------
# Merge
# -------------------------------------------------------------------------
# Merge! It's easy...
trcon_info = dict(
default_group_id=default_group_id,
default_group_name=default_group_name,
src_iddefs=src_iddefs,
whichidnum_map=whichidnum_map,
groupnum_map=groupnum_map,
)
merge_db(
base_class=Base,
src_engine=src_engine,
dst_session=dst_session,
allow_missing_src_tables=True,
allow_missing_src_columns=True,
translate_fn=translate_fn,
skip_tables=skip_tables,
only_tables=None,
tables_to_keep_pks_for=None,
# extra_table_dependencies=test_dependencies,
extra_table_dependencies=None,
dummy_run=dummy_run,
info_only=info_only,
report_every=report_every,
flush_per_table=True,
flush_per_record=False,
commit_with_flush=False,
commit_at_end=True,
prevent_eager_load=True,
trcon_info=trcon_info,
)
# -------------------------------------------------------------------------
# Postprocess
# -------------------------------------------------------------------------
postprocess(src_engine=src_engine, dst_session=dst_session)
# -------------------------------------------------------------------------
# Done
# -------------------------------------------------------------------------
dst_session.commit()
