def created_on_server(self, req: "CamcopsRequest") -> bool:
"""
Was this record created on the server?
"""
from camcops_server.cc_modules.cc_device import (
Device, ) # delayed import
server_device = Device.get_server_device(req.dbsession)
return self._era == ERA_NOW and self._device_id == server_device.id
def merge_camcops_db(
src: str,
echo: bool,
report_every: int,
dummy_run: bool,
info_only: bool,
default_group_id: Optional[int],
default_group_name: Optional[str],
groupnum_map: Dict[int, int],
whichidnum_map: Dict[int, int],
skip_export_logs: bool = True,
skip_audit_logs: bool = True,
) -> None:
"""
Merge an existing database (with a pre-v2 or later structure) into a
comtemporary CamCOPS database.
Args:
src:
source database SQLAlchemy URL
echo:
echo the SQL that is produced?
report_every:
provide a progress report every *n* records
dummy_run:
don't alter the destination database
info_only:
show info, then stop
default_group_id:
integer group ID (in the destination database) to use for source
records that have no group (because they come from a very old
source database) but need one
default_group_name:
group name (in the destination database) to use for source
records that have no group (because they come from a very old
source database) but need one
groupnum_map:
dictionary mapping group ID values from the source database to
the destination database
whichidnum_map:
dictionary mapping ``which_idnum`` values from the source database
to the destination database
skip_export_logs:
skip export log tables
skip_audit_logs:
skip audit log table
"""
req = get_command_line_request() # requires manual COMMIT; see below
src_engine = create_engine(src, echo=echo, pool_pre_ping=True)
log.info("SOURCE: " + get_safe_url_from_engine(src_engine))
log.info("DESTINATION: " + get_safe_url_from_engine(req.engine))
log.info(
"Destination ID number type map (source:destination) is: {!r}",
whichidnum_map,
)
log.info("Group number type map (source:destination) is {!r}",
groupnum_map)
# Delay the slow import until we've checked our syntax
log.info("Loading all models...")
# noinspection PyUnresolvedReferences
import camcops_server.cc_modules.cc_all_models # delayed import # import side effects (ensure all models registered) # noqa
log.info("Models loaded.")
# Now, any special dependencies?
# From the point of view of translating any tablet-related fields, the
# actual (server) PK values are irrelevant; all relationships will be
# identical if you change any PK (not standard database practice, but
# convenient here).
# The dependencies that do matter are server-side things, like user_id
# variables.
# For debugging only, some junk:
# test_dependencies = [
# TableDependency(parent_tablename="patient",
# child_tablename="_dirty_tables")
# ]
# -------------------------------------------------------------------------
# Tables to skip
# -------------------------------------------------------------------------
skip_tables = [
# Transient stuff we don't want to copy across, or wouldn't want to
# overwrite the destination with, or where the PK structure has
# changed and we don't care about old data:
TableIdentity(tablename=x) for x in (
CamcopsSession.__tablename__,
DirtyTable.__tablename__,
ServerSettings.__tablename__,
SecurityAccountLockout.__tablename__,
SecurityLoginFailure.__tablename__,
UserGroupMembership.__tablename__,
group_group_table.name,
)
]
# Tedious and bulky stuff the user may want to skip:
if skip_export_logs:
skip_tables.extend([
TableIdentity(tablename=x) for x in (
Email.__tablename__,
ExportRecipient.__tablename__,
ExportedTask.__tablename__,
ExportedTaskEmail.__tablename__,
ExportedTaskFileGroup.__tablename__,
ExportedTaskHL7Message.__tablename__,
)
])
if skip_audit_logs:
skip_tables.append(TableIdentity(tablename=AuditEntry.__tablename__))
# -------------------------------------------------------------------------
# Initial operations on SOURCE database
# -------------------------------------------------------------------------
src_tables = get_table_names(src_engine)
skip_tables += get_skip_tables(src_tables=src_tables)
src_iddefs = get_src_iddefs(src_engine, src_tables)
log.info("Source ID number definitions: {!r}", src_iddefs)
# -------------------------------------------------------------------------
# Initial operations on DESTINATION database
# -------------------------------------------------------------------------
dst_session = req.dbsession
# So that system users get the first ID (cosmetic!):
_ = User.get_system_user(dbsession=dst_session)
_ = Device.get_server_device(dbsession=dst_session)
# -------------------------------------------------------------------------
# Set up source-to-destination mappings
# -------------------------------------------------------------------------
# Map source to destination ID number types
for src_which_idnum, dest_which_idnum in whichidnum_map.items():
assert isinstance(src_which_idnum, int)
assert isinstance(dest_which_idnum, int)
src_iddef = src_iddefs[src_which_idnum]
dst_iddef = ensure_dest_iddef_exists(dest_which_idnum, dst_session)
ensure_no_iddef_clash(src_iddef, dst_iddef)
# Map source to destination group numbers
for src_groupnum, dest_groupnum in groupnum_map.items():
assert isinstance(src_groupnum, int)
assert isinstance(dest_groupnum, int)
_ = get_dst_group(dest_groupnum, dst_session)
# -------------------------------------------------------------------------
# Merge
# -------------------------------------------------------------------------
# Merge! It's easy...
trcon_info = dict(
default_group_id=default_group_id,
default_group_name=default_group_name,
src_iddefs=src_iddefs,
whichidnum_map=whichidnum_map,
groupnum_map=groupnum_map,
)
merge_db(
base_class=Base,
src_engine=src_engine,
dst_session=dst_session,
allow_missing_src_tables=True,
allow_missing_src_columns=True,
translate_fn=translate_fn,
skip_tables=skip_tables,
only_tables=None,
tables_to_keep_pks_for=None,
# extra_table_dependencies=test_dependencies,
extra_table_dependencies=None,
dummy_run=dummy_run,
info_only=info_only,
report_every=report_every,
flush_per_table=True,
flush_per_record=False,
commit_with_flush=False,
commit_at_end=True,
prevent_eager_load=True,
trcon_info=trcon_info,
)
# -------------------------------------------------------------------------
# Postprocess
# -------------------------------------------------------------------------
postprocess(src_engine=src_engine, dst_session=dst_session)
# -------------------------------------------------------------------------
# Done
# -------------------------------------------------------------------------
dst_session.commit()
def setUp(self) -> None:
super().setUp()
from cardinal_pythonlib.datetimefunc import (
convert_datetime_to_utc,
format_datetime,
)
from camcops_server.cc_modules.cc_blob import Blob
from camcops_server.cc_modules.cc_constants import DateFormat
from camcops_server.cc_modules.cc_device import Device
from camcops_server.cc_modules.cc_group import Group
from camcops_server.cc_modules.cc_patient import Patient
from camcops_server.cc_modules.cc_patientidnum import PatientIdNum
from camcops_server.cc_modules.cc_task import Task
from camcops_server.cc_modules.cc_user import User
from camcops_server.tasks.photo import Photo
Base.metadata.create_all(self.engine)
self.era_time = pendulum.parse("2010-07-07T13:40+0100")
self.era_time_utc = convert_datetime_to_utc(self.era_time)
self.era = format_datetime(self.era_time, DateFormat.ISO8601)
# Set up groups, users, etc.
# ... ID number definitions
iddef1 = IdNumDefinition(which_idnum=1,
description="NHS number",
short_description="NHS#",
hl7_assigning_authority="NHS",
hl7_id_type="NHSN")
self.dbsession.add(iddef1)
iddef2 = IdNumDefinition(which_idnum=2,
description="RiO number",
short_description="RiO",
hl7_assigning_authority="CPFT",
hl7_id_type="CPFT_RiO")
self.dbsession.add(iddef2)
# ... group
self.group = Group()
self.group.name = "testgroup"
self.group.description = "Test group"
self.group.upload_policy = "sex AND anyidnum"
self.group.finalize_policy = "sex AND idnum1"
self.dbsession.add(self.group)
self.dbsession.flush() # sets PK fields
# ... users
self.user = User.get_system_user(self.dbsession)
self.user.upload_group_id = self.group.id
self.req._debugging_user = self.user # improve our debugging user
# ... devices
self.server_device = Device.get_server_device(self.dbsession)
self.other_device = Device()
self.other_device.name = "other_device"
self.other_device.friendly_name = "Test device that may upload"
self.other_device.registered_by_user = self.user
self.other_device.when_registered_utc = self.era_time_utc
self.other_device.camcops_version = CAMCOPS_SERVER_VERSION
self.dbsession.add(self.other_device)
self.dbsession.flush() # sets PK fields
# Populate database with two of everything
p1 = Patient()
p1.id = 1
self._apply_standard_db_fields(p1)
p1.forename = "Forename1"
p1.surname = "Surname1"
p1.dob = pendulum.parse("1950-01-01")
self.dbsession.add(p1)
p1_idnum1 = PatientIdNum()
p1_idnum1.id = 1
self._apply_standard_db_fields(p1_idnum1)
p1_idnum1.patient_id = p1.id
p1_idnum1.which_idnum = iddef1.which_idnum
p1_idnum1.idnum_value = 333
self.dbsession.add(p1_idnum1)
p1_idnum2 = PatientIdNum()
p1_idnum2.id = 2
self._apply_standard_db_fields(p1_idnum2)
p1_idnum2.patient_id = p1.id
p1_idnum2.which_idnum = iddef2.which_idnum
p1_idnum2.idnum_value = 444
self.dbsession.add(p1_idnum2)
p2 = Patient()
p2.id = 2
self._apply_standard_db_fields(p2)
p2.forename = "Forename2"
p2.surname = "Surname2"
p2.dob = pendulum.parse("1975-12-12")
self.dbsession.add(p2)
p2_idnum1 = PatientIdNum()
p2_idnum1.id = 3
self._apply_standard_db_fields(p2_idnum1)
p2_idnum1.patient_id = p2.id
p2_idnum1.which_idnum = iddef1.which_idnum
p2_idnum1.idnum_value = 555
self.dbsession.add(p2_idnum1)
self.dbsession.flush()
for cls in Task.all_subclasses_by_tablename():
t1 = cls()
t1.id = 1
self._apply_standard_task_fields(t1)
if t1.has_patient:
t1.patient_id = p1.id
if isinstance(t1, Photo):
b = Blob()
b.id = 1
self._apply_standard_db_fields(b)
b.tablename = t1.tablename
b.tablepk = t1.id
b.fieldname = 'photo_blobid'
b.filename = "some_picture.png"
b.mimetype = MimeType.PNG
b.image_rotation_deg_cw = 0
b.theblob = DEMO_PNG_BYTES
self.dbsession.add(b)
t1.photo_blobid = b.id
self.dbsession.add(t1)
t2 = cls()
t2.id = 2
self._apply_standard_task_fields(t2)
if t2.has_patient:
t2.patient_id = p2.id
self.dbsession.add(t2)
self.dbsession.commit()
def setUp(self) -> None:
super().setUp()
self.set_era("2010-07-07T13:40+0100")
# Set up groups, users, etc.
# ... ID number definitions
idnum_type_nhs = 1
idnum_type_rio = 2
idnum_type_study = 3
self.nhs_iddef = IdNumDefinition(
which_idnum=idnum_type_nhs,
description="NHS number",
short_description="NHS#",
hl7_assigning_authority="NHS",
hl7_id_type="NHSN",
)
self.dbsession.add(self.nhs_iddef)
self.rio_iddef = IdNumDefinition(
which_idnum=idnum_type_rio,
description="RiO number",
short_description="RiO",
hl7_assigning_authority="CPFT",
hl7_id_type="CPRiO",
)
self.dbsession.add(self.rio_iddef)
self.study_iddef = IdNumDefinition(
which_idnum=idnum_type_study,
description="Study number",
short_description="Study",
)
self.dbsession.add(self.study_iddef)
# ... group
self.group = Group()
self.group.name = "testgroup"
self.group.description = "Test group"
self.group.upload_policy = "sex AND anyidnum"
self.group.finalize_policy = "sex AND idnum1"
self.group.ip_use = IpUse()
self.dbsession.add(self.group)
self.dbsession.flush() # sets PK fields
# ... users
self.user = User.get_system_user(self.dbsession)
self.user.upload_group_id = self.group.id
self.req._debugging_user = self.user # improve our debugging user
# ... devices
self.server_device = Device.get_server_device(self.dbsession)
self.other_device = Device()
self.other_device.name = "other_device"
self.other_device.friendly_name = "Test device that may upload"
self.other_device.registered_by_user = self.user
self.other_device.when_registered_utc = self.era_time_utc
self.other_device.camcops_version = CAMCOPS_SERVER_VERSION
self.dbsession.add(self.other_device)
# ... export recipient definition (the minimum)
self.recipdef.primary_idnum = idnum_type_nhs
self.dbsession.flush() # sets PK fields
self.create_tasks()