def full_import(org_only: bool, mora_base: str, mox_base: str):
"""Tool to do an initial full import."""
# Check connection to MO before we fire requests against SD
mh = MoraHelper(mora_base)
if not mh.check_connection():
raise click.ClickException("No MO reply, aborting.")
importer = ImportHelper(
create_defaults=True,
mox_base=mox_base,
mora_base=mora_base,
store_integration_data=False,
seperate_names=True,
)
sd = SdImport(importer,
settings=get_importer_settings(),
org_only=org_only,
ad_info=None)
sd.create_ou_tree(create_orphan_container=False,
sub_tree=None,
super_unit=None)
if not org_only:
sd.create_employees()
importer.import_all()
print("IMPORT DONE")
def create_new_root_and_it(settings=None) -> None:
"""Setup all necessary classes etc to perform opus-import."""
logger.warning(
"'create_new_root_and_it' is deprecated. Use os2mo-init instead.")
settings = settings or load_settings()
mox_base = settings.get("mox.base", "http://localhost:8080")
mora_base = settings.get("mora.base", "http://localhost:5000")
# Init
os2mo = ImportHelper(
create_defaults=True,
store_integration_data=True,
mox_base=mox_base,
mora_base=mora_base,
)
# The Organisation class is the main entry point,
# It exposes the related sub classes such as:
# Facet, Klasse, Itsystem, OrganisationUnit, Employee
main_name = settings.get("municipality.name", "Magenta ApS")
main_uuid = opus_helpers.generate_uuid(main_name)
os2mo.add_organisation(
identifier=main_name,
uuid=str(main_uuid),
user_key=main_name,
municipality_code=settings.get("municipality.code", 1234),
)
os2mo.new_itsystem(identifier=constants.Opus_it_system,
system_name=constants.Opus_it_system)
os2mo.new_itsystem(identifier=constants.AD_it_system,
system_name=constants.AD_it_system)
# Perfom setup of root unit and it systems.
os2mo.import_all()
def setUp(self):
self.importer = ImportHelper(
create_defaults=True,
mox_base=MOX_BASE,
mora_base=MORA_BASE,
system_name="SD-Import",
end_marker="SDSTOP",
store_integration_data=False,
)
def cli(**args):
SETTINGS = load_settings()
ad_reader = ADParameterReader()
if args['update']:
try:
start_opus_diff(ad_reader=ad_reader)
except RunDBInitException:
print('RunDB not initialized')
if args['import']:
importer = ImportHelper(
create_defaults=True,
mox_base=SETTINGS['mox.base'],
mora_base=SETTINGS['mora.base'],
store_integration_data=False,
seperate_names=True,
demand_consistent_uuids=False
)
try:
start_opus_import(importer, ad_reader=ad_reader, force=True)
except RunDBInitException:
print('RunDB not initialized')
if args['update_single_user']:
employment_number = args['update_single_user']
days = args['days']
update_employee(employment_number, days)
async def perform_initial_setup():
"""
Perform all initial bootstrapping of OS2mo.
Imports an organisation if missing, and adds all base facets
Imports all pretedetermined classes and it systems
"""
settings = config.get_config()
mox_helper = await create_mox_helper(settings.mox_base)
try:
await mox_helper.read_element_organisation_organisation(bvn="%")
except ElementNotFound:
print("No org found in LoRa. Performing initial setup.")
importer = ImportHelper(
create_defaults=True,
mox_base=settings.mox_base,
mora_base=settings.mora_base,
store_integration_data=False,
seperate_names=True,
)
importer.add_organisation(
identifier="Århus Kommune",
user_key="Århus Kommune",
municipality_code=751,
uuid=uuids.ORG_UUID,
)
# Perform initial import of org and facets
importer.import_all()
await import_remaining_classes()
await import_it()
def cli(**args):
SETTINGS = load_settings()
ad_reader = ADParameterReader()
if args['update']:
try:
start_opus_diff(ad_reader=ad_reader)
except RunDBInitException:
print('RunDB not initialized')
if args['import']:
importer = ImportHelper(
create_defaults=True,
mox_base=SETTINGS['mox.base'],
mora_base=SETTINGS['mora.base'],
store_integration_data=False,
seperate_names=True,
demand_consistent_uuids=False
)
med_name = 'MED Organisation'
importer.add_klasse(
identifier=med_name,
facet_type_ref='org_unit_type',
user_key=med_name,
scope='TEXT',
title=med_name
)
importer.add_organisation_unit(
identifier=med_name,
name=med_name,
user_key=med_name,
type_ref=med_name,
date_from='1930-01-01',
date_to=None,
parent_ref=None
)
try:
start_opus_import(importer, ad_reader=ad_reader, force=True)
except RunDBInitException:
print('RunDB not initialized')
if args['update_single_user']:
employment_number = args['update_single_user']
days = args['days']
update_employee(employment_number, days)
def example_import(mox_base, mora_base):
"""
Run the example to import the fictional organisation Magenta.
"""
# Init
os2mo = ImportHelper(create_defaults=True,
store_integration_data=True,
mox_base=mox_base,
mora_base=mora_base)
# The Organisation class is the main entry point,
# It exposes the related sub classes such as:
# Facet, Klasse, Itsystem, OrganisationUnit, Employee
os2mo.add_organisation(identifier="Magenta Aps",
user_key="Magenta",
municipality_code=101)
# Add klasse with reference to facet "org_unit_type"
os2mo.add_klasse(
identifier="Hovedenhed",
facet_type_ref="org_unit_type",
user_key="D1ED90C5-643A-4C12-8889-6B4174EF4467",
title="Hovedenhed" # This is the displayed value
)
os2mo.add_klasse(identifier="Afdeling",
facet_type_ref="org_unit_type",
user_key="91154D1E-E7CA-439B-B910-D4622FD3FD21",
title="Afdeling")
os2mo.add_klasse(identifier="Niveau 1",
facet_type_ref="org_unit_level",
user_key="Niveau 1",
title="Niveau 1")
os2mo.add_klasse(identifier="Niveau 2",
facet_type_ref="org_unit_level",
user_key="Niveau 2",
title="Niveau 2")
os2mo.add_klasse(identifier="Niveau 3",
facet_type_ref="org_unit_level",
user_key="Niveau 3",
title="Niveau 3")
os2mo.add_klasse(identifier="manuelt_primær",
facet_type_ref="primary_type",
user_key="Manuelt primær",
title="Manuelt primær",
scope='5000')
os2mo.add_klasse(identifier="primær",
facet_type_ref="primary_type",
user_key="Primær",
title="Primær",
scope='3000')
os2mo.add_klasse(identifier="ikke-primær",
facet_type_ref="primary_type",
user_key="Ikke-primær",
title="Ikke-primær",
scope='0')
os2mo.add_klasse(identifier="Arbejdstidsplaner",
facet_type_ref="time_planning",
user_key="Arbejdstidsplaner",
title="Arbejdstidsplaner")
os2mo.add_klasse(identifier="Tjenestetid",
facet_type_ref="time_planning",
user_key="Tjenestetid",
title="Tjenestetid")
# Root unit: Magenta
# Belongs to unit type: "Hovedenhed"
os2mo.add_organisation_unit(
identifier="Magenta",
name="Magenta Aps",
org_unit_level_ref="Niveau 1",
type_ref="Hovedenhed", # Reference to the unit type
time_planning_ref="Arbejdstidsplaner",
date_from="1986-01-01")
# Use parent_ref to make it a sub group of "Magenta"
os2mo.add_organisation_unit(
identifier="Pilestræde",
org_unit_level_ref="Niveau 2",
type_ref="Afdeling", # This unit is of type: Afdeling
parent_ref="Magenta", # Sub unit of/Belongs to Magenta
time_planning_ref="Tjenestetid",
date_from="1986-01-01")
os2mo.add_organisation_unit(
identifier="SJA2",
org_unit_level_ref="Niveau 2",
type_ref="Afdeling",
parent_ref="Magenta", # Sub unit of/Belongs to Magenta
date_from="1986-01-01",
)
# Adding sub units to the example
os2mo.add_organisation_unit(
identifier="Sysadmins",
type_ref="Afdeling",
org_unit_level_ref="Niveau 3",
parent_ref="SJA2", # Sub unit of/Belongs to SJA2
date_from="1986-01-01")
os2mo.add_organisation_unit(
identifier="Dummy",
type_ref="Afdeling",
org_unit_level_ref="Niveau 3",
parent_ref="Sysadmins", # Sub unit of/Belongs to SJA2
date_from="1986-01-01")
# Address Types
os2mo.add_klasse(identifier="AddressMailUnit",
facet_type_ref="org_unit_address_type",
title="Adresse",
scope="DAR",
example="<UUID>")
os2mo.add_address_type(organisation_unit="Magenta",
value="0a3f50c4-379f-32b8-e044-0003ba298018",
type_ref="AddressMailUnit",
date_from="1986-01-01")
os2mo.add_klasse(identifier="EAN",
facet_type_ref="org_unit_address_type",
user_key="C8EC85B4-A088-434A-B034-CA08A9FD655A",
title="EAN-nr.",
scope="EAN",
example="1234567890123")
os2mo.add_address_type(
organisation_unit="Magenta",
value="5264144870223",
type_ref="EAN",
date_from="1986-01-01",
)
os2mo.add_klasse(identifier="PhoneUnit",
facet_type_ref="org_unit_address_type",
title="Tlf",
scope="PHONE",
example="20304060")
os2mo.add_address_type(
organisation_unit="Magenta",
value="11223344",
type_ref="PhoneUnit",
date_from="1986-01-01",
)
# Create job functions and assign to employees
# Add job functions
os2mo.add_klasse(identifier="Direktør",
facet_type_ref="engagement_type",
user_key="Direktør",
title="Direktør")
os2mo.add_klasse(identifier="Projektleder",
facet_type_ref="engagement_type",
user_key="Projektleder",
title="Projektleder")
os2mo.add_klasse(identifier="Udvikler",
facet_type_ref="engagement_type",
user_key="Udvikler",
title="Udvikler")
os2mo.add_klasse(identifier="Projektmedarbejder",
facet_type_ref="engagement_type",
user_key="Projektmedarbejder",
title="Projektmedarbejder")
os2mo.add_employee(identifier="Susanne Chæf",
name=('Susanne', 'Chæf'),
cpr_no="0101862233")
os2mo.add_employee(identifier="Odin Perskov",
name=('Odin', 'Perskov'),
cpr_no="0102862234")
os2mo.add_employee(identifier="Ronja Rwander",
name=('Ronja', 'Rwander'),
cpr_no="0103862234")
os2mo.add_employee(identifier="Jens Mortensen",
name=('Jens', 'Mortensen'),
cpr_no="0104862235")
os2mo.add_employee(identifier="Bolette Buhl",
name=('Bolette', 'Buhl'),
cpr_no="0105862235")
os2mo.add_employee(identifier="Carl Sand Holth",
name=('Carl Sand', 'Holth'),
cpr_no="0106862235")
# Assign job functions
os2mo.add_klasse(identifier="Ansat",
facet_type_ref="engagement_type",
user_key="CF297115-309B-4535-88C8-5BA41C90929B",
title="Ansat")
os2mo.add_engagement(employee="Susanne Chæf",
organisation_unit="Magenta",
fraction=7,
job_function_ref="Direktør",
engagement_type_ref="Ansat",
primary_ref='primær',
date_from="2014-11-07")
os2mo.add_engagement(employee="Odin Perskov",
organisation_unit="Pilestræde",
job_function_ref="Projektleder",
engagement_type_ref="Ansat",
primary_ref='primær',
fraction=8,
date_from="2004-02-01")
os2mo.add_engagement(employee="Ronja Rwander",
organisation_unit="SJA2",
job_function_ref="Projektleder",
engagement_type_ref="Ansat",
primary_ref='primær',
date_from="2011-09-01")
os2mo.add_engagement(employee="Jens Mortensen",
organisation_unit="Pilestræde",
job_function_ref="Udvikler",
engagement_type_ref="Ansat",
primary_ref='primær',
fraction=2,
date_from="2003-04-01")
os2mo.add_engagement(employee="Bolette Buhl",
organisation_unit="SJA2",
job_function_ref="Udvikler",
engagement_type_ref="Ansat",
primary_ref='primær',
fraction=1,
date_from="2001-05-01")
os2mo.add_engagement(employee="Carl Sand Holth",
organisation_unit="Pilestræde",
job_function_ref="Projektmedarbejder",
primary_ref='primær',
engagement_type_ref="Ansat",
fraction=4,
date_from="1981-06-01")
os2mo.add_engagement(employee="Carl Sand Holth",
organisation_unit="Sysadmins",
job_function_ref="Udvikler",
primary_ref='ikke-primær',
engagement_type_ref="Ansat",
fraction=4,
date_from="1989-02-01")
# Association
os2mo.add_klasse(identifier="Ekstern Konsulent",
facet_type_ref="association_type",
user_key="F997F306-71DF-477C-AD42-E753F9C21B42",
title="Ekstern Konsulent")
os2mo.add_association(employee="Carl Sand Holth",
organisation_unit="Pilestræde",
association_type_ref="Ekstern Konsulent",
address_uuid="0a3f50c4-379f-32b8-e044-0003ba298018",
date_from="1986-10-01")
os2mo.add_klasse(identifier="AdressePostEmployee",
facet_type_ref="employee_address_type",
user_key="2F29C717-5D78-4AA9-BDAE-7CDB3A378018",
title="Adresse",
scope="DAR",
example="<UUID>")
os2mo.add_address_type(
employee="Susanne Chæf",
value="0a3f50a0-ef5a-32b8-e044-0003ba298018",
type_ref="AdressePostEmployee",
date_from="1986-11-01",
)
os2mo.add_address_type(
employee="Odin Perskov",
value="0a3f50a0-ef5a-32b8-e044-0003ba298018",
type_ref="AdressePostEmployee",
date_from="1986-11-01",
)
os2mo.add_address_type(
employee="Ronja Rwander",
value="0a3f50a0-ef5a-32b8-e044-0003ba298018",
type_ref="AdressePostEmployee",
date_from="1986-11-01",
)
os2mo.add_address_type(
employee="Jens Mortensen",
value="0a3f50a0-ef5a-32b8-e044-0003ba298018",
type_ref="AdressePostEmployee",
date_from="1986-11-01",
)
os2mo.add_address_type(
employee="Bolette Buhl",
value="0a3f50a0-ef5a-32b8-e044-0003ba298018",
type_ref="AdressePostEmployee",
date_from="1986-11-01",
)
os2mo.add_address_type(
employee="Carl Sand Holth",
value="0a3f50a0-ef5a-32b8-e044-0003ba298018",
type_ref="AdressePostEmployee",
date_from="1986-11-01",
)
# Add roles and assign to employees
os2mo.add_klasse(identifier="Medarbejder repræsentant",
facet_type_ref="role_type",
user_key="893A0670-BAFB-4DDF-8270-0EDACE6C520C",
title="Medarbejder repræsentant")
os2mo.add_klasse(identifier="Nøgleansvarlig",
facet_type_ref="role_type",
user_key="0E078F23-A5B4-4FB4-909B-60E49295C5E9",
title="Nøgleansvarlig")
os2mo.add_role(employee="Susanne Chæf",
organisation_unit="Magenta",
role_type_ref="Nøgleansvarlig",
date_from="1986-12-01")
os2mo.add_role(employee="Bolette Buhl",
organisation_unit="SJA2",
role_type_ref="Medarbejder repræsentant",
date_from="1986-12-01")
os2mo.add_role(employee="Jens Mortensen",
organisation_unit="Pilestræde",
role_type_ref="Medarbejder repræsentant",
date_from="1986-12-01")
# Create manager type, level and responsibilites
# and assign to employee
# Manager type
os2mo.add_klasse(identifier="Leder",
facet_type_ref="manager_type",
user_key="55BD7A09-86C3-4E15-AF5D-EAD20EB12F81",
title="Virksomhedens direktør")
# Manager level
os2mo.add_klasse(identifier="Højeste niveau",
facet_type_ref="manager_level",
user_key="6EAA7DA7-212D-4FD0-A068-BA3F932FDB10",
title="Højeste niveau")
# Add responsabilities
os2mo.add_klasse(identifier="Tage beslutninger",
facet_type_ref="responsibility",
user_key="A9ABDCCB-EC83-468F-AB7D-175B95E94956",
title="Tage beslutninger")
os2mo.add_klasse(identifier="Motivere medarbejdere",
facet_type_ref="responsibility",
user_key="DC475AF8-21C9-4112-94AE-E9FB13FE8D14",
title="Motivere medarbejdere")
os2mo.add_klasse(identifier="Betale løn",
facet_type_ref="responsibility",
user_key="0A929060-3392-4C07-8F4E-EF5F9B6AFDE2",
title="Betale løn")
os2mo.add_manager(
employee="Susanne Chæf",
organisation_unit="Magenta",
manager_type_ref="Leder",
manager_level_ref="Højeste niveau",
responsibility_list=[
"Tage beslutninger", "Motivere medarbejdere", "Betale løn"
],
date_from="1987-12-01",
)
# Leave of absence (Does not work after release 0.10)
# Leave type requires an exisiting engagement type
os2mo.add_klasse(identifier="Sygeorlov",
facet_type_ref="leave_type",
user_key="DB8E39C3-9160-47DB-A314-B0F8D1A2D536",
title="Sygeorlov")
os2mo.add_leave(employee="Jens Mortensen",
leave_type_ref="Sygeorlov",
date_from="2018-01-22",
date_to="2018-11-02")
# Create IT system and assign to employee
os2mo.new_itsystem(identifier="Servermiljø", system_name="Servermiljø")
os2mo.join_itsystem(employee="Jens Mortensen",
user_key="jmort",
itsystem_ref="Servermiljø",
date_from="1987-10-01")
os2mo.join_itsystem(employee="Bolette Buhl",
user_key="bolbu",
itsystem_ref="Servermiljø",
date_from="1987-10-01")
# Terminate an employee (only relevant for multiple imports)
# os2mo.terminate_employee(
# employee="Susanne Chæf",
# date_from="2019-06-01",
# )
os2mo.import_all()
# TODO: Soon we have done this 4 times. Should we make a small settings
# importer, that will also handle datatype for specicic keys?
cfg_file = pathlib.Path.cwd() / 'settings' / 'settings.json'
if not cfg_file.is_file():
raise Exception('No setting file')
settings = json.loads(cfg_file.read_text())
cpr_map = pathlib.Path.cwd() / 'settings' / 'cpr_uuid_map.csv'
if not cpr_map.is_file():
raise Exception('No mapping file')
employee_mapping = cpr_mapper.employee_mapper(str(cpr_map))
importer = ImportHelper(create_defaults=True,
mox_base=settings['mox.base'],
mora_base=settings['mora.base'],
store_integration_data=False,
seperate_names=True)
ad_reader = ad_reader.ADParameterReader()
detector = UniversalDetector()
manager_file = settings['integrations.SD_Lon.import.manager_file']
with open(manager_file, 'rb') as csvfile:
for row in csvfile:
detector.feed(row)
if detector.done:
break
detector.close()
encoding = detector.result['encoding']
def get_sd_importer(
municipality_name: str = "Andeby Kommune",
municipality_code: int = 100,
org_only: bool = False,
override_settings: Optional[Dict[str, Any]] = None,
) -> SdImportTest:
override_settings = override_settings or {}
mora_base = "http://mora/"
mox_base = "http://mox/"
importer = ImportHelper(
create_defaults=True,
mox_base=mox_base,
mora_base=mora_base,
store_integration_data=False,
seperate_names=True,
)
settings_dict = {
"municipality_name": municipality_name,
"municipality_code": municipality_code,
"sd_global_from_date": "1970-01-01",
"sd_employment_field": "extension_1",
"sd_import_run_db": "run_db.sqlite",
"sd_institution_identifier": "XY",
"sd_job_function": "EmploymentName",
"sd_monthly_hourly_divide": 9000,
"sd_password": "******",
"sd_user": "******",
"sd_importer_create_associations": False,
}
settings_dict.update(override_settings)
settings = ImporterSettings.parse_obj(settings_dict)
sd = SdImportTest(importer,
settings=settings,
org_only=org_only,
ad_info=None)
# add_people should not be called when rg-only
if org_only:
sd.add_people_mock.assert_not_called()
else:
sd.add_people_mock.assert_called_once()
sd.read_department_info_mock.assert_called_once()
assert sd.importer.organisation is not None
assert sd.importer.organisation[0] == municipality_name
assert sd.importer.organisation[1].uuid is None
assert sd.importer.organisation[1].name == municipality_name
assert sd.importer.organisation[1].user_key == municipality_name
assert sd.importer.organisation[1].municipality_code == str(
municipality_code)
assert sd.importer.organisation[1].date_from == "1930-01-01"
assert sd.importer.organisation[1].date_to == "infinity"
assert sd.importer.organisation[1].integration_data == {}
assert sd.importer.klassifikation is not None
assert sd.importer.klassifikation[0] == municipality_name
assert sd.importer.klassifikation[1].organisation_uuid is None
assert sd.importer.klassifikation[1].parent_name == municipality_name
assert sd.importer.klassifikation[1].user_key == municipality_name
assert sd.importer.klassifikation[1].description == "umbrella"
assert sd.importer.klassifikation[1].date_from == "1930-01-01"
assert sd.importer.klassifikation[1].date_to == "infinity"
assert sd.importer.klassifikation[1].integration_data == {}
# 29 classes exist hardcoded within sd_importer.py
assert len(sd.importer.klasse_objects) == 29
# 18 facets in os2mo_data_import/defaults.py
assert len(sd.importer.facet_objects) == 18
# No other objects
assert len(sd.importer.addresses) == 0
assert len(sd.importer.itsystems) == 0
assert len(sd.importer.organisation_units) == 0
assert len(sd.importer.organisation_unit_details) == 0
assert len(sd.importer.employees) == 0
assert len(sd.importer.employee_details) == 0
return sd
import os
import sys
import datetime
import pathlib
import json
from os2mo_data_import import ImportHelper
from integrations.SD_Lon import sd_importer
settings = json.loads(pathlib.Path("settings/settings.json").read_text())
importer = ImportHelper(create_defaults=True,
mox_base=settings["mox.base"],
mora_base=settings["mora.base"],
system_name='SD-Import',
end_marker='SDSTOP',
store_integration_data=True,
seperate_names=False)
sd = sd_importer.SdImport(
importer=importer,
ad_info=None,
org_only=True,
)
sd.create_ou_tree(create_orphan_container=True)
importer.import_all()
import os
import sys
from os2mo_data_import import ImportHelper
sys.path.append('..')
import opus_import
MUNICIPALTY_NAME = os.environ.get('MUNICIPALITY_NAME', 'Opus Import')
MOX_BASE = os.environ.get('MOX_BASE', 'http://localhost:8080')
MORA_BASE = os.environ.get('MORA_BASE', 'http://localhost:80')
XML_FILE_PATH = os.environ.get('XML_FILE_PATH', '')
importer = ImportHelper(
create_defaults=True,
mox_base=MOX_BASE,
mora_base=MORA_BASE,
system_name='Opus-Import',
end_marker='OPUS_STOP!',
store_integration_data=True
)
# importer.new_itsystem(
# identifier='AD',
# system_name='Active Directory'
# )
opus = opus_import.OpusImport(
importer,
MUNICIPALTY_NAME,
XML_FILE_PATH
)
def update_fixture(fixture_name, target):
'''Update a given fixture'''
with make_dummy_instance() as (psql, lora_server, mora_server):
from oio_rest import settings as lora_settings
from tests import util as test_util
threading.Thread(
target=lora_server.serve_forever,
args=(),
daemon=True,
).start()
if fixture_name == 'simple':
test_util.load_sample_structures()
elif fixture_name == 'minimal':
test_util.load_sample_structures(minimal=True)
else:
from os2mo_data_import import ImportHelper
from fixture_generator import populate_mo
from fixture_generator import dummy_data_creator
try:
org_size = getattr(dummy_data_creator.Size,
fixture_name.title())
except AttributeError:
raise click.exceptions.BadArgumentUsage(
'Unexpected fixture name {!r}'.format(fixture_name), )
threading.Thread(
target=mora_server.serve_forever,
args=(),
daemon=True,
).start()
importer = ImportHelper(
create_defaults=True,
mox_base=settings.LORA_URL,
mora_base='http://localhost:{}/'.format(mora_server.port),
system_name="Artificial import",
end_marker="STOP",
store_integration_data=True,
)
populate_mo.CreateDummyOrg(
importer,
'0860',
'Hjørring Kommune',
scale=1,
org_size=org_size,
extra_root=True,
)
importer.import_all()
mora_server.shutdown()
lora_server.shutdown()
if target is None:
target = open(os.path.join(test_util.FIXTURE_DIR,
fixture_name.lower() + '.sql'),
mode='w')
with target:
subprocess.check_call(
[
'pg_dump',
'--data-only',
'--dbname',
lora_settings.DATABASE,
'--host',
lora_settings.DB_HOST,
'--port',
str(lora_settings.DB_PORT),
'--username',
lora_settings.DB_USER,
],
stdout=target,
)
MUNICIPALTY_NAME = os.environ.get('MUNICIPALITY_NAME', 'APOS Import')
MOX_BASE = os.environ.get('MOX_BASE', 'http://localhost:5000')
MORA_BASE = os.environ.get('MORA_BASE', 'http://localhost:80')
MUNICIPALTY_CODE = os.environ.get('MUNICIPALITY_NAME', 0)
ean_file = '/opt/customer/dataimport/ballerup_udvalg/ean.csv'
ean_rows = {}
with open(ean_file) as csvfile:
reader = csv.reader(csvfile, delimiter=';')
for row in reader:
ean_rows[row[0]] = row[1]
importer = ImportHelper(create_defaults=True,
mox_base=MOX_BASE,
mora_base=MORA_BASE,
system_name='APOS-Import',
end_marker='APOSSTOP',
store_integration_data=False,
seperate_names=True)
apos_import = apos_importer.AposImport(
importer,
MUNICIPALTY_NAME,
MUNICIPALTY_CODE,
org_uuid='00000000-0000-43b4-8415-1c70c2cd0cbd',
ean=ean_rows)
apos_import.create_facetter_and_klasser()
# Org træ
apos_import.create_ou_tree('b78993bb-d67f-405f-acc0-27653bd8c116')
import os
from os2mo_data_import import ImportHelper
from integrations.SD_Lon import sd_importer
from os2mo_data_import.caching_import import CachingImportUtility
MOX_BASE = os.environ.get('MOX_BASE', 'http://localhost:8080')
MORA_BASE = os.environ.get('MORA_BASE', 'http://localhost:5000')
# Import of Administration
adm_name = 'AdmOrg'
importer = ImportHelper(
create_defaults=True,
mox_base=MOX_BASE,
mora_base=MORA_BASE,
seperate_names=True,
store_integration_data=False,
ImportUtility=CachingImportUtility,
)
# importer.add_klasse(
# identifier=adm_name,
# facet_type_ref='org_unit_type',
# user_key=adm_name,
# scope='TEXT',
# title=adm_name
# )
#
# importer.add_organisation_unit(
# identifier=adm_name,
# name=adm_name,
MOX_BASE = os.environ.get('MOX_BASE', 'http://localhost:8080')
MORA_BASE = os.environ.get('MORA_BASE', 'http://localhost:80')
MANAGER_FILE = os.environ.get('MANAGER_FILE', 'Organisationsdata.csv')
# ORIGIN FOR TESTS WIH ACTUAL API
# GLOBAL_GET_DATE = datetime.datetime(2006, 1, 1, 0, 0) # will not work
# GLOBAL_GET_DATE = datetime.datetime(2009, 1, 1, 0, 0)
# GLOBAL_GET_DATE = datetime.datetime(2011, 1, 1, 0, 0)
# GLOBAL_GET_DATE = datetime.datetime(2014, 2, 15, 0, 0)
GLOBAL_GET_DATE = datetime.datetime(2019, 2, 15, 0, 0)
# GLOBAL_GET_DATE = datetime.datetime(2019, 5, 10, 0, 0)
importer = ImportHelper(create_defaults=True,
mox_base=MOX_BASE,
mora_base=MORA_BASE,
system_name='SD-Import',
end_marker='SDSTOP',
store_integration_data=True,
seperate_names=True)
ad_usernames, ad_uuids = viborg_uuids.read_ad_and_uuids()
detector = UniversalDetector()
with open(MANAGER_FILE, 'rb') as csvfile:
for row in csvfile:
detector.feed(row)
if detector.done:
break
detector.close()
encoding = detector.result['encoding']
from os2mo_data_import import ImportHelper
from integrations.SD_Lon import sd_importer
from integrations.ad_integration import ad_reader
cfg_file = pathlib.Path.cwd() / 'settings' / 'settings.json'
if not cfg_file.is_file():
raise Exception('No setting file')
settings = json.loads(cfg_file.read_text())
cpr_map = pathlib.Path.cwd() / 'settings' / 'cpr_uuid_map.csv'
if not cpr_map.is_file():
raise Exception('No mapping file')
employee_mapping = cpr_mapper.employee_mapper(str(cpr_map))
importer = ImportHelper(create_defaults=True,
mox_base=settings['mox.base'],
mora_base=settings['mora.base'],
store_integration_data=False,
seperate_names=True)
ad_info_reader = ad_reader.ADParameterReader()
sd = sd_importer.SdImport(importer,
ad_info=ad_info_reader,
employee_mapping=employee_mapping)
sd.create_ou_tree(create_orphan_container=True)
sd.create_employees()
importer.import_all()
# TODO: Soon we have done this 4 times. Should we make a small settings
# importer, that will also handle datatype for specicic keys?
cfg_file = pathlib.Path.cwd() / 'settings' / 'settings.json'
if not cfg_file.is_file():
raise Exception('No setting file')
settings = json.loads(cfg_file.read_text())
cpr_map = pathlib.Path.cwd() / 'settings' / 'cpr_uuid_map.csv'
if not cpr_map.is_file():
raise Exception('No mapping file')
employee_mapping = cpr_mapper.employee_mapper(str(cpr_map))
importer = ImportHelper(create_defaults=True,
mox_base=settings['mox.base'],
mora_base=settings['mora.base'],
store_integration_data=False,
seperate_names=True)
ad_info_reader = ad_reader.ADParameterReader()
sd = sd_importer.SdImport(importer,
ad_info=ad_info_reader,
employee_mapping=employee_mapping)
importer.add_klasse(identifier='IT-Org. Alias',
uuid='df948904-7ad3-49b0-92a0-e35c50a4bccf',
facet_type_ref='org_unit_address_type',
user_key='IT-Org. Alias',
scope='TEXT',
title='IT-Org. Alias')
class IntegrationDataTests(unittest.TestCase):
@classmethod
def setUpClass(self):
self.morah = MoraHelper(use_cache=False)
def _clear_database(self):
conn = psycopg2.connect(user="******",
dbname="mox",
host="localhost",
password="******")
cursor = conn.cursor()
query = ("select relname from pg_class where relkind='r' " +
"and relname !~ '^(pg_|sql_)';")
cursor.execute(query)
for row in cursor.fetchall():
query = "truncate {} cascade;".format(row[0])
cursor.execute(query)
conn.commit()
def setUp(self):
self.importer = ImportHelper(
create_defaults=True,
mox_base=MOX_BASE,
mora_base=MORA_BASE,
system_name="SD-Import",
end_marker="SDSTOP",
store_integration_data=False,
)
@classmethod
def tearDownClass(self):
pass
def test_010_standard_import(self):
"""
self a standard SD import
"""
print("Test 010")
self._clear_database()
sd = sd_importer.SdImport(
self.importer,
MUNICIPALTY_NAME,
MUNICIPALTY_CODE,
import_date_from=GLOBAL_GET_DATE,
org_only=True,
org_id_prefix=None,
)
sd.create_ou_tree(create_orphan_container=True)
self.importer.import_all()
count = _count(MOX_BASE)
print(count)
self.assertTrue(count["unit_count"] in range(550, 560))
def test_011_test_kommunaldirektør(self):
print("Test 011")
kommunaldirektør = self.morah.read_ou(
"3e890083-589b-4a00-ba00-000001350001")
self.assertFalse("error_key" in kommunaldirektør)
def test_012_test_udgåede(self):
print("Test 012")
udgåede = self.morah.read_ou("8df2c488-8038-4b00-a600-000001460003")
self.assertFalse("error_key" in udgåede)
def test_013_test_orphan(self):
print("Test 013")
org = self.morah.read_organisation()
top_units = self.morah.read_top_units(org)
orphans = 0
for unit in top_units:
if unit["name"] == "Forældreløse enheder":
orphans += 1
self.assertTrue(orphans == 1)
# Forældreløse, Kommunalbestyrelsen, Kommnaldirektør, udgåede
self.assertTrue(len(top_units) == 4)
def test_020_standard_import(self):
"""
Test a standard SD import, withoug grouping of orphan units
"""
print("Test 020")
self._clear_database()
sd = sd_importer.SdImport(
self.importer,
MUNICIPALTY_NAME,
MUNICIPALTY_CODE,
import_date_from=GLOBAL_GET_DATE,
org_only=True,
org_id_prefix=None,
)
sd.create_ou_tree(create_orphan_container=False)
self.importer.import_all()
count = _count(MOX_BASE)
print(count)
self.assertTrue(count["unit_count"] in range(550, 555))
def test_021_test_orphan(self):
print("Test 021")
org = self.morah.read_organisation()
top_units = self.morah.read_top_units(org)
orphans = 0
for unit in top_units:
if unit["name"] == "Forældreløse enheder":
orphans += 1
self.assertTrue(orphans == 0)
self.assertTrue(len(top_units) > 6)
def test_030_import_subtree(self):
"""
Test an import of a single sub-tree.
"""
print("Test 030")
self._clear_database()
sd = sd_importer.SdImport(
self.importer,
MUNICIPALTY_NAME,
MUNICIPALTY_CODE,
import_date_from=GLOBAL_GET_DATE,
org_only=True,
org_id_prefix=None,
)
sd.create_ou_tree(
create_orphan_container=False,
# Direktør m.a. CAY,CBK,CTM, CEI
sub_tree="3e890083-589b-4a00-ba00-000001350001",
)
self.importer.import_all()
count = _count(MOX_BASE)
print(count)
self.assertTrue(count["unit_count"] in range(460, 465))
def test_031_test_roots(self):
print("Test 031")
org = self.morah.read_organisation()
top_units = self.morah.read_top_units(org)
self.assertTrue(len(top_units) == 1)
def test_040_import_subtree_with_super(self):
"""
Test an import of a single sub-tree with a super unit.
"""
print("Test 040")
self._clear_database()
sd = sd_importer.SdImport(
self.importer,
MUNICIPALTY_NAME,
MUNICIPALTY_CODE,
import_date_from=GLOBAL_GET_DATE,
org_only=True,
org_id_prefix=None,
)
adm_name = "AdmOrg"
self.importer.add_klasse(
identifier=adm_name,
facet_type_ref="org_unit_type",
user_key=adm_name,
scope="TEXT",
title=adm_name,
)
self.importer.add_organisation_unit(
identifier=adm_name,
name=adm_name,
user_key=adm_name,
type_ref=adm_name,
date_from="1930-01-01",
date_to=None,
parent_ref=None,
)
sd.create_ou_tree(
create_orphan_container=False,
super_unit=adm_name,
sub_tree="3e890083-589b-4a00-ba00-000001350001",
)
self.importer.import_all()
count = _count(MOX_BASE)
print(count)
self.assertTrue(count["unit_count"] in range(450, 470))
def test_041_test_top_units(self):
print("Test 041")
org = self.morah.read_organisation()
top_units = self.morah.read_top_units(org)
super_unit = 0
for unit in top_units:
if unit["name"] == "AdmOrg":
super_unit += 1
self.assertTrue(super_unit == 1)
self.assertTrue(len(top_units) == 1)
def test_050_import_subtree_with_prefix(self):
"""
Test an import of bvn-prefixed units.
"""
print("Test 050")
self._clear_database()
sd = sd_importer.SdImport(
self.importer,
MUNICIPALTY_NAME,
MUNICIPALTY_CODE,
import_date_from=GLOBAL_GET_DATE,
org_only=True,
org_id_prefix="adm_",
)
sd.create_ou_tree(
create_orphan_container=False,
# 'Direktionen encoded with org prefix 'adm_'
sub_tree="fff9e2a6-d670-b656-c719-994eeac03a74",
)
self.importer.import_all()
count = _count(MOX_BASE)
print(count)
self.assertTrue(count["unit_count"] in range(450, 470))
def test_051_test_prefix(self):
print("Test 510")
org = self.morah.read_organisation()
top_units = self.morah.read_top_units(org)
bvn = top_units[0]["user_key"]
self.assertTrue(bvn[0:4] == "adm_")
def test_060_full_double_import(self):
# Import of Administration
adm_name = "AdmOrg"
print("Test 060")
self._clear_database()
self.importer.add_klasse(
identifier=adm_name,
facet_type_ref="org_unit_type",
user_key=adm_name,
scope="TEXT",
title=adm_name,
)
self.importer.add_organisation_unit(
identifier=adm_name,
uuid="11111111-1111-1111-1111-111111111111",
name=adm_name,
user_key=adm_name,
type_ref=adm_name,
date_from="1930-01-01",
date_to=None,
parent_ref=None,
)
sd = sd_importer.SdImport(
self.importer,
MUNICIPALTY_NAME,
MUNICIPALTY_CODE,
import_date_from=GLOBAL_GET_DATE,
org_only=True,
org_id_prefix="adm_",
)
sd.create_ou_tree(
create_orphan_container=False,
# 'Direktionen encoded with org prefix 'adm_'
sub_tree="fff9e2a6-d670-b656-c719-994eeac03a74",
super_unit=adm_name,
)
# Import of Lønorganisation
løn_name = "LønOrg"
self.importer.add_klasse(
identifier=løn_name,
facet_type_ref="org_unit_type",
user_key=løn_name,
scope="TEXT",
title=løn_name,
)
self.importer.add_organisation_unit(
identifier=løn_name,
uuid="22222222-2222-2222-2222-222222222222",
name=løn_name,
user_key=løn_name,
type_ref=løn_name,
date_from="1930-01-01",
date_to=None,
parent_ref=None,
)
sd = sd_importer.SdImport(
self.importer,
MUNICIPALTY_NAME,
MUNICIPALTY_CODE,
import_date_from=GLOBAL_GET_DATE,
org_only=True,
org_id_prefix=None,
)
sd.create_ou_tree(create_orphan_container=False, super_unit=løn_name)
self.importer.import_all()
count = _count(MOX_BASE)
print(count)
self.assertTrue(count["unit_count"] in range(1000, 1050))
def test_061_test_top_units(self):
print("Test 061")
org = self.morah.read_organisation()
top_units = self.morah.read_top_units(org)
self.assertTrue(len(top_units) == 2)
def test_062_test_kommunaldirektør(self):
print("Test 062")
kommunaldirektør = self.morah.read_ou(
"3e890083-589b-4a00-ba00-000001350001")
self.assertFalse("error_key" in kommunaldirektør)
def test_063_test_udgåede(self):
print("Test 63")
udgåede = self.morah.read_ou("8df2c488-8038-4b00-a600-000001460003")
self.assertFalse("error_key" in udgåede)