def __init__(self, system_name, end_marker, mox_base, mora_base,
demand_consistent_uuids, store_integration_data=False,
dry_run=False):
# Import Params
self.demand_consistent_uuids = demand_consistent_uuids
self.store_integration_data = store_integration_data
if store_integration_data:
self.ia = IntegrationAbstraction(mox_base, system_name, end_marker)
# Service endpoint base
self.mox_base = mox_base
self.mora_base = mora_base
# Session
self.mh = MoraHelper(self.mora_base, use_cache=False)
self.session = Session()
# Placeholder for UUID import
self.organisation_uuid = None
# Existing UUIDS
# TODO: More elegant version of this please
self.existing_uuids = []
# UUID map
self.inserted_organisation = {}
self.inserted_facet_map = {}
self.inserted_klasse_map = {}
self.inserted_itsystem_map = {}
self.inserted_org_unit_map = {}
self.inserted_employee_map = {}
# Deprecated
self.dry_run = dry_run
def get_manager(org_unit_uuid, mh: MoraHelper):
present = mh._mo_lookup(org_unit_uuid,
"ou/{}/details/manager?validity=present")
future = mh._mo_lookup(org_unit_uuid,
"ou/{}/details/manager?validity=future")
managers = present + future
if not managers:
return None
if len(managers) > 1:
logger.warning("More than one manager exists for {}".format(
org_unit_uuid))
manager = managers[0]
person = manager.get("person")
if not person:
return None
ad_guid, sam_account_name = get_employee_from_map(
person["uuid"], mapping_file_path)
# Only import users who are in AD
if not ad_guid or not sam_account_name:
return {}
return {"uuid": person["uuid"], "userId": sam_account_name}
def cli(mox_base, mora_base, delete, full_history, opus_id, use_ad, dry_run):
"""Reimport object from opus with given opus-ID to MO
Optionally deletes the object and all related orgfuncs directly from Lora.
Defaults to reading latest file only, but supports reading full history
"""
helper = MoraHelper(hostname=mora_base)
object_type, obj = find_type(opus_id, full_history)
if object_type == "bruger":
cpr = opus_helpers.read_cpr(obj)
user = helper.read_user(user_cpr=cpr)
uuid = user["uuid"] if user else None
else:
uuid = opus_helpers.generate_uuid(obj["@id"])
if delete and uuid and not dry_run:
delete_object_and_orgfuncs(uuid, mox_base, object_type)
if dry_run:
click.echo(
f"Dry-run: {'Delete and reimport' if delete else 'Reimport'} '{object_type}' with {uuid=}"
)
else:
AD = ad_reader.ADParameterReader() if use_ad else None
import_opus(
ad_reader=AD,
import_all=full_history,
import_last=not full_history,
opus_id=opus_id,
rundb_write=False,
)
def cli(mox_base: str, mora_base: str, dry_run: bool):
helper = MoraHelper(hostname=mora_base)
org_unit_types, _ = helper.read_classes_in_facet("org_unit_type")
org_unit_types = sorted(org_unit_types, key=lambda x: x["name"])
groups = groupby(org_unit_types, key=lambda x: x["name"])
split_classes = map(split, groups)
split_classes = filter(is_duplicate, split_classes)
if dry_run:
click.echo(
f"Dry-run: Found {len(list(split_classes))} duplicated classes to fix."
)
return
session = requests.session()
for no_scope, scope in tqdm(split_classes, desc="Moving relations to one class"):
old_uuid = no_scope["uuid"]
move_class_helper(
old_uuid=old_uuid,
new_uuid=scope["uuid"],
copy=False,
mox_base=mox_base,
relation_type="organisation/organisationenhed",
)
delete_class(session=session, base=mox_base, uuid=old_uuid)
def full_import(org_only: bool, mora_base: str, mox_base: str):
"""Tool to do an initial full import."""
# Check connection to MO before we fire requests against SD
mh = MoraHelper(mora_base)
if not mh.check_connection():
raise click.ClickException("No MO reply, aborting.")
importer = ImportHelper(
create_defaults=True,
mox_base=mox_base,
mora_base=mora_base,
store_integration_data=False,
seperate_names=True,
)
sd = SdImport(importer,
settings=get_importer_settings(),
org_only=org_only,
ad_info=None)
sd.create_ou_tree(create_orphan_container=False,
sub_tree=None,
super_unit=None)
if not org_only:
sd.create_employees()
importer.import_all()
print("IMPORT DONE")
def fixup_single_user(
mora_base: AnyHttpUrl,
person_uuid: UUID,
engagement_uuid: UUID,
dry_run: bool = False,
) -> Tuple[Dict[str, Any], Any]:
"""Fixup the end-date of a single engagement for a single user."""
helper = MoraHelper(hostname=mora_base, use_cache=False)
# Fetch all present engagements for the user
engagements: Iterator[Dict[str, Any]] = helper._mo_lookup(
person_uuid,
"e/{}/details/engagement",
validity="present",
only_primary=False,
use_cache=False,
calculate_primary=False,
)
# Find the engagement we are looking for in the list
engagements = filter(
lambda engagement: engagement["uuid"] == str(engagement_uuid),
engagements)
engagement: Dict[str, Any] = one(engagements)
# Construct data-part of our payload using current data.
uuid_keys = [
"engagement_type",
"job_function",
"org_unit",
"person",
"primary",
]
direct_keys = ["extension_" + str(i) for i in range(1, 11)] + [
"fraction",
"is_primary",
"user_key",
"uuid",
]
data: Dict[str, Any] = {}
data.update({key: {"uuid": engagement[key]["uuid"]} for key in uuid_keys})
data.update({key: engagement[key] for key in direct_keys})
data.update(
{"validity": {
"from": engagement["validity"]["from"],
"to": None,
}})
# Construct entire payload
payload: Dict[str, Any] = {
"type": "engagement",
"uuid": str(engagement_uuid),
"data": data,
"person": {
"uuid": str(person_uuid, )
},
}
if dry_run:
return payload, AttrDict({"status_code": 200, "text": "Dry-run"})
response = helper._mo_post("details/edit", payload)
return payload, response
def __init__(self, enddate_field, uuid_field, settings=None):
super().__init__(all_settings=settings)
self.helper = MoraHelper(
hostname=self.all_settings["global"]["mora.base"], use_cache=False
)
self.enddate_field = enddate_field
self.uuid_field = uuid_field
self.cpr_field = self.all_settings["primary"]["cpr_field"]
def main(mora_base: str, use_ad: bool, output_file_path: str) -> None:
mh = MoraHelper(hostname=mora_base, export_ansi=True)
employees: List[ExportUser] = create_mapping(mh, use_ad)
employee_dicts: List[Dict] = list(map(methodcaller("dict"), employees))
fields = ["cpr", "mo_uuid", "ad_guid", "sam_account_name"]
mh._write_csv(fields, employee_dicts, output_file_path)
def __init__(self, session):
self.session = session
self.top_per_unit = {}
self.helper = MoraHelper(hostname=settings["mora.base"],
use_cache=False)
self.unit_types, self.unit_type_facet = self._find_classes(
"org_unit_type")
self.unit_levels, self.unit_level_facet = self._find_classes(
"org_unit_level")
def setUp(self):
super().setUp()
util.amqp.publish_message = lambda a, b, c, d, e: None
self.mh = MoraHelper()
if not getattr(requests, "_orgget", False):
requests._orgget = requests.get
requests.get = self.get
self._test_data_result = str(testdata /
(pathlib.Path(__file__).stem + "_" +
self._testMethodName + "_result.json"))
def _read_classes(self):
"""Read engagement_types and job_function types from MO."""
mora_base = self.settings.mora_base
helper = MoraHelper(hostname=mora_base, use_cache=False)
self.engagement_types = helper.read_classes_in_facet("engagement_type")
if self.update_job_functions:
self.job_function_types = helper.read_classes_in_facet(
"engagement_job_function"
)
def __init__(self,
system_name="Import",
end_marker="_|-STOP",
mox_base="http://localhost:8080",
mora_base="http://localhost:5000",
store_integration_data=False,
create_defaults=True,
seperate_names=False,
demand_consistent_uuids=True,
ImportUtility=ImportUtility):
self.seperate_names = seperate_names
mora_type_config(mox_base=mox_base,
system_name=system_name,
end_marker=end_marker)
self.mox_base = mox_base
# Import Utility
self.store = ImportUtility(
mox_base=mox_base,
mora_base=mora_base,
system_name=system_name,
end_marker=end_marker,
demand_consistent_uuids=demand_consistent_uuids,
store_integration_data=store_integration_data)
# TODO: store_integration_data could be passed to ImportUtility by passing
# the actual self.ia object
if store_integration_data:
self.morah = MoraHelper(use_cache=False)
self.ia = IntegrationAbstraction(mox_base, system_name, end_marker)
self.organisation = None
self.klassifikation = None
self.klasse_objects = {}
self.facet_objects = {}
self.addresses = []
self.itsystems = {}
self.organisation_units = {}
self.employees = {}
# Compatibility map
self.available_types = {
"klasse": "klasse_objects",
"facet": "facet_objects",
"organisation_unit": "organisation_units",
"employee": "employees"
}
self.organisation_unit_details = {}
self.employee_details = {}
# Create default facet and klasse
if create_defaults:
self.create_default_facet_types()
def find_bad_engagements(
mora_base: AnyHttpUrl) -> Iterator[Tuple[UUID, List[UUID]]]:
"""Find users with engagements that ends after 9999-01-01."""
def enrich_user_uuid(user_uuid: UUID) -> Tuple[UUID, List[UUID]]:
"""Enrich each user_uuid with engagements that end after 9999-01-01."""
# Fetch all engagements for the user
mo_engagements: List[Dict] = helper.read_user_engagement(
user=str(user_uuid),
only_primary=False,
read_all=True,
skip_past=True)
# Extract uuid and end-date, filter out infinity end-dates.
mo_engagement_tuples_str: Iterator[Tuple[UUID, str]] = map(
lambda mo_engagement: (
mo_engagement["uuid"],
mo_engagement["validity"]["to"],
),
mo_engagements,
)
mo_engagement_tuples_str = filter(
apply(lambda mo_uuid, end_date: end_date is not None),
mo_engagement_tuples_str,
)
# Convert end-date to datetime.date, and filter out dates before 9999-01-01
mo_engagement_tuples: Iterator[Tuple[UUID, date]] = map(
apply(lambda mo_uuid, end_date: (
mo_uuid,
datetime.strptime(end_date, "%Y-%m-%d").date(),
)),
mo_engagement_tuples_str,
)
mo_engagement_tuples = filter(
apply(lambda mo_uuid, end_date: end_date >= date(9999, 1, 1)),
mo_engagement_tuples,
)
# Extract and convert resulting engagement uuids
mo_engagement_uuid_strings: Iterator[str] = map(
itemgetter(0), mo_engagement_tuples)
mo_engagement_uuids: Iterator[UUID] = map(UUID,
mo_engagement_uuid_strings)
return user_uuid, list(mo_engagement_uuids)
helper = MoraHelper(hostname=mora_base, use_cache=False)
# Read all users and map to just their UUIDs
users: Iterator[Dict] = tqdm(helper.read_all_users())
user_uuid_strings: Iterator[str] = map(itemgetter("uuid"), users)
user_uuids: Iterator[UUID] = map(UUID, user_uuid_strings)
# Enrich each user_uuid with a list of uuids from engagements that has a bad end-date, filter empty lists
user_tuples: Iterator[Tuple[UUID,
List[UUID]]] = map(enrich_user_uuid,
user_uuids)
user_tuples = filter(
apply(lambda user_uuid, engagement_uuids: bool(engagement_uuids)),
user_tuples)
return user_tuples
def __init__(self):
self.helper = MoraHelper(hostname=MORA_BASE, use_cache=False)
self.org_uuid = self.helper.read_organisation()
self.mo_person = None
# Currently primary is set first by engagement type (order given in
# settings) and secondly by job_id. self.primary is an ordered list of
# classes that can considered to be primary. self.primary_types is a dict
# with all classes in the primary facet.
self.eng_types_order = SETTINGS[
'integrations.opus.eng_types_primary_order']
self.primary_types, self.primary = self._find_primary_types()
def __init__(self):
self.settings = load_settings()
self.root_ou_uuid = self.settings["integrations.ad.import_ou.mo_unit_uuid"]
self.helper = MoraHelper(hostname=self.settings["mora.base"], use_cache=False)
self.org_uuid = self.helper.read_organisation()
self.ad_reader = ADParameterReader()
self.ad_reader.cache_all(print_progress=True)
its = self.helper.read_it_systems()
AD_its = only(filter(lambda x: x["name"] == constants.AD_it_system, its))
self.AD_it_system_uuid = AD_its["uuid"]
def _find_manager(self, org_unit_uuid, mora_helper: MoraHelper):
url = "ou/{}/details/manager"
managers = mora_helper._mo_lookup(org_unit_uuid, url)
responsibility_class = self.settings[
"exporters.viborg.primary_manager_responsibility"]
for manager in managers:
if responsibility_class in map(lambda x: x.get("uuid"),
manager["responsibility"]):
return manager
parent = mora_helper.read_ou(org_unit_uuid).get("parent")
if not parent:
return {}
return self._find_manager(parent["uuid"], mora_helper)
def setUpClass(self):
self.morah = MoraHelper()
org = self.morah.read_organisation()
# This assumes a single top-unit. Tests will fail if we have more.
roots = self.morah.read_top_units(org)
self.nodes = self.morah.read_ou_tree(roots[0]['uuid'])
self.counts = self.morah._mo_lookup(org, 'o/{}/')
cq.export_orgs(self.morah, self.nodes, 'all_employees.csv')
cq.export_orgs(self.morah,
self.nodes,
'all_orgs.csv',
include_employees=False)
cq.export_managers(self.morah, self.nodes, 'all_managers.csv')
cq.export_adm_org(self.morah, self.nodes, 'adm_org.csv')
def get_employee_positions(employee_uuid, mh: MoraHelper):
present = mh._mo_lookup(
employee_uuid, "e/{}/details/engagement?validity=present")
future = mh._mo_lookup(employee_uuid,
"e/{}/details/engagement?validity=future")
engagements = present + future
converted_positions = []
for engagement in engagements:
converted_positions.append({
"name":
engagement["job_function"]["name"],
"orgUnitUuid":
engagement["org_unit"]["uuid"],
})
return converted_positions
def main(
emus_xml_file,
root_org_unit_uuid=MORA_ROOT_ORG_UNIT_UUID,
mh=MoraHelper(),
t=time.time(),
):
if not root_org_unit_uuid:
logger.error("root_org_unit_uuid must be specified")
exit(1)
logger.warning("caching all ou's,"
" so program may seem unresponsive temporarily")
nodes = mh.read_ou_tree(root_org_unit_uuid)
# Write the xml file
emus_xml_file.write('<?xml version="1.0" encoding="utf-8"?>\n')
emus_xml_file.write("<OS2MO>\n")
# had to switch the sequence - write e to tmp before append after ou
temp_file = io.StringIO()
export_e_emus(mh, nodes, temp_file)
export_ou_emus(mh, nodes, emus_xml_file)
emus_xml_file.write(temp_file.getvalue())
emus_xml_file.write("</OS2MO>")
def main(read_from_cache):
# fortiden er termineret i forvejen
# ad_sync kan ikke lave fremtidige, derfor ingen historik i cache
mh = MoraHelper(hostname=SETTINGS["mora.base"])
lc = LoraCache(resolve_dar=False, full_history=False)
lc.populate_cache(dry_run=read_from_cache, skip_associations=True)
kill_addresses(mh, lc)
kill_it_connections(mh, lc)
def main(**args):
user_file_path = args['user_file_path']
mapping_file_path = "cpr_mo_ad_map.csv"
settings = load_settings()
mora_helper = MoraHelper(settings['mora.base'])
comparison = UserComparison(settings, mora_helper, mapping_file_path,
user_file_path)
comparison.run()
def get_employee_email(employee_uuid, mh: MoraHelper):
present = mh._mo_lookup(employee_uuid,
"e/{}/details/address?validity=present")
future = mh._mo_lookup(employee_uuid,
"e/{}/details/address?validity=future")
addresses = present + future
emails = list(
filter(
lambda address: address["address_type"]["scope"] ==
"EMAIL",
addresses,
))
if emails:
if len(emails) > 1:
logger.warning(
"More than one email exists for user {}".format(
employee_uuid))
return emails[0]["value"]
return None
def __init__(self, lc=None, lc_historic=None, **kwargs):
super().__init__(**kwargs)
self.settings = self.all_settings
self.skip_occupied_names = kwargs.get("skip_occupied_names", False)
# Setup datasource for getting MO data.
# TODO: Create a factory instead of this hackery?
# Default to using MORESTSource as data source
self.datasource = MORESTSource(self.settings)
# Use LoraCacheSource if LoraCache is provided
if lc and lc_historic:
self.datasource = LoraCacheSource(lc, lc_historic, self.datasource)
# NOTE: These should be eliminated when all uses are gone
# NOTE: Once fully utilized, tests should be able to just implement a
# MODataSource for all their mocking needs.
self.lc = lc
self.lc_historic = lc_historic
self.helper = MoraHelper(hostname=self.settings["global"]["mora.base"],
use_cache=False)
self._init_name_creator()
def get_kle(org_unit_uuid: str,
mh: MoraHelper) -> Tuple[List[str], List[str]]:
present = mh._mo_lookup(org_unit_uuid,
"ou/{}/details/kle?validity=present")
future = mh._mo_lookup(org_unit_uuid,
"ou/{}/details/kle?validity=future")
kles = present + future
def get_kle_tuples(
kles: List[dict], ) -> Generator[Tuple[str, str], None, None]:
for kle in kles:
number = kle["kle_number"]["user_key"]
for aspect in kle["kle_aspect"]:
yield number, aspect["scope"]
kle_tuples = get_kle_tuples(kles)
buckets = bucket(kle_tuples, key=itemgetter(1))
interest = map(itemgetter(0), buckets["INDSIGT"])
performing = map(itemgetter(0), buckets["UDFOERENDE"])
return list(interest), list(performing)
def ensure_class_in_facet(mora_base, bvn, facet, title, uuid, scope):
"""Creates a class if it doesn't allready exist
Example:
metacli ensure_class_exists --bvn=Orlov --facet=leave_type
Returns the uuid of the created/existing class.
"""
helper = MoraHelper(hostname=mora_base, use_cache=False)
title = title or bvn
assert all(arg != ""
for arg in (bvn, title, facet, scope)), "Inputs can't be empty"
class_uuid = helper.ensure_class_in_facet(facet=facet,
bvn=bvn,
title=title,
uuid=uuid,
scope=scope)
if uuid:
assert (
class_uuid == uuid
), f"This class allready existed with another uuid {class_uuid}"
click.echo(class_uuid)
def export_engagement(self, mh: MoraHelper, filename, lc, lc_historic):
rows = []
logger.info("Reading users")
if lc:
employees = list(map(lambda x: x[0], lc.users.values()))
else:
employees = mh.read_all_users()
logger.info("Reading engagements")
# Todo: This O(#employees x #engagments), a pre-sorting of engagements would
# make it O(#employees + #engagments) - consider if this is worth the effort
for employee in employees:
logger.info("employee: %r", employee)
if lc:
for row in self._gen_from_loracache(employee, lc, lc_historic):
rows.append(row)
else:
for row in self._gen_from_mo(employee, mh):
rows.append(row)
mh._write_csv(self.fieldnames, rows, filename)
def main(
report_outfile,
root_org_unit_name=MORA_ROOT_ORG_UNIT_NAME,
mh=MoraHelper(),
):
root_org_unit_uuid = get_root_org_unit_uuid(mh, root_org_unit_name)
if not root_org_unit_uuid:
logger.error("%s not found in root-ous", root_org_unit_name)
exit(1)
logger.warning("caching all ou's,"
" so program may seem unresponsive temporarily")
nodes = mh.read_ou_tree(root_org_unit_uuid)
find_people(mh, nodes)
fieldnames, rows = prepare_report(mh, nodes)
rows = collapse_same_manager_more_departments(rows)
mh._write_csv(fieldnames, rows, report_outfile)
class SnurreBasse:
def __init__(self, session):
self.session = session
self.helper = MoraHelper(hostname=settings["mora.base"],
use_cache=False)
def terminate(self, typ, uuid):
response = self.helper._mo_post(
"details/terminate",
{
"type": typ,
"uuid": uuid,
"validity": {
"to": "2020-10-01"
}
},
)
if response.status_code == 400:
assert response.text.find("raise to a new registration") > 0
else:
response.raise_for_status()
def run_it(self):
for i in (session.query(ItForbindelse.uuid).filter(
and_(ItForbindelse.bruger_uuid != None)).all()):
try:
print(i)
self.terminate("it", i[0])
except:
pass
def run_adresse(self):
for i in (session.query(Adresse.uuid).filter(
and_(Adresse.adressetype_scope == "E-mail",
Adresse.bruger_uuid != None)).all()):
try:
print(i)
self.terminate("address", i[0])
except:
pass
def run(self, speedup=False, dry_run=True):
mora_base = self.settings["mora.base"]
query_exports_dir = pathlib.Path(
self.settings["mora.folder.query_export"])
if "exports_viborg_eksterne.outfile_basename" not in self.settings:
print(
"Missing key in settings: exports_viborg_eksterne.outfile_basename"
)
exit(1)
outfile_name = (
query_exports_dir /
self.settings["exports_viborg_eksterne.outfile_basename"])
logger.info("writing to file %s", outfile_name)
t = time.time()
mh = MoraHelper(hostname=mora_base, export_ansi=False)
if speedup:
# Here we should activate read-only mode, actual state and
# full history dumps needs to be in sync.
# Full history does not calculate derived data, we must
# fetch both kinds.
lc = LoraCache(resolve_dar=True, full_history=False)
lc.populate_cache(dry_run=dry_run, skip_associations=True)
lc.calculate_derived_unit_data()
lc_historic = LoraCache(resolve_dar=False,
full_history=True,
skip_past=True)
lc_historic.populate_cache(dry_run=dry_run, skip_associations=True)
# Here we should de-activate read-only mode
else:
lc = None
lc_historic = None
self.export_engagement(mh, str(outfile_name), lc, lc_historic)
logger.info("Time: {}s".format(time.time() - t))
logger.info("Export completed")
def export_from_mo(root, threaded_speedup, hostname):
threaded_speedup = threaded_speedup
t = time.time()
mh = MoraHelper(hostname=hostname, export_ansi=False)
org = mh.read_organisation()
roots = mh.read_top_units(org)
if root is None:
trees = {}
max_height = 0
for root in roots:
name = root['name']
uuid = root['uuid']
trees[name] = mh.read_ou_tree(uuid)
if trees[name]['root'].height > max_height:
main_root = name
nodes = trees[main_root]
else:
nodes = mh.read_ou_tree(root)
print('Find main tree: {}'.format(time.time() - t))
if threaded_speedup:
cq.pre_cache_users(mh)
print('Build cache: {}'.format(time.time() - t))
filename = 'alle_lederfunktioner_os2mo.csv'
cq.export_managers(mh, nodes, filename)
print('Alle ledere: {}s'.format(time.time() - t))
filename = 'alle-medarbejdere-stilling-email_os2mo.csv'
cq.export_all_employees(mh, nodes, filename)
print('alle-medarbejdere-stilling-email_os2mo.csv: {}s'.format(time.time() - t))
filename = 'org_incl-medarbejdere.csv'
cq.export_orgs(mh, nodes, filename)
print('org_incl-medarbejdere.csv: {}s'.format(time.time() - t))
filename = 'adm-org-incl-start-og-stopdata-og-enhedstyper-os2mo.csv'
cq.export_adm_org(mh, nodes, filename)
print('adm-org-incl-start-stop: {}s'.format(time.time() - t))
filename = 'tilknytninger.csv'
cq.export_all_teams(mh, nodes, filename)
print('tilknytninger: {}s'.format(time.time() - t))