def fixup_single_user(
mora_base: AnyHttpUrl,
person_uuid: UUID,
engagement_uuid: UUID,
dry_run: bool = False,
) -> Tuple[Dict[str, Any], Any]:
"""Fixup the end-date of a single engagement for a single user."""
helper = MoraHelper(hostname=mora_base, use_cache=False)
# Fetch all present engagements for the user
engagements: Iterator[Dict[str, Any]] = helper._mo_lookup(
person_uuid,
"e/{}/details/engagement",
validity="present",
only_primary=False,
use_cache=False,
calculate_primary=False,
)
# Find the engagement we are looking for in the list
engagements = filter(
lambda engagement: engagement["uuid"] == str(engagement_uuid),
engagements)
engagement: Dict[str, Any] = one(engagements)
# Construct data-part of our payload using current data.
uuid_keys = [
"engagement_type",
"job_function",
"org_unit",
"person",
"primary",
]
direct_keys = ["extension_" + str(i) for i in range(1, 11)] + [
"fraction",
"is_primary",
"user_key",
"uuid",
]
data: Dict[str, Any] = {}
data.update({key: {"uuid": engagement[key]["uuid"]} for key in uuid_keys})
data.update({key: engagement[key] for key in direct_keys})
data.update(
{"validity": {
"from": engagement["validity"]["from"],
"to": None,
}})
# Construct entire payload
payload: Dict[str, Any] = {
"type": "engagement",
"uuid": str(engagement_uuid),
"data": data,
"person": {
"uuid": str(person_uuid, )
},
}
if dry_run:
return payload, AttrDict({"status_code": 200, "text": "Dry-run"})
response = helper._mo_post("details/edit", payload)
return payload, response
class SnurreBasse:
def __init__(self, session):
self.session = session
self.helper = MoraHelper(hostname=settings["mora.base"],
use_cache=False)
def terminate(self, typ, uuid):
response = self.helper._mo_post(
"details/terminate",
{
"type": typ,
"uuid": uuid,
"validity": {
"to": "2020-10-01"
}
},
)
if response.status_code == 400:
assert response.text.find("raise to a new registration") > 0
else:
response.raise_for_status()
def run_it(self):
for i in (session.query(ItForbindelse.uuid).filter(
and_(ItForbindelse.bruger_uuid != None)).all()):
try:
print(i)
self.terminate("it", i[0])
except:
pass
def run_adresse(self):
for i in (session.query(Adresse.uuid).filter(
and_(Adresse.adressetype_scope == "E-mail",
Adresse.bruger_uuid != None)).all()):
try:
print(i)
self.terminate("address", i[0])
except:
pass
class MOPrimaryEngagementUpdater(object):
def __init__(self):
self.helper = MoraHelper(hostname=MORA_BASE, use_cache=False)
self.org_uuid = self.helper.read_organisation()
self.mo_person = None
# Currently primary is set first by engagement type (order given in
# settings) and secondly by job_id. self.primary is an ordered list of
# classes that can considered to be primary. self.primary_types is a dict
# with all classes in the primary facet.
self.eng_types_order = SETTINGS[
'integrations.opus.eng_types_primary_order']
self.primary_types, self.primary = self._find_primary_types()
def _find_primary_types(self):
"""
Read the engagement types from MO and match them up against the three
known types in the OPUS->MO import.
:param helper: An instance of mora-helpers.
:return: A dict matching up the engagement types with LoRa class uuids.
"""
# These constants are global in all OPUS municipalities (because they are
# created by the OPUS->MO importer.
PRIMARY = 'primary'
NON_PRIMARY = 'non-primary'
FIXED_PRIMARY = 'explicitly-primary'
logger.info('Read primary types')
primary_dict = {
'fixed_primary': None,
'primary': None,
'non_primary': None
}
primary_types = self.helper.read_classes_in_facet('primary_type')
for primary_type in primary_types[0]:
if primary_type['user_key'] == PRIMARY:
primary_dict['primary'] = primary_type['uuid']
if primary_type['user_key'] == NON_PRIMARY:
primary_dict['non_primary'] = primary_type['uuid']
if primary_type['user_key'] == FIXED_PRIMARY:
primary_dict['fixed_primary'] = primary_type['uuid']
if None in primary_dict.values():
raise Exception('Missing primary types: {}'.format(primary_dict))
primary_list = [primary_dict['fixed_primary'], primary_dict['primary']]
return primary_dict, primary_list
def set_current_person(self, cpr=None, uuid=None, mo_person=None):
"""
Set a new person as the current user. Either a cpr-number or
an uuid should be given, not both.
:param cpr: cpr number of the person.
:param uuid: MO uuid of the person.
:param mo_person: An already existing user object from mora_helper.
:return: True if current user is valid, otherwise False.
"""
if uuid:
mo_person = self.helper.read_user(user_uuid=uuid)
elif cpr:
mo_person = self.helper.read_user(user_cpr=cpr,
org_uuid=self.org_uuid)
elif mo_person:
pass
else:
mo_person = None
# print('Read user: {}s'.format(time.time() - t))
if mo_person:
self.mo_person = mo_person
success = True
else:
self.mo_person = None
success = False
return success
def _engagements_included_in_primary_calculation(self, engagements):
included = []
for eng in engagements:
# disregard engagements from externals
if eng["org_unit"]["uuid"] in SETTINGS.get(
"integrations.ad.import_ou.mo_unit_uuid", ""):
logger.warning(
'disregarding external engagement: {}'.format(eng))
continue
included.append(eng)
return included
def _calculate_rate_and_ids(self, mo_engagement):
min_type_pri = 9999
min_id = 9999999
for eng in mo_engagement:
logger.debug('Calculate rate, engagement: {}'.format(eng))
try:
employment_id = int(eng['user_key'])
except ValueError:
logger.warning(
"Skippning engangement with non-integer employment_id: {}".
format(eng['user_key']))
continue
stat = 'Current eng_type, min_id: {}, {}. This rate, eng_pos: {}, {}'
logger.debug(
stat.format(min_type_pri, min_id, employment_id,
eng['fraction']))
if eng['engagement_type'] in self.eng_types_order:
type_pri = self.eng_types_order.index(eng['engagement_type'])
else:
type_pri = 9999
if type_pri == min_type_pri:
if employment_id < min_id:
min_id = employment_id
if type_pri < min_type_pri:
min_id = employment_id
min_type_pri = type_pri
logger.debug('Min id: {}, Prioritied type: {}'.format(
min_id, min_type_pri))
return (min_id, min_type_pri)
def check_all_for_primary(self):
"""
Check all users for the existence of primary engagements.
:return: TODO
"""
count = 0
all_users = self.helper.read_all_users()
for user in all_users:
if count % 250 == 0:
print('{}/{}'.format(count, len(all_users)))
count += 1
self.set_current_person(uuid=user['uuid'])
date_list = self.helper.find_cut_dates(user['uuid'])
for i in range(0, len(date_list) - 1):
date = date_list[i]
mo_engagement = self.helper.read_user_engagement(
user=self.mo_person['uuid'], at=date, only_primary=True)
primary_count = 0
for eng in mo_engagement:
if eng['primary']['uuid'] in self.primary:
primary_count += 1
if primary_count == 0:
print('No primary for {} at {}'.format(user['uuid'], date))
elif primary_count > 1:
print('Too many primaries for {} at {}'.format(
user['uuid'], date))
else:
# print('Correct')
pass
def recalculate_primary(self, no_past=True):
"""
Re-calculate primary engagement for the entire history of the current user.
"""
logger.info('Calculate primary engagement: {}'.format(self.mo_person))
date_list = self.helper.find_cut_dates(self.mo_person['uuid'],
no_past=no_past)
number_of_edits = 0
for i in range(0, len(date_list) - 1):
date = date_list[i]
logger.info('Recalculate primary, date: {}'.format(date))
mo_engagement = self.helper.read_user_engagement(
user=self.mo_person['uuid'],
at=date,
only_primary=True,
use_cache=False)
mo_engagement = self._engagements_included_in_primary_calculation(
mo_engagement)
if len(mo_engagement) == 0:
continue
logger.debug('MO engagement: {}'.format(mo_engagement))
(min_id,
min_type_pri) = self._calculate_rate_and_ids(mo_engagement)
if (min_id is None) or (min_type_pri is None):
# continue
raise Exception('Cannot calculate primary')
fixed = None
for eng in mo_engagement:
if eng['primary']:
if eng['primary']['uuid'] == self.primary_types[
'fixed_primary']:
logger.info('Engagment {} is fixed primary'.format(
eng['uuid']))
fixed = eng['uuid']
exactly_one_primary = False
for eng in mo_engagement:
to = datetime.datetime.strftime(
date_list[i + 1] - datetime.timedelta(days=1), "%Y-%m-%d")
if date_list[i + 1] == datetime.datetime(9999, 12, 30, 0, 0):
to = None
validity = {
'from': datetime.datetime.strftime(date, "%Y-%m-%d"),
'to': to
}
try:
employment_id = int(eng['user_key'])
except ValueError:
logger.warning(
"Skippning engangement with non-integer employment_id: {}"
.format(eng['user_key']))
continue
if eng['engagement_type'] in self.eng_types_order:
type_pri = self.eng_types_order.index(
eng['engagement_type'])
else:
type_pri = 9999
msg = 'Current type pri and id: {}, {}'
logger.debug(msg.format(type_pri, employment_id))
if type_pri == min_type_pri and employment_id == min_id:
assert (exactly_one_primary is False)
logger.debug('Primary is: {}'.format(employment_id))
exactly_one_primary = True
current_type = self.primary_types['primary']
else:
logger.debug('{} is not primary'.format(employment_id))
current_type = self.primary_types['non_primary']
if fixed is not None and eng['uuid'] != fixed:
# A fixed primary exits, but this is not it.
logger.debug('Manual override, this is not primary!')
current_type = self.primary_types['non_primary']
if eng['uuid'] == fixed:
# This is a fixed primary.
current_type = self.primary_types['fixed_primary']
data = {
'primary': {
'uuid': current_type
},
'validity': validity
}
payload = payloads.edit_engagement(data, eng['uuid'])
if not payload['data']['primary'] == eng['primary']:
logger.debug('Edit payload: {}'.format(payload))
response = self.helper._mo_post('details/edit', payload)
assert response.status_code == 200
number_of_edits += 1
else:
logger.debug('No edit, primary type not changed.')
return_dict = {self.mo_person['uuid']: number_of_edits}
return return_dict
def recalculate_all(self, no_past=False):
"""
Recalculate all primary engagements
:return: TODO
"""
all_users = self.helper.read_all_users()
edit_status = {}
for user in all_users:
t = time.time()
self.set_current_person(uuid=user['uuid'])
status = self.recalculate_primary(no_past=no_past)
edit_status.update(status)
logger.debug(
'Time for primary calculation: {}'.format(time.time() - t))
print('Total edits: {}'.format(sum(edit_status.values())))
class ADMOImporter(object):
def __init__(self):
self.settings = load_settings()
self.root_ou_uuid = self.settings["integrations.ad.import_ou.mo_unit_uuid"]
self.helper = MoraHelper(hostname=self.settings["mora.base"], use_cache=False)
self.org_uuid = self.helper.read_organisation()
self.ad_reader = ADParameterReader()
self.ad_reader.cache_all(print_progress=True)
its = self.helper.read_it_systems()
AD_its = only(filter(lambda x: x["name"] == constants.AD_it_system, its))
self.AD_it_system_uuid = AD_its["uuid"]
def _find_or_create_unit_and_classes(self):
"""
Find uuids of the needed unit and classes for the import. If any unit or
class is missing from MO, it will be created.The function returns a dict
containg uuids needed to create users and engagements.
"""
# TODO: Add a dynamic creation of classes
job_type, _ = ensure_class_in_lora("engagement_job_function", "Ekstern")
eng_type, _ = ensure_class_in_lora("engagement_type", "Ekstern")
org_unit_type, _ = ensure_class_in_lora("org_unit_type", "Ekstern")
unit = self.helper.read_ou(uuid=self.root_ou_uuid)
if "status" in unit: # Unit does not exist
payload = payloads.create_unit(
self.root_ou_uuid, "Eksterne Medarbejdere", org_unit_type, self.org_uuid
)
logger.debug("Create department payload: {}".format(payload))
response = self.helper._mo_post("ou/create", payload)
assert response.status_code == 201
logger.info("Created unit for external employees")
logger.debug("Response: {}".format(response.text))
uuids = {
"job_function": job_type,
"engagement_type": eng_type,
"unit_uuid": self.root_ou_uuid,
}
return uuids
def _find_ou_users_in_ad(self) -> Dict[UUID, List]:
"""
find users from AD that match a search string in DistinguishedName.
"""
search_string = self.settings["integrations.ad.import_ou.search_string"]
def filter_users(user: Dict) -> bool:
name = user.get("DistinguishedName")
if name:
if search_string in name:
return True
return False
users = list(filter(filter_users, self.ad_reader.results.values()))
uuids = map(itemgetter("ObjectGUID"), users)
users_dict = dict(zip(uuids, users))
return users_dict
def _create_user(
self, ad_user: Dict, cpr_field: str, uuid: Optional[str] = None
) -> Optional[UUID]:
"""
Create or update a user in MO using an AD user as a template.
The user will share uuid between MO and AD.
:param ad_user: The ad_object to use as template for MO.
:return: uuid of the the user.
"""
cpr_raw = ad_user.get(cpr_field)
if cpr_raw is None:
return None
cpr = cpr_raw.replace("-", "")
payload = payloads.create_user(cpr, ad_user, self.org_uuid, uuid=uuid)
logger.info("Create user payload: {}".format(payload))
r = self.helper._mo_post("e/create", payload)
assert r.status_code == 201
user_uuid = UUID(r.json())
logger.info("Created employee {}".format(user_uuid))
return user_uuid
def _connect_user_to_ad(self, ad_user: Dict) -> None:
"""Write user AD username to the AD it system"""
logger.info("Connect user to AD: {}".format(ad_user["SamAccountName"]))
payload = payloads.connect_it_system_to_user(ad_user, self.AD_it_system_uuid)
logger.debug("AD account payload: {}".format(payload))
response = self.helper._mo_post("details/create", payload)
assert response.status_code == 201
logger.debug("Added AD account info to {}".format(ad_user["SamAccountName"]))
def _create_engagement(
self, ad_user: Dict, uuids: Dict[str, UUID], mo_uuid: UUID = None
) -> None:
"""Create the engagement in MO"""
# TODO: Check if we have start/end date of engagements in AD
validity = {"from": datetime.datetime.now().strftime("%Y-%m-%d"), "to": None}
person_uuid = ad_user["ObjectGUID"]
if mo_uuid:
person_uuid = mo_uuid
# TODO: Check if we can use job title from AD
payload = payloads.create_engagement(
ad_user=ad_user, validity=validity, person_uuid=person_uuid, **uuids
)
logger.info("Create engagement payload: {}".format(payload))
response = self.helper._mo_post("details/create", payload)
assert response.status_code == 201
logger.info("Added engagement to {}".format(ad_user["SamAccountName"]))
def cleanup_removed_users_from_mo(self) -> None:
"""Remove users in MO if they are no longer found as external users in AD."""
yesterday = datetime.datetime.now() - datetime.timedelta(days=1)
users = self._find_ou_users_in_ad()
mo_users = self.helper.read_organisation_people(self.root_ou_uuid)
for key, user in mo_users.items():
if key not in users:
# This users is in MO but not in AD:
payload = payloads.terminate_engagement(
user["Engagement UUID"], yesterday
)
logger.debug("Terminate payload: {}".format(payload))
response = self.helper._mo_post("details/terminate", payload)
logger.debug("Terminate response: {}".format(response.text))
response.raise_for_status()
def create_or_update_users_in_mo(self) -> None:
"""
Create users in MO that exist in AD but not in MO.
Update name of users that has changed name in AD.
"""
uuids = self._find_or_create_unit_and_classes()
users = self._find_ou_users_in_ad()
for AD in self.settings["integrations.ad"]:
cpr_field = AD["cpr_field"]
for user_uuid, ad_user in tqdm(
users.items(), unit="Users", desc="Updating units"
):
logger.info("Updating {}".format(ad_user["SamAccountName"]))
cpr = ad_user[cpr_field]
# Sometimes there is a temporary change of cpr in wich the
# last character is replaced with an 'x'.
# This user is ignored by the importer
# until the cpr has been changed back.
if cpr[-1].lower() == "x":
logger.info("Skipped due to 'x' in cpr.")
continue
mo_user = self.helper.read_user(user_cpr=cpr)
logger.info("Existing MO info: {}".format(mo_user))
if mo_user:
mo_uuid = mo_user.get("uuid")
else:
mo_uuid = self._create_user(ad_user, cpr_field)
AD_username = self.helper.get_e_itsystems(
mo_uuid, self.AD_it_system_uuid
)
if not AD_username:
self._connect_user_to_ad(ad_user)
current_engagements = self.helper.read_user_engagement(user=mo_uuid)
this_engagement = list(
filter(
lambda x: x.get("org_unit").get("uuid") == uuids["unit_uuid"],
current_engagements,
)
)
if not this_engagement:
self._create_engagement(ad_user, uuids, mo_uuid)
class ImportUtility(object):
"""
ImportUtility
TODO: This class relies heavily on asserts, which are gennerally not wanted in
this context. We should create meaningfull exceptions and raise these when
needed.
"""
def __init__(self, system_name, end_marker, mox_base, mora_base,
demand_consistent_uuids, store_integration_data=False,
dry_run=False):
# Import Params
self.demand_consistent_uuids = demand_consistent_uuids
self.store_integration_data = store_integration_data
if store_integration_data:
self.ia = IntegrationAbstraction(mox_base, system_name, end_marker)
# Service endpoint base
self.mox_base = mox_base
self.mora_base = mora_base
# Session
self.mh = MoraHelper(self.mora_base, use_cache=False)
self.session = Session()
# Placeholder for UUID import
self.organisation_uuid = None
# Existing UUIDS
# TODO: More elegant version of this please
self.existing_uuids = []
# UUID map
self.inserted_organisation = {}
self.inserted_facet_map = {}
self.inserted_klasse_map = {}
self.inserted_itsystem_map = {}
self.inserted_org_unit_map = {}
self.inserted_employee_map = {}
# Deprecated
self.dry_run = dry_run
def import_organisation(self, reference, organisation):
"""
Convert organisation to OIO formatted post data
and import into the MOX datastore.
:param str reference: Reference to the user defined identifier
:param object organisation: Organisation object
:returns: Inserted UUID
:rtype: str/uuid
"""
if not isinstance(organisation, Organisation):
raise TypeError("Not of type Organisation")
resource = "organisation/organisation"
integration_data = self._integration_data(
resource=resource,
reference=reference,
payload={}
)
organisation.integration_data = integration_data
payload = organisation.build()
if organisation.uuid is not None:
organisation_uuid = organisation.uuid
else:
organisation_uuid = integration_data.get('uuid', None)
self.organisation_uuid = self.insert_mox_data(
resource=resource,
data=payload,
uuid=organisation_uuid
)
# Global validity
self.date_from = organisation.date_from
self.date_to = organisation.date_to
return self.organisation_uuid
def import_klassifikation(self, reference, klassifikation):
"""
Begin import of klassifikation
:param str reference: Reference to the user defined identifier
:param object klassifikation: Klassifikation object
:returns: Inserted UUID
:rtype: str/uuid
"""
if not isinstance(klassifikation, Klassifikation):
raise TypeError("Not of type Klassifikation")
resource = "klassifikation/klassifikation"
klassifikation.organisation_uuid = self.organisation_uuid
integration_data = self._integration_data(
resource=resource,
reference=reference,
payload={}
)
klassifikation.integration_data = integration_data
payload = klassifikation.build()
klassifikation_uuid = integration_data.get('uuid', None)
self.klassifikation_uuid = self.insert_mox_data(
resource=resource,
data=payload,
uuid=klassifikation_uuid
)
return self.klassifikation_uuid
def import_facet(self, reference, facet):
"""
Begin import of facet
:param str reference: Reference to the user defined identifier
:param object facet: Facet object
:returns: Inserted UUID
:rtype: str/uuid
"""
if not isinstance(facet, Facet):
raise TypeError("Not of type Facet")
resource = "klassifikation/facet"
facet.organisation_uuid = self.organisation_uuid
facet.klassifikation_uuid = self.klassifikation_uuid
# NEED TO BE FIXED
facet.date_from = self.date_from
facet.date_to = self.date_to
integration_data = self._integration_data(
resource=resource,
reference=reference,
payload={}
)
facet.integration_data = integration_data
payload = facet.build()
facet_uuid = integration_data.get('uuid', None)
self.inserted_facet_map[reference] = self.insert_mox_data(
resource=resource,
data=payload,
uuid=facet_uuid
)
return self.inserted_facet_map[reference]
def import_klasse(self, reference, klasse):
"""
Insert a klasse object
Begin import of klassifikation
:param str reference: Reference to the user defined identifier
:param object organisation: Organisation object
:returns: Inserted UUID
:rtype: str/uuid
"""
if not isinstance(klasse, Klasse):
raise TypeError("Not of type Facet")
uuid = klasse.uuid
facet_ref = klasse.facet_type_ref
facet_uuid = self.inserted_facet_map.get(facet_ref)
if not facet_uuid:
error_message = "Facet ref: {} does not exist for {}".format(
facet_ref, klasse
)
logger.error(error_message)
raise KeyError(error_message)
resource = "klassifikation/klasse"
klasse.organisation_uuid = self.organisation_uuid
klasse.facet_uuid = facet_uuid
klasse.date_from = self.date_from
klasse.date_to = self.date_to
integration_data = self._integration_data(
resource=resource,
reference=reference,
payload={}
)
if 'uuid' in integration_data:
klasse_uuid = integration_data['uuid']
assert(uuid is None or klasse_uuid == uuid)
else:
if uuid is None:
klasse_uuid = None
else:
klasse_uuid = uuid
klasse.integration_data = integration_data
payload = klasse.build()
import_uuid = self.insert_mox_data(
resource="klassifikation/klasse",
data=payload,
uuid=klasse_uuid
)
assert(uuid is None or import_uuid == str(klasse_uuid))
self.inserted_klasse_map[reference] = import_uuid
return self.inserted_klasse_map[reference]
def import_itsystem(self, reference, itsystem):
"""
Create IT System
:param str reference: Reference to the user defined identifier
:param object organisation: Organisation object
:returns: Inserted UUID
:rtype: str/uuid
"""
if not isinstance(itsystem, Itsystem):
raise TypeError("Not of type Itsystem")
resource = 'organisation/itsystem'
itsystem.organisation_uuid = self.organisation_uuid
itsystem.date_from = self.date_from
itsystem.date_to = self.date_to
integration_data = self._integration_data(
resource=resource,
reference=reference,
payload={}
)
if 'uuid' in integration_data:
itsystem_uuid = integration_data['uuid']
else:
itsystem_uuid = None
itsystem.integration_data = integration_data
payload = itsystem.build()
self.inserted_itsystem_map[reference] = self.insert_mox_data(
resource=resource,
data=payload,
uuid=itsystem_uuid
)
return self.inserted_itsystem_map[reference]
def import_org_unit(self, reference, organisation_unit, details=[]):
"""
Insert organisation unit and details
.. note::
Optional data objects are relational objects which
belong to the organisation unit, such as an address type
:param str reference: Reference to the user defined identifier
:param object organisation_unit: Organisation object
:param list details: List of details
:returns: Inserted UUID
:rtype: str/uuid
"""
if not isinstance(organisation_unit, OrganisationUnitType):
raise TypeError("Not of type OrganisationUnitType")
if reference in self.inserted_org_unit_map:
return False
resource = 'organisation/organisationenhed'
# payload = self.build_mo_payload(organisation_unit_data)
parent_ref = organisation_unit.parent_ref
if parent_ref:
parent_uuid = self.inserted_org_unit_map.get(parent_ref)
organisation_unit.parent_uuid = parent_uuid
if not organisation_unit.parent_uuid:
organisation_unit.parent_uuid = self.organisation_uuid
type_ref_uuid = self.inserted_klasse_map.get(
organisation_unit.type_ref
)
if hasattr(organisation_unit, "time_planning_ref"):
organisation_unit.time_planning_ref_uuid = self.inserted_klasse_map.get(
organisation_unit.time_planning_ref
)
if hasattr(organisation_unit, 'org_unit_level_ref'):
organisation_unit.org_unit_level_uuid = self.inserted_klasse_map.get(
organisation_unit.org_unit_level_ref
)
organisation_unit.type_ref_uuid = type_ref_uuid
payload = organisation_unit.build()
payload = self._integration_data(
resource=resource,
reference=reference,
payload=payload,
encode_integration=False
)
if 'uuid' in payload:
if payload['uuid'] in self.existing_uuids:
logger.info('Re-import org-unit: {}'.format(payload['uuid']))
re_import = 'NO'
resource = 'details/edit'
payload_keys = list(payload.keys())
payload['data'] = {}
for key in payload_keys:
payload['data'][key] = payload[key]
del payload[key]
payload['type'] = 'org_unit'
else:
re_import = 'NEW'
logger.info('New unit - Forced uuid: {}'.format(payload['uuid']))
resource = 'ou/create'
else:
re_import = 'NEW'
logger.info('New unit, random uuid')
resource = 'ou/create'
logger.debug('Unit payload: {}'.format(payload))
uuid = self.insert_mora_data(
resource=resource,
data=payload
)
if 'uuid' in payload:
assert (uuid == payload['uuid'])
if not uuid:
raise ConnectionError("Something went wrong")
# Add to the inserted map
self.inserted_org_unit_map[reference] = uuid
data = {}
data['address'] = self.mh._get_detail(uuid, 'address', object_type='ou')
# Build details (if any)
details_payload = []
for detail in details:
detail.org_unit_uuid = uuid
date_from = detail.date_from
if not date_from:
date_from = organisation_unit.date_from
build_detail = self.build_detail(
detail=detail
)
if not build_detail:
continue
found_hit = self._payload_compare(build_detail, data)
logger.debug('Found hit for {}: {}'.format(uuid, found_hit))
if not found_hit and re_import == 'NO':
re_import = 'YES'
# TODO: SHOLD WE UPDATE FROM TO TODAY IN CASE OF RE-IMPORT?
if re_import == 'YES':
valid_from = datetime.now().strftime('%Y-%m-%d') # today
build_detail['validity']['from'] = valid_from
details_payload.append(build_detail)
if re_import == 'YES':
logger.info('Terminating details for unit {}'.format(uuid))
for item in data['address']:
self._terminate_details(item['uuid'], 'address')
if re_import in ('YES', 'NEW'):
logger.info('Re-import unit: {}'.format(re_import))
self.insert_mora_data(
resource="details/create",
data=details_payload
)
return uuid
def import_employee(self, reference, employee, details=[]):
"""
Import employee
:param str reference: Reference to the user defined identifier
:param object employee: Employee object
:param list details: List of details
:returns: Inserted UUID
:rtype: str/uuid
"""
if not isinstance(employee, EmployeeType):
raise TypeError("Not of type EmployeeType")
employee.org_uuid = self.organisation_uuid
payload = employee.build()
mox_resource = 'organisation/bruger'
integration_data = self._integration_data(
resource=mox_resource,
reference=reference,
payload=payload,
encode_integration=False
)
if 'uuid' in payload and payload['uuid'] in self.existing_uuids:
logger.info('Re-import employee {}'.format(payload['uuid']))
re_import = 'NO'
else:
re_import = 'NEW'
logger.info('New employee, uuid {}'.format(payload.get('uuid')))
# We unconditionally create or update the user, this should
# ensure that we are always updated with correct current information.
mora_resource = "e/create"
uuid = self.insert_mora_data(
resource=mora_resource,
data=integration_data
)
if 'uuid' in integration_data:
assert (uuid == integration_data['uuid'])
# Add uuid to the inserted employee map
self.inserted_employee_map[reference] = uuid
data = {}
data['it'] = self.mh._get_detail(uuid, 'it')
data['role'] = self.mh._get_detail(uuid, 'role')
data['leave'] = self.mh._get_detail(uuid, 'leave')
data['address'] = self.mh._get_detail(uuid, 'address')
data['manager'] = self.mh._get_detail(uuid, 'manager')
data['engagement'] = self.mh._get_detail(uuid, 'engagement')
data['association'] = self.mh._get_detail(uuid, 'association')
# In case of en explicit termination, we terminate the employee or
# employment and return imidiately.
for detail in details:
if isinstance(detail, TerminationType):
logger.info(
'Explicit termination of eng {} from {}'.format(
uuid, detail.date_from
)
)
self._terminate_employee(uuid, date_from=detail.date_from)
return uuid
if isinstance(detail, EngagementTerminationType):
logger.info(
'Explicit termination of eng {}'.format(
detail.engagement_uuid
)
)
self._terminate_details(detail.engagement_uuid, 'engagement')
return uuid
if details:
additional_payload = []
for detail in details:
if not detail.date_from:
detail.date_from = self.date_from
# Create payload (as dict)
detail_payload = self.build_detail(
detail=detail,
employee_uuid=uuid
)
if not detail_payload:
continue
# If we do not have existing data, the new data should be imported
if len(data[detail_payload['type']]) == 0 and re_import == 'NO':
re_import = 'UPDATE'
elif data[detail_payload['type']]:
found_hit = self._payload_compare(detail_payload, data)
if not found_hit:
re_import = 'YES'
additional_payload.append(detail_payload)
for item in additional_payload:
valid_from = item['validity']['from']
valid_to = item['validity']['to']
now = datetime.now()
py_from = datetime.strptime(valid_from, '%Y-%m-%d')
if valid_to is not None:
py_to = datetime.strptime(valid_to, '%Y-%m-%d')
else:
py_to = datetime.strptime('2200-01-01', '%Y-%m-%d')
logger.debug(
'Py-from:{}, Py-to:{}, Now:{}'.format(py_from, py_to, now)
)
if re_import == 'YES' and py_from < now and py_to > now:
logger.debug('Updating valid_from')
valid_from = datetime.now().strftime('%Y-%m-%d') # today
item['validity']['from'] = valid_from
logger.info('Re-import: {}'.format(re_import))
if re_import == 'YES':
logger.info('Non-explicit termination: {}'.format(uuid))
self._terminate_employee(uuid)
if re_import in ('YES', 'NEW', 'UPDATE'):
self.insert_mora_data(
resource="details/create",
data=additional_payload
)
return uuid
def build_detail(self, detail, employee_uuid=None):
"""
Build detail payload
:param MoType detail: Detail object
.. note::
A detail can be one of the following types:
- address
- asso
- role
- itsystem
- engagement
- manager
:Reference: :mod:`os2mo_data_import.mora_data_types`
:param str employee_uuid: (Option) Employee uuid if it exists
:return: Detail POST data payload
:rtype: dict
"""
if employee_uuid:
detail.person_uuid = employee_uuid
common_attributes = [
("type_ref", "type_ref_uuid"),
("job_function_ref", "job_function_uuid"),
("address_type_ref", "address_type_uuid"),
("manager_level_ref", "manager_level_uuid")
]
for check_value, set_value in common_attributes:
if not hasattr(detail, check_value):
continue
uuid = self.inserted_klasse_map.get(
getattr(detail, check_value)
)
if not uuid:
if self.store_integration_data:
klasse_res = 'klassifikation/klasse'
uuid = self.ia.find_object(klasse_res, getattr(detail, check_value))
else:
# print('Detail: {}, check_value: {}'.format(detail, check_value))
pass
setattr(detail, set_value, uuid)
# Uncommon attributes
if hasattr(detail, "visibility_ref"):
detail.visibility_ref_uuid = self.inserted_klasse_map.get(
detail.visibility_ref
)
if hasattr(detail, 'primary_ref'):
detail.primary_uuid = self.inserted_klasse_map.get(
detail.primary_ref
)
if hasattr(detail, "org_unit_ref"):
detail.org_unit_uuid = self.inserted_org_unit_map.get(
detail.org_unit_ref
)
if hasattr(detail, "organisation_uuid"):
detail.organisation_uuid = self.organisation_uuid
if hasattr(detail, "itsystem_ref"):
detail.itsystem_uuid = self.inserted_itsystem_map.get(
detail.itsystem_ref
)
if hasattr(detail, "responsibilities"):
detail.responsibilities = [
self.inserted_klasse_map[reference]
for reference in detail.responsibility_list
]
return detail.build()
def _integration_data(self, resource, reference, payload={},
encode_integration=True):
"""
Update the payload with integration data. Checks if an object with this
integration data already exists. In this case the uuid of the exisiting
object is put into the payload. If a supplied uuid is inconsistent with the
uuid found from integration data, an exception is raised.
:param resource:
LoRa resource URL.
:param referece:
Unique label that will be stored in the integration data to identify the
object on re-import.
:param payload:
The supplied payload will be updated with values for integration and uuid (if
the integration data was found from an earlier import). For MO objects,
payload will typically be pre-populated and will then be ready for import
when returned. For MOX objects, the initial payload will typically be empty,
and the returned values can be fed to the relevant adapter.
:param encode_integration:
If True, the integration data will be returned in json-encoded form.
:return:
The original payload updated with integration data and object uuid, if the
object was already imported.
"""
# TODO: We need to have a list of all objects with integration data to
# be able to make a list of objects that has disappeared
if self.store_integration_data:
uuid = self.ia.find_object(resource, reference)
if uuid:
if 'uuid' in payload:
if self.demand_consistent_uuids:
assert(payload['uuid'] == uuid)
else:
payload['uuid'] = uuid
payload['uuid'] = uuid
self.existing_uuids.append(uuid)
payload['integration_data'] = self.ia.integration_data_payload(
resource,
reference,
uuid,
encode_integration
)
return payload
def insert_mox_data(self, resource, data, uuid=None):
service_url = urljoin(
base=self.mox_base,
url=resource
)
if uuid:
update_url = "{service}/{uuid}".format(
service=service_url,
uuid=uuid
)
response = self.session.put(
url=update_url,
json=data
)
if response.status_code != 200:
logger.error(
'Mox put. Response: {}, data'.format(response.text, data)
)
raise HTTPError("Inserting mox data failed")
else:
response = self.session.post(
url=service_url,
json=data
)
if response.status_code != 201:
logger.error(
'Mox post. Response: {}, data'.format(response.text, data)
)
raise HTTPError("Inserting mox data failed")
response_data = response.json()
return response_data["uuid"]
def insert_mora_data(self, resource, data, uuid=None):
# TESTING
if self.dry_run:
uuid = uuid4()
return str(uuid)
response = self.mh._mo_post(
url=resource,
payload=data,
force=True,
)
if response.status_code == 400:
error = response.json()['description']
if error.find('does not give raise to a new registration') > 0:
uuid_start = error.find('with id [')
uuid = error[uuid_start+9:uuid_start+45]
try:
UUID(uuid, version=4)
except ValueError:
raise Exception('Unable to read uuid')
else:
logger.error(
'MO post. Error: {}, data: {}'.format(error, data)
)
raise HTTPError("Inserting mora data failed")
elif response.status_code not in (200, 201):
logger.error(
'MO post. Response: {}, data: {}'.format(response.text, data)
)
raise HTTPError("Inserting mora data failed")
else:
uuid = response.json()
return uuid
def _terminate_employee(self, uuid, date_from=None):
endpoint = 'e/{}/terminate'
yesterday = datetime.now() - timedelta(days=1)
if date_from:
to = date_from
else:
to = yesterday.strftime('%Y-%m-%d')
payload = {
'terminate_all': True,
'validity': {
'to': to
}
}
resource = endpoint.format(uuid)
self.insert_mora_data(
resource=resource,
data=payload
)
return uuid
def _terminate_details(self, uuid, detail_type):
logger.info('Terminate detail {}: {}'.format(uuid, detail_type))
yesterday = datetime.now() - timedelta(days=1)
payload = {
'type': detail_type,
'uuid': uuid,
'validity': {
'to': yesterday.strftime('%Y-%m-%d')
}
}
logger.debug('Terminate detail payload: {}'.format(payload))
try:
self.insert_mora_data(
resource='details/terminate',
data=payload
)
except HTTPError:
logger.info('Tried to terminate a non-existing uuid')
return uuid
def _std_compare(self, item_payload, data_item, extra_field=None):
""" Helper for _payload_compare, performs the checks that are identical
for most object types.
:param item_payload: The new payload data.
:param data_item: The existing set of data.
:param extra_field: If not None the comparison will also be done on this
field, otherwise the comparison is only performed on uuid and validity.
:return: True if identical, otherwise False
"""
identical = (
(data_item['org_unit']['uuid'] == item_payload['org_unit']['uuid']) and
(data_item['validity']['from'] == item_payload['validity']['from']) and
(data_item['validity']['to'] == item_payload['validity']['to'])
)
if extra_field is not None:
identical = (
identical and
data_item[extra_field]['uuid'] == item_payload[extra_field]['uuid']
)
return identical
def _payload_compare(self, item_payload, data):
""" Compare an exising data-set with a new payload and tell whether
the new payload is different from the exiting data.
:param item_payload: New the payload data.
:param data_item: The existing set of data.
:param extra_field: If not None the comparison will also be done on this
field, otherwise the comparison is only performed on uuid and validity.
:return: True if identical, otherwise False
"""
data_type = item_payload['type']
logger.debug(
'Payload compare. item_payload: {}, data: {}'.format(item_payload, data)
)
found_hit = False
if data_type == 'engagement':
# In priciple, we should be able to re-calculate the hash stored in
# integration data and compare directly from that.
for data_item in data[data_type]:
if (self._std_compare(item_payload, data_item, 'job_function') and
item_payload['user_key'] == data_item['user_key'] and
item_payload['fraction'] == data_item['fraction']):
found_hit = True
elif data_type == 'role':
for data_item in data[data_type]:
if self._std_compare(item_payload, data_item, 'role_type'):
found_hit = True
logger.debug('Found hit in role')
elif data_type == 'leave':
for data_item in data[data_type]:
if ((data_item['validity']['from'] ==
item_payload['validity']['from']) and
(data_item['validity']['to'] ==
item_payload['validity']['to'])):
found_hit = True
logger.debug('Found hit in leave')
elif data_type == 'it':
for data_item in data[data_type]:
if (
(data_item['validity']['from'] ==
item_payload['validity']['from']) and
(data_item['validity']['to'] ==
item_payload['validity']['to']) and
(data_item['itsystem']['uuid'] ==
item_payload['itsystem']['uuid'])
):
found_hit = True
logger.debug('Found hit in it')
elif data_type == 'address':
for data_item in data[data_type]:
if (
(data_item['validity']['from'] ==
item_payload['validity']['from']) and
(data_item['validity']['to'] ==
item_payload['validity']['to']) and
(data_item['value'] == item_payload['value'])
):
found_hit = True
logger.debug('Found hit in adress')
elif data_type == 'manager':
for data_item in data[data_type]:
identical = self._std_compare(item_payload, data_item,
'manager_level')
uuids = []
for item in item_payload['responsibility']:
uuids.append(item['uuid'])
for responsibility in data_item['responsibility']:
identical = identical and (responsibility['uuid'] in uuids)
identical = (identical and
(len(data_item['responsibility']) == len(uuids)))
if identical:
found_hit = True
logger.debug('Found hit in manager')
elif data_type == 'association':
for data_item in data[data_type]:
if self._std_compare(item_payload, data_item, 'association_type'):
found_hit = True
logger.debug('Found hit in association')
else:
raise Exception('Uknown detail!')
logger.info('Found hit: {}'.format(found_hit))
return found_hit
class SnurreBasse:
def _find_classes(self, facet):
class_types = self.helper.read_classes_in_facet(facet)
types_dict = {}
facet = class_types[1]
for class_type in class_types[0]:
types_dict[class_type["user_key"]] = class_type["uuid"]
return types_dict, facet
def __init__(self, session):
self.session = session
self.top_per_unit = {}
self.helper = MoraHelper(hostname=settings["mora.base"],
use_cache=False)
self.unit_types, self.unit_type_facet = self._find_classes(
"org_unit_type")
self.unit_levels, self.unit_level_facet = self._find_classes(
"org_unit_level")
def get_top_unit(self, lc_enhed):
"""
return the top unit for a unit
"""
top_unit = self.top_per_unit.get(lc_enhed.uuid)
if top_unit:
return top_unit
branch = [lc_enhed.uuid]
# walk as far up as necessary
while lc_enhed.forældreenhed_uuid is not None:
uuid = lc_enhed.forældreenhed_uuid
top_unit = self.top_per_unit.get(uuid)
if top_unit:
break
branch.append(uuid)
lc_enhed = self.session.query(Enhed).filter(
Enhed.uuid == uuid).one()
top_unit = uuid # last one effective
# register top unit for all encountered
for buuid in branch:
self.top_per_unit[buuid] = top_unit
return top_unit
def level_from_type(self, outype_bvn):
return self.unit_levels[outype_bvn]
def update_unit(self, lc_enhed):
mo_unit = self.helper.read_ou(lc_enhed.uuid)
if (mo_unit["org_unit_level"] is not None
and mo_unit["org_unit_level"]["user_key"] in self.unit_levels):
logger.debug("already done: %s", lc_enhed.uuid)
return
payload = {
"type": "org_unit",
"data": {
"uuid": lc_enhed.uuid,
"org_unit_level": {
"uuid":
self.level_from_type(mo_unit["org_unit_type"]["user_key"]),
},
"org_unit_type": {
"uuid":
settings[
"snurrebasser.lcdb_traverse_units.org_unit_type_uuid"],
},
"validity": {
"from": mo_unit["validity"]["from"],
"to": None
},
},
}
logger.info("Edit unit: {}".format(payload))
response = self.helper._mo_post("details/edit", payload)
if (response.status_code == 400
and response.text.find("raise to a new registration") > 0):
pass
else:
response.raise_for_status()
def run(self):
def is_relevant(session, lc_enhed):
top_unit = self.get_top_unit(lc_enhed)
return (top_unit ==
settings["snurrebasser.lcdb_traverse_units.top_unit_uuid"])
relevant = partial(is_relevant, session)
alle_enheder = session.query(Enhed)
for i in filter(relevant, alle_enheder):
self.update_unit(i)
