def fixup_single_user( mora_base: AnyHttpUrl, person_uuid: UUID, engagement_uuid: UUID, dry_run: bool = False, ) -> Tuple[Dict[str, Any], Any]: """Fixup the end-date of a single engagement for a single user.""" helper = MoraHelper(hostname=mora_base, use_cache=False) # Fetch all present engagements for the user engagements: Iterator[Dict[str, Any]] = helper._mo_lookup( person_uuid, "e/{}/details/engagement", validity="present", only_primary=False, use_cache=False, calculate_primary=False, ) # Find the engagement we are looking for in the list engagements = filter( lambda engagement: engagement["uuid"] == str(engagement_uuid), engagements) engagement: Dict[str, Any] = one(engagements) # Construct data-part of our payload using current data. uuid_keys = [ "engagement_type", "job_function", "org_unit", "person", "primary", ] direct_keys = ["extension_" + str(i) for i in range(1, 11)] + [ "fraction", "is_primary", "user_key", "uuid", ] data: Dict[str, Any] = {} data.update({key: {"uuid": engagement[key]["uuid"]} for key in uuid_keys}) data.update({key: engagement[key] for key in direct_keys}) data.update( {"validity": { "from": engagement["validity"]["from"], "to": None, }}) # Construct entire payload payload: Dict[str, Any] = { "type": "engagement", "uuid": str(engagement_uuid), "data": data, "person": { "uuid": str(person_uuid, ) }, } if dry_run: return payload, AttrDict({"status_code": 200, "text": "Dry-run"}) response = helper._mo_post("details/edit", payload) return payload, response
class SnurreBasse: def __init__(self, session): self.session = session self.helper = MoraHelper(hostname=settings["mora.base"], use_cache=False) def terminate(self, typ, uuid): response = self.helper._mo_post( "details/terminate", { "type": typ, "uuid": uuid, "validity": { "to": "2020-10-01" } }, ) if response.status_code == 400: assert response.text.find("raise to a new registration") > 0 else: response.raise_for_status() def run_it(self): for i in (session.query(ItForbindelse.uuid).filter( and_(ItForbindelse.bruger_uuid != None)).all()): try: print(i) self.terminate("it", i[0]) except: pass def run_adresse(self): for i in (session.query(Adresse.uuid).filter( and_(Adresse.adressetype_scope == "E-mail", Adresse.bruger_uuid != None)).all()): try: print(i) self.terminate("address", i[0]) except: pass
class MOPrimaryEngagementUpdater(object): def __init__(self): self.helper = MoraHelper(hostname=MORA_BASE, use_cache=False) self.org_uuid = self.helper.read_organisation() self.mo_person = None # Currently primary is set first by engagement type (order given in # settings) and secondly by job_id. self.primary is an ordered list of # classes that can considered to be primary. self.primary_types is a dict # with all classes in the primary facet. self.eng_types_order = SETTINGS[ 'integrations.opus.eng_types_primary_order'] self.primary_types, self.primary = self._find_primary_types() def _find_primary_types(self): """ Read the engagement types from MO and match them up against the three known types in the OPUS->MO import. :param helper: An instance of mora-helpers. :return: A dict matching up the engagement types with LoRa class uuids. """ # These constants are global in all OPUS municipalities (because they are # created by the OPUS->MO importer. PRIMARY = 'primary' NON_PRIMARY = 'non-primary' FIXED_PRIMARY = 'explicitly-primary' logger.info('Read primary types') primary_dict = { 'fixed_primary': None, 'primary': None, 'non_primary': None } primary_types = self.helper.read_classes_in_facet('primary_type') for primary_type in primary_types[0]: if primary_type['user_key'] == PRIMARY: primary_dict['primary'] = primary_type['uuid'] if primary_type['user_key'] == NON_PRIMARY: primary_dict['non_primary'] = primary_type['uuid'] if primary_type['user_key'] == FIXED_PRIMARY: primary_dict['fixed_primary'] = primary_type['uuid'] if None in primary_dict.values(): raise Exception('Missing primary types: {}'.format(primary_dict)) primary_list = [primary_dict['fixed_primary'], primary_dict['primary']] return primary_dict, primary_list def set_current_person(self, cpr=None, uuid=None, mo_person=None): """ Set a new person as the current user. Either a cpr-number or an uuid should be given, not both. :param cpr: cpr number of the person. :param uuid: MO uuid of the person. :param mo_person: An already existing user object from mora_helper. :return: True if current user is valid, otherwise False. """ if uuid: mo_person = self.helper.read_user(user_uuid=uuid) elif cpr: mo_person = self.helper.read_user(user_cpr=cpr, org_uuid=self.org_uuid) elif mo_person: pass else: mo_person = None # print('Read user: {}s'.format(time.time() - t)) if mo_person: self.mo_person = mo_person success = True else: self.mo_person = None success = False return success def _engagements_included_in_primary_calculation(self, engagements): included = [] for eng in engagements: # disregard engagements from externals if eng["org_unit"]["uuid"] in SETTINGS.get( "integrations.ad.import_ou.mo_unit_uuid", ""): logger.warning( 'disregarding external engagement: {}'.format(eng)) continue included.append(eng) return included def _calculate_rate_and_ids(self, mo_engagement): min_type_pri = 9999 min_id = 9999999 for eng in mo_engagement: logger.debug('Calculate rate, engagement: {}'.format(eng)) try: employment_id = int(eng['user_key']) except ValueError: logger.warning( "Skippning engangement with non-integer employment_id: {}". format(eng['user_key'])) continue stat = 'Current eng_type, min_id: {}, {}. This rate, eng_pos: {}, {}' logger.debug( stat.format(min_type_pri, min_id, employment_id, eng['fraction'])) if eng['engagement_type'] in self.eng_types_order: type_pri = self.eng_types_order.index(eng['engagement_type']) else: type_pri = 9999 if type_pri == min_type_pri: if employment_id < min_id: min_id = employment_id if type_pri < min_type_pri: min_id = employment_id min_type_pri = type_pri logger.debug('Min id: {}, Prioritied type: {}'.format( min_id, min_type_pri)) return (min_id, min_type_pri) def check_all_for_primary(self): """ Check all users for the existence of primary engagements. :return: TODO """ count = 0 all_users = self.helper.read_all_users() for user in all_users: if count % 250 == 0: print('{}/{}'.format(count, len(all_users))) count += 1 self.set_current_person(uuid=user['uuid']) date_list = self.helper.find_cut_dates(user['uuid']) for i in range(0, len(date_list) - 1): date = date_list[i] mo_engagement = self.helper.read_user_engagement( user=self.mo_person['uuid'], at=date, only_primary=True) primary_count = 0 for eng in mo_engagement: if eng['primary']['uuid'] in self.primary: primary_count += 1 if primary_count == 0: print('No primary for {} at {}'.format(user['uuid'], date)) elif primary_count > 1: print('Too many primaries for {} at {}'.format( user['uuid'], date)) else: # print('Correct') pass def recalculate_primary(self, no_past=True): """ Re-calculate primary engagement for the entire history of the current user. """ logger.info('Calculate primary engagement: {}'.format(self.mo_person)) date_list = self.helper.find_cut_dates(self.mo_person['uuid'], no_past=no_past) number_of_edits = 0 for i in range(0, len(date_list) - 1): date = date_list[i] logger.info('Recalculate primary, date: {}'.format(date)) mo_engagement = self.helper.read_user_engagement( user=self.mo_person['uuid'], at=date, only_primary=True, use_cache=False) mo_engagement = self._engagements_included_in_primary_calculation( mo_engagement) if len(mo_engagement) == 0: continue logger.debug('MO engagement: {}'.format(mo_engagement)) (min_id, min_type_pri) = self._calculate_rate_and_ids(mo_engagement) if (min_id is None) or (min_type_pri is None): # continue raise Exception('Cannot calculate primary') fixed = None for eng in mo_engagement: if eng['primary']: if eng['primary']['uuid'] == self.primary_types[ 'fixed_primary']: logger.info('Engagment {} is fixed primary'.format( eng['uuid'])) fixed = eng['uuid'] exactly_one_primary = False for eng in mo_engagement: to = datetime.datetime.strftime( date_list[i + 1] - datetime.timedelta(days=1), "%Y-%m-%d") if date_list[i + 1] == datetime.datetime(9999, 12, 30, 0, 0): to = None validity = { 'from': datetime.datetime.strftime(date, "%Y-%m-%d"), 'to': to } try: employment_id = int(eng['user_key']) except ValueError: logger.warning( "Skippning engangement with non-integer employment_id: {}" .format(eng['user_key'])) continue if eng['engagement_type'] in self.eng_types_order: type_pri = self.eng_types_order.index( eng['engagement_type']) else: type_pri = 9999 msg = 'Current type pri and id: {}, {}' logger.debug(msg.format(type_pri, employment_id)) if type_pri == min_type_pri and employment_id == min_id: assert (exactly_one_primary is False) logger.debug('Primary is: {}'.format(employment_id)) exactly_one_primary = True current_type = self.primary_types['primary'] else: logger.debug('{} is not primary'.format(employment_id)) current_type = self.primary_types['non_primary'] if fixed is not None and eng['uuid'] != fixed: # A fixed primary exits, but this is not it. logger.debug('Manual override, this is not primary!') current_type = self.primary_types['non_primary'] if eng['uuid'] == fixed: # This is a fixed primary. current_type = self.primary_types['fixed_primary'] data = { 'primary': { 'uuid': current_type }, 'validity': validity } payload = payloads.edit_engagement(data, eng['uuid']) if not payload['data']['primary'] == eng['primary']: logger.debug('Edit payload: {}'.format(payload)) response = self.helper._mo_post('details/edit', payload) assert response.status_code == 200 number_of_edits += 1 else: logger.debug('No edit, primary type not changed.') return_dict = {self.mo_person['uuid']: number_of_edits} return return_dict def recalculate_all(self, no_past=False): """ Recalculate all primary engagements :return: TODO """ all_users = self.helper.read_all_users() edit_status = {} for user in all_users: t = time.time() self.set_current_person(uuid=user['uuid']) status = self.recalculate_primary(no_past=no_past) edit_status.update(status) logger.debug( 'Time for primary calculation: {}'.format(time.time() - t)) print('Total edits: {}'.format(sum(edit_status.values())))
class ADMOImporter(object): def __init__(self): self.settings = load_settings() self.root_ou_uuid = self.settings["integrations.ad.import_ou.mo_unit_uuid"] self.helper = MoraHelper(hostname=self.settings["mora.base"], use_cache=False) self.org_uuid = self.helper.read_organisation() self.ad_reader = ADParameterReader() self.ad_reader.cache_all(print_progress=True) its = self.helper.read_it_systems() AD_its = only(filter(lambda x: x["name"] == constants.AD_it_system, its)) self.AD_it_system_uuid = AD_its["uuid"] def _find_or_create_unit_and_classes(self): """ Find uuids of the needed unit and classes for the import. If any unit or class is missing from MO, it will be created.The function returns a dict containg uuids needed to create users and engagements. """ # TODO: Add a dynamic creation of classes job_type, _ = ensure_class_in_lora("engagement_job_function", "Ekstern") eng_type, _ = ensure_class_in_lora("engagement_type", "Ekstern") org_unit_type, _ = ensure_class_in_lora("org_unit_type", "Ekstern") unit = self.helper.read_ou(uuid=self.root_ou_uuid) if "status" in unit: # Unit does not exist payload = payloads.create_unit( self.root_ou_uuid, "Eksterne Medarbejdere", org_unit_type, self.org_uuid ) logger.debug("Create department payload: {}".format(payload)) response = self.helper._mo_post("ou/create", payload) assert response.status_code == 201 logger.info("Created unit for external employees") logger.debug("Response: {}".format(response.text)) uuids = { "job_function": job_type, "engagement_type": eng_type, "unit_uuid": self.root_ou_uuid, } return uuids def _find_ou_users_in_ad(self) -> Dict[UUID, List]: """ find users from AD that match a search string in DistinguishedName. """ search_string = self.settings["integrations.ad.import_ou.search_string"] def filter_users(user: Dict) -> bool: name = user.get("DistinguishedName") if name: if search_string in name: return True return False users = list(filter(filter_users, self.ad_reader.results.values())) uuids = map(itemgetter("ObjectGUID"), users) users_dict = dict(zip(uuids, users)) return users_dict def _create_user( self, ad_user: Dict, cpr_field: str, uuid: Optional[str] = None ) -> Optional[UUID]: """ Create or update a user in MO using an AD user as a template. The user will share uuid between MO and AD. :param ad_user: The ad_object to use as template for MO. :return: uuid of the the user. """ cpr_raw = ad_user.get(cpr_field) if cpr_raw is None: return None cpr = cpr_raw.replace("-", "") payload = payloads.create_user(cpr, ad_user, self.org_uuid, uuid=uuid) logger.info("Create user payload: {}".format(payload)) r = self.helper._mo_post("e/create", payload) assert r.status_code == 201 user_uuid = UUID(r.json()) logger.info("Created employee {}".format(user_uuid)) return user_uuid def _connect_user_to_ad(self, ad_user: Dict) -> None: """Write user AD username to the AD it system""" logger.info("Connect user to AD: {}".format(ad_user["SamAccountName"])) payload = payloads.connect_it_system_to_user(ad_user, self.AD_it_system_uuid) logger.debug("AD account payload: {}".format(payload)) response = self.helper._mo_post("details/create", payload) assert response.status_code == 201 logger.debug("Added AD account info to {}".format(ad_user["SamAccountName"])) def _create_engagement( self, ad_user: Dict, uuids: Dict[str, UUID], mo_uuid: UUID = None ) -> None: """Create the engagement in MO""" # TODO: Check if we have start/end date of engagements in AD validity = {"from": datetime.datetime.now().strftime("%Y-%m-%d"), "to": None} person_uuid = ad_user["ObjectGUID"] if mo_uuid: person_uuid = mo_uuid # TODO: Check if we can use job title from AD payload = payloads.create_engagement( ad_user=ad_user, validity=validity, person_uuid=person_uuid, **uuids ) logger.info("Create engagement payload: {}".format(payload)) response = self.helper._mo_post("details/create", payload) assert response.status_code == 201 logger.info("Added engagement to {}".format(ad_user["SamAccountName"])) def cleanup_removed_users_from_mo(self) -> None: """Remove users in MO if they are no longer found as external users in AD.""" yesterday = datetime.datetime.now() - datetime.timedelta(days=1) users = self._find_ou_users_in_ad() mo_users = self.helper.read_organisation_people(self.root_ou_uuid) for key, user in mo_users.items(): if key not in users: # This users is in MO but not in AD: payload = payloads.terminate_engagement( user["Engagement UUID"], yesterday ) logger.debug("Terminate payload: {}".format(payload)) response = self.helper._mo_post("details/terminate", payload) logger.debug("Terminate response: {}".format(response.text)) response.raise_for_status() def create_or_update_users_in_mo(self) -> None: """ Create users in MO that exist in AD but not in MO. Update name of users that has changed name in AD. """ uuids = self._find_or_create_unit_and_classes() users = self._find_ou_users_in_ad() for AD in self.settings["integrations.ad"]: cpr_field = AD["cpr_field"] for user_uuid, ad_user in tqdm( users.items(), unit="Users", desc="Updating units" ): logger.info("Updating {}".format(ad_user["SamAccountName"])) cpr = ad_user[cpr_field] # Sometimes there is a temporary change of cpr in wich the # last character is replaced with an 'x'. # This user is ignored by the importer # until the cpr has been changed back. if cpr[-1].lower() == "x": logger.info("Skipped due to 'x' in cpr.") continue mo_user = self.helper.read_user(user_cpr=cpr) logger.info("Existing MO info: {}".format(mo_user)) if mo_user: mo_uuid = mo_user.get("uuid") else: mo_uuid = self._create_user(ad_user, cpr_field) AD_username = self.helper.get_e_itsystems( mo_uuid, self.AD_it_system_uuid ) if not AD_username: self._connect_user_to_ad(ad_user) current_engagements = self.helper.read_user_engagement(user=mo_uuid) this_engagement = list( filter( lambda x: x.get("org_unit").get("uuid") == uuids["unit_uuid"], current_engagements, ) ) if not this_engagement: self._create_engagement(ad_user, uuids, mo_uuid)
class ImportUtility(object): """ ImportUtility TODO: This class relies heavily on asserts, which are gennerally not wanted in this context. We should create meaningfull exceptions and raise these when needed. """ def __init__(self, system_name, end_marker, mox_base, mora_base, demand_consistent_uuids, store_integration_data=False, dry_run=False): # Import Params self.demand_consistent_uuids = demand_consistent_uuids self.store_integration_data = store_integration_data if store_integration_data: self.ia = IntegrationAbstraction(mox_base, system_name, end_marker) # Service endpoint base self.mox_base = mox_base self.mora_base = mora_base # Session self.mh = MoraHelper(self.mora_base, use_cache=False) self.session = Session() # Placeholder for UUID import self.organisation_uuid = None # Existing UUIDS # TODO: More elegant version of this please self.existing_uuids = [] # UUID map self.inserted_organisation = {} self.inserted_facet_map = {} self.inserted_klasse_map = {} self.inserted_itsystem_map = {} self.inserted_org_unit_map = {} self.inserted_employee_map = {} # Deprecated self.dry_run = dry_run def import_organisation(self, reference, organisation): """ Convert organisation to OIO formatted post data and import into the MOX datastore. :param str reference: Reference to the user defined identifier :param object organisation: Organisation object :returns: Inserted UUID :rtype: str/uuid """ if not isinstance(organisation, Organisation): raise TypeError("Not of type Organisation") resource = "organisation/organisation" integration_data = self._integration_data( resource=resource, reference=reference, payload={} ) organisation.integration_data = integration_data payload = organisation.build() if organisation.uuid is not None: organisation_uuid = organisation.uuid else: organisation_uuid = integration_data.get('uuid', None) self.organisation_uuid = self.insert_mox_data( resource=resource, data=payload, uuid=organisation_uuid ) # Global validity self.date_from = organisation.date_from self.date_to = organisation.date_to return self.organisation_uuid def import_klassifikation(self, reference, klassifikation): """ Begin import of klassifikation :param str reference: Reference to the user defined identifier :param object klassifikation: Klassifikation object :returns: Inserted UUID :rtype: str/uuid """ if not isinstance(klassifikation, Klassifikation): raise TypeError("Not of type Klassifikation") resource = "klassifikation/klassifikation" klassifikation.organisation_uuid = self.organisation_uuid integration_data = self._integration_data( resource=resource, reference=reference, payload={} ) klassifikation.integration_data = integration_data payload = klassifikation.build() klassifikation_uuid = integration_data.get('uuid', None) self.klassifikation_uuid = self.insert_mox_data( resource=resource, data=payload, uuid=klassifikation_uuid ) return self.klassifikation_uuid def import_facet(self, reference, facet): """ Begin import of facet :param str reference: Reference to the user defined identifier :param object facet: Facet object :returns: Inserted UUID :rtype: str/uuid """ if not isinstance(facet, Facet): raise TypeError("Not of type Facet") resource = "klassifikation/facet" facet.organisation_uuid = self.organisation_uuid facet.klassifikation_uuid = self.klassifikation_uuid # NEED TO BE FIXED facet.date_from = self.date_from facet.date_to = self.date_to integration_data = self._integration_data( resource=resource, reference=reference, payload={} ) facet.integration_data = integration_data payload = facet.build() facet_uuid = integration_data.get('uuid', None) self.inserted_facet_map[reference] = self.insert_mox_data( resource=resource, data=payload, uuid=facet_uuid ) return self.inserted_facet_map[reference] def import_klasse(self, reference, klasse): """ Insert a klasse object Begin import of klassifikation :param str reference: Reference to the user defined identifier :param object organisation: Organisation object :returns: Inserted UUID :rtype: str/uuid """ if not isinstance(klasse, Klasse): raise TypeError("Not of type Facet") uuid = klasse.uuid facet_ref = klasse.facet_type_ref facet_uuid = self.inserted_facet_map.get(facet_ref) if not facet_uuid: error_message = "Facet ref: {} does not exist for {}".format( facet_ref, klasse ) logger.error(error_message) raise KeyError(error_message) resource = "klassifikation/klasse" klasse.organisation_uuid = self.organisation_uuid klasse.facet_uuid = facet_uuid klasse.date_from = self.date_from klasse.date_to = self.date_to integration_data = self._integration_data( resource=resource, reference=reference, payload={} ) if 'uuid' in integration_data: klasse_uuid = integration_data['uuid'] assert(uuid is None or klasse_uuid == uuid) else: if uuid is None: klasse_uuid = None else: klasse_uuid = uuid klasse.integration_data = integration_data payload = klasse.build() import_uuid = self.insert_mox_data( resource="klassifikation/klasse", data=payload, uuid=klasse_uuid ) assert(uuid is None or import_uuid == str(klasse_uuid)) self.inserted_klasse_map[reference] = import_uuid return self.inserted_klasse_map[reference] def import_itsystem(self, reference, itsystem): """ Create IT System :param str reference: Reference to the user defined identifier :param object organisation: Organisation object :returns: Inserted UUID :rtype: str/uuid """ if not isinstance(itsystem, Itsystem): raise TypeError("Not of type Itsystem") resource = 'organisation/itsystem' itsystem.organisation_uuid = self.organisation_uuid itsystem.date_from = self.date_from itsystem.date_to = self.date_to integration_data = self._integration_data( resource=resource, reference=reference, payload={} ) if 'uuid' in integration_data: itsystem_uuid = integration_data['uuid'] else: itsystem_uuid = None itsystem.integration_data = integration_data payload = itsystem.build() self.inserted_itsystem_map[reference] = self.insert_mox_data( resource=resource, data=payload, uuid=itsystem_uuid ) return self.inserted_itsystem_map[reference] def import_org_unit(self, reference, organisation_unit, details=[]): """ Insert organisation unit and details .. note:: Optional data objects are relational objects which belong to the organisation unit, such as an address type :param str reference: Reference to the user defined identifier :param object organisation_unit: Organisation object :param list details: List of details :returns: Inserted UUID :rtype: str/uuid """ if not isinstance(organisation_unit, OrganisationUnitType): raise TypeError("Not of type OrganisationUnitType") if reference in self.inserted_org_unit_map: return False resource = 'organisation/organisationenhed' # payload = self.build_mo_payload(organisation_unit_data) parent_ref = organisation_unit.parent_ref if parent_ref: parent_uuid = self.inserted_org_unit_map.get(parent_ref) organisation_unit.parent_uuid = parent_uuid if not organisation_unit.parent_uuid: organisation_unit.parent_uuid = self.organisation_uuid type_ref_uuid = self.inserted_klasse_map.get( organisation_unit.type_ref ) if hasattr(organisation_unit, "time_planning_ref"): organisation_unit.time_planning_ref_uuid = self.inserted_klasse_map.get( organisation_unit.time_planning_ref ) if hasattr(organisation_unit, 'org_unit_level_ref'): organisation_unit.org_unit_level_uuid = self.inserted_klasse_map.get( organisation_unit.org_unit_level_ref ) organisation_unit.type_ref_uuid = type_ref_uuid payload = organisation_unit.build() payload = self._integration_data( resource=resource, reference=reference, payload=payload, encode_integration=False ) if 'uuid' in payload: if payload['uuid'] in self.existing_uuids: logger.info('Re-import org-unit: {}'.format(payload['uuid'])) re_import = 'NO' resource = 'details/edit' payload_keys = list(payload.keys()) payload['data'] = {} for key in payload_keys: payload['data'][key] = payload[key] del payload[key] payload['type'] = 'org_unit' else: re_import = 'NEW' logger.info('New unit - Forced uuid: {}'.format(payload['uuid'])) resource = 'ou/create' else: re_import = 'NEW' logger.info('New unit, random uuid') resource = 'ou/create' logger.debug('Unit payload: {}'.format(payload)) uuid = self.insert_mora_data( resource=resource, data=payload ) if 'uuid' in payload: assert (uuid == payload['uuid']) if not uuid: raise ConnectionError("Something went wrong") # Add to the inserted map self.inserted_org_unit_map[reference] = uuid data = {} data['address'] = self.mh._get_detail(uuid, 'address', object_type='ou') # Build details (if any) details_payload = [] for detail in details: detail.org_unit_uuid = uuid date_from = detail.date_from if not date_from: date_from = organisation_unit.date_from build_detail = self.build_detail( detail=detail ) if not build_detail: continue found_hit = self._payload_compare(build_detail, data) logger.debug('Found hit for {}: {}'.format(uuid, found_hit)) if not found_hit and re_import == 'NO': re_import = 'YES' # TODO: SHOLD WE UPDATE FROM TO TODAY IN CASE OF RE-IMPORT? if re_import == 'YES': valid_from = datetime.now().strftime('%Y-%m-%d') # today build_detail['validity']['from'] = valid_from details_payload.append(build_detail) if re_import == 'YES': logger.info('Terminating details for unit {}'.format(uuid)) for item in data['address']: self._terminate_details(item['uuid'], 'address') if re_import in ('YES', 'NEW'): logger.info('Re-import unit: {}'.format(re_import)) self.insert_mora_data( resource="details/create", data=details_payload ) return uuid def import_employee(self, reference, employee, details=[]): """ Import employee :param str reference: Reference to the user defined identifier :param object employee: Employee object :param list details: List of details :returns: Inserted UUID :rtype: str/uuid """ if not isinstance(employee, EmployeeType): raise TypeError("Not of type EmployeeType") employee.org_uuid = self.organisation_uuid payload = employee.build() mox_resource = 'organisation/bruger' integration_data = self._integration_data( resource=mox_resource, reference=reference, payload=payload, encode_integration=False ) if 'uuid' in payload and payload['uuid'] in self.existing_uuids: logger.info('Re-import employee {}'.format(payload['uuid'])) re_import = 'NO' else: re_import = 'NEW' logger.info('New employee, uuid {}'.format(payload.get('uuid'))) # We unconditionally create or update the user, this should # ensure that we are always updated with correct current information. mora_resource = "e/create" uuid = self.insert_mora_data( resource=mora_resource, data=integration_data ) if 'uuid' in integration_data: assert (uuid == integration_data['uuid']) # Add uuid to the inserted employee map self.inserted_employee_map[reference] = uuid data = {} data['it'] = self.mh._get_detail(uuid, 'it') data['role'] = self.mh._get_detail(uuid, 'role') data['leave'] = self.mh._get_detail(uuid, 'leave') data['address'] = self.mh._get_detail(uuid, 'address') data['manager'] = self.mh._get_detail(uuid, 'manager') data['engagement'] = self.mh._get_detail(uuid, 'engagement') data['association'] = self.mh._get_detail(uuid, 'association') # In case of en explicit termination, we terminate the employee or # employment and return imidiately. for detail in details: if isinstance(detail, TerminationType): logger.info( 'Explicit termination of eng {} from {}'.format( uuid, detail.date_from ) ) self._terminate_employee(uuid, date_from=detail.date_from) return uuid if isinstance(detail, EngagementTerminationType): logger.info( 'Explicit termination of eng {}'.format( detail.engagement_uuid ) ) self._terminate_details(detail.engagement_uuid, 'engagement') return uuid if details: additional_payload = [] for detail in details: if not detail.date_from: detail.date_from = self.date_from # Create payload (as dict) detail_payload = self.build_detail( detail=detail, employee_uuid=uuid ) if not detail_payload: continue # If we do not have existing data, the new data should be imported if len(data[detail_payload['type']]) == 0 and re_import == 'NO': re_import = 'UPDATE' elif data[detail_payload['type']]: found_hit = self._payload_compare(detail_payload, data) if not found_hit: re_import = 'YES' additional_payload.append(detail_payload) for item in additional_payload: valid_from = item['validity']['from'] valid_to = item['validity']['to'] now = datetime.now() py_from = datetime.strptime(valid_from, '%Y-%m-%d') if valid_to is not None: py_to = datetime.strptime(valid_to, '%Y-%m-%d') else: py_to = datetime.strptime('2200-01-01', '%Y-%m-%d') logger.debug( 'Py-from:{}, Py-to:{}, Now:{}'.format(py_from, py_to, now) ) if re_import == 'YES' and py_from < now and py_to > now: logger.debug('Updating valid_from') valid_from = datetime.now().strftime('%Y-%m-%d') # today item['validity']['from'] = valid_from logger.info('Re-import: {}'.format(re_import)) if re_import == 'YES': logger.info('Non-explicit termination: {}'.format(uuid)) self._terminate_employee(uuid) if re_import in ('YES', 'NEW', 'UPDATE'): self.insert_mora_data( resource="details/create", data=additional_payload ) return uuid def build_detail(self, detail, employee_uuid=None): """ Build detail payload :param MoType detail: Detail object .. note:: A detail can be one of the following types: - address - asso - role - itsystem - engagement - manager :Reference: :mod:`os2mo_data_import.mora_data_types` :param str employee_uuid: (Option) Employee uuid if it exists :return: Detail POST data payload :rtype: dict """ if employee_uuid: detail.person_uuid = employee_uuid common_attributes = [ ("type_ref", "type_ref_uuid"), ("job_function_ref", "job_function_uuid"), ("address_type_ref", "address_type_uuid"), ("manager_level_ref", "manager_level_uuid") ] for check_value, set_value in common_attributes: if not hasattr(detail, check_value): continue uuid = self.inserted_klasse_map.get( getattr(detail, check_value) ) if not uuid: if self.store_integration_data: klasse_res = 'klassifikation/klasse' uuid = self.ia.find_object(klasse_res, getattr(detail, check_value)) else: # print('Detail: {}, check_value: {}'.format(detail, check_value)) pass setattr(detail, set_value, uuid) # Uncommon attributes if hasattr(detail, "visibility_ref"): detail.visibility_ref_uuid = self.inserted_klasse_map.get( detail.visibility_ref ) if hasattr(detail, 'primary_ref'): detail.primary_uuid = self.inserted_klasse_map.get( detail.primary_ref ) if hasattr(detail, "org_unit_ref"): detail.org_unit_uuid = self.inserted_org_unit_map.get( detail.org_unit_ref ) if hasattr(detail, "organisation_uuid"): detail.organisation_uuid = self.organisation_uuid if hasattr(detail, "itsystem_ref"): detail.itsystem_uuid = self.inserted_itsystem_map.get( detail.itsystem_ref ) if hasattr(detail, "responsibilities"): detail.responsibilities = [ self.inserted_klasse_map[reference] for reference in detail.responsibility_list ] return detail.build() def _integration_data(self, resource, reference, payload={}, encode_integration=True): """ Update the payload with integration data. Checks if an object with this integration data already exists. In this case the uuid of the exisiting object is put into the payload. If a supplied uuid is inconsistent with the uuid found from integration data, an exception is raised. :param resource: LoRa resource URL. :param referece: Unique label that will be stored in the integration data to identify the object on re-import. :param payload: The supplied payload will be updated with values for integration and uuid (if the integration data was found from an earlier import). For MO objects, payload will typically be pre-populated and will then be ready for import when returned. For MOX objects, the initial payload will typically be empty, and the returned values can be fed to the relevant adapter. :param encode_integration: If True, the integration data will be returned in json-encoded form. :return: The original payload updated with integration data and object uuid, if the object was already imported. """ # TODO: We need to have a list of all objects with integration data to # be able to make a list of objects that has disappeared if self.store_integration_data: uuid = self.ia.find_object(resource, reference) if uuid: if 'uuid' in payload: if self.demand_consistent_uuids: assert(payload['uuid'] == uuid) else: payload['uuid'] = uuid payload['uuid'] = uuid self.existing_uuids.append(uuid) payload['integration_data'] = self.ia.integration_data_payload( resource, reference, uuid, encode_integration ) return payload def insert_mox_data(self, resource, data, uuid=None): service_url = urljoin( base=self.mox_base, url=resource ) if uuid: update_url = "{service}/{uuid}".format( service=service_url, uuid=uuid ) response = self.session.put( url=update_url, json=data ) if response.status_code != 200: logger.error( 'Mox put. Response: {}, data'.format(response.text, data) ) raise HTTPError("Inserting mox data failed") else: response = self.session.post( url=service_url, json=data ) if response.status_code != 201: logger.error( 'Mox post. Response: {}, data'.format(response.text, data) ) raise HTTPError("Inserting mox data failed") response_data = response.json() return response_data["uuid"] def insert_mora_data(self, resource, data, uuid=None): # TESTING if self.dry_run: uuid = uuid4() return str(uuid) response = self.mh._mo_post( url=resource, payload=data, force=True, ) if response.status_code == 400: error = response.json()['description'] if error.find('does not give raise to a new registration') > 0: uuid_start = error.find('with id [') uuid = error[uuid_start+9:uuid_start+45] try: UUID(uuid, version=4) except ValueError: raise Exception('Unable to read uuid') else: logger.error( 'MO post. Error: {}, data: {}'.format(error, data) ) raise HTTPError("Inserting mora data failed") elif response.status_code not in (200, 201): logger.error( 'MO post. Response: {}, data: {}'.format(response.text, data) ) raise HTTPError("Inserting mora data failed") else: uuid = response.json() return uuid def _terminate_employee(self, uuid, date_from=None): endpoint = 'e/{}/terminate' yesterday = datetime.now() - timedelta(days=1) if date_from: to = date_from else: to = yesterday.strftime('%Y-%m-%d') payload = { 'terminate_all': True, 'validity': { 'to': to } } resource = endpoint.format(uuid) self.insert_mora_data( resource=resource, data=payload ) return uuid def _terminate_details(self, uuid, detail_type): logger.info('Terminate detail {}: {}'.format(uuid, detail_type)) yesterday = datetime.now() - timedelta(days=1) payload = { 'type': detail_type, 'uuid': uuid, 'validity': { 'to': yesterday.strftime('%Y-%m-%d') } } logger.debug('Terminate detail payload: {}'.format(payload)) try: self.insert_mora_data( resource='details/terminate', data=payload ) except HTTPError: logger.info('Tried to terminate a non-existing uuid') return uuid def _std_compare(self, item_payload, data_item, extra_field=None): """ Helper for _payload_compare, performs the checks that are identical for most object types. :param item_payload: The new payload data. :param data_item: The existing set of data. :param extra_field: If not None the comparison will also be done on this field, otherwise the comparison is only performed on uuid and validity. :return: True if identical, otherwise False """ identical = ( (data_item['org_unit']['uuid'] == item_payload['org_unit']['uuid']) and (data_item['validity']['from'] == item_payload['validity']['from']) and (data_item['validity']['to'] == item_payload['validity']['to']) ) if extra_field is not None: identical = ( identical and data_item[extra_field]['uuid'] == item_payload[extra_field]['uuid'] ) return identical def _payload_compare(self, item_payload, data): """ Compare an exising data-set with a new payload and tell whether the new payload is different from the exiting data. :param item_payload: New the payload data. :param data_item: The existing set of data. :param extra_field: If not None the comparison will also be done on this field, otherwise the comparison is only performed on uuid and validity. :return: True if identical, otherwise False """ data_type = item_payload['type'] logger.debug( 'Payload compare. item_payload: {}, data: {}'.format(item_payload, data) ) found_hit = False if data_type == 'engagement': # In priciple, we should be able to re-calculate the hash stored in # integration data and compare directly from that. for data_item in data[data_type]: if (self._std_compare(item_payload, data_item, 'job_function') and item_payload['user_key'] == data_item['user_key'] and item_payload['fraction'] == data_item['fraction']): found_hit = True elif data_type == 'role': for data_item in data[data_type]: if self._std_compare(item_payload, data_item, 'role_type'): found_hit = True logger.debug('Found hit in role') elif data_type == 'leave': for data_item in data[data_type]: if ((data_item['validity']['from'] == item_payload['validity']['from']) and (data_item['validity']['to'] == item_payload['validity']['to'])): found_hit = True logger.debug('Found hit in leave') elif data_type == 'it': for data_item in data[data_type]: if ( (data_item['validity']['from'] == item_payload['validity']['from']) and (data_item['validity']['to'] == item_payload['validity']['to']) and (data_item['itsystem']['uuid'] == item_payload['itsystem']['uuid']) ): found_hit = True logger.debug('Found hit in it') elif data_type == 'address': for data_item in data[data_type]: if ( (data_item['validity']['from'] == item_payload['validity']['from']) and (data_item['validity']['to'] == item_payload['validity']['to']) and (data_item['value'] == item_payload['value']) ): found_hit = True logger.debug('Found hit in adress') elif data_type == 'manager': for data_item in data[data_type]: identical = self._std_compare(item_payload, data_item, 'manager_level') uuids = [] for item in item_payload['responsibility']: uuids.append(item['uuid']) for responsibility in data_item['responsibility']: identical = identical and (responsibility['uuid'] in uuids) identical = (identical and (len(data_item['responsibility']) == len(uuids))) if identical: found_hit = True logger.debug('Found hit in manager') elif data_type == 'association': for data_item in data[data_type]: if self._std_compare(item_payload, data_item, 'association_type'): found_hit = True logger.debug('Found hit in association') else: raise Exception('Uknown detail!') logger.info('Found hit: {}'.format(found_hit)) return found_hit
class SnurreBasse: def _find_classes(self, facet): class_types = self.helper.read_classes_in_facet(facet) types_dict = {} facet = class_types[1] for class_type in class_types[0]: types_dict[class_type["user_key"]] = class_type["uuid"] return types_dict, facet def __init__(self, session): self.session = session self.top_per_unit = {} self.helper = MoraHelper(hostname=settings["mora.base"], use_cache=False) self.unit_types, self.unit_type_facet = self._find_classes( "org_unit_type") self.unit_levels, self.unit_level_facet = self._find_classes( "org_unit_level") def get_top_unit(self, lc_enhed): """ return the top unit for a unit """ top_unit = self.top_per_unit.get(lc_enhed.uuid) if top_unit: return top_unit branch = [lc_enhed.uuid] # walk as far up as necessary while lc_enhed.forældreenhed_uuid is not None: uuid = lc_enhed.forældreenhed_uuid top_unit = self.top_per_unit.get(uuid) if top_unit: break branch.append(uuid) lc_enhed = self.session.query(Enhed).filter( Enhed.uuid == uuid).one() top_unit = uuid # last one effective # register top unit for all encountered for buuid in branch: self.top_per_unit[buuid] = top_unit return top_unit def level_from_type(self, outype_bvn): return self.unit_levels[outype_bvn] def update_unit(self, lc_enhed): mo_unit = self.helper.read_ou(lc_enhed.uuid) if (mo_unit["org_unit_level"] is not None and mo_unit["org_unit_level"]["user_key"] in self.unit_levels): logger.debug("already done: %s", lc_enhed.uuid) return payload = { "type": "org_unit", "data": { "uuid": lc_enhed.uuid, "org_unit_level": { "uuid": self.level_from_type(mo_unit["org_unit_type"]["user_key"]), }, "org_unit_type": { "uuid": settings[ "snurrebasser.lcdb_traverse_units.org_unit_type_uuid"], }, "validity": { "from": mo_unit["validity"]["from"], "to": None }, }, } logger.info("Edit unit: {}".format(payload)) response = self.helper._mo_post("details/edit", payload) if (response.status_code == 400 and response.text.find("raise to a new registration") > 0): pass else: response.raise_for_status() def run(self): def is_relevant(session, lc_enhed): top_unit = self.get_top_unit(lc_enhed) return (top_unit == settings["snurrebasser.lcdb_traverse_units.top_unit_uuid"]) relevant = partial(is_relevant, session) alle_enheder = session.query(Enhed) for i in filter(relevant, alle_enheder): self.update_unit(i)