def dependency_pending(dep_id, local_db=db, local_co=const): if not dep_id: return False br = BofhdRequests(local_db, local_co) for dr in br.get_requests(request_id=dep_id): logger.debug("waiting for request %d" % int(dep_id)) return True return False
def read_pending_moves(self): br = BofhdRequests(self._db, self.const) # We define near future as 15 minutes from now. near_future = mx.DateTime.now() + mx.DateTime.DateTimeDelta(0, 0, 15) for op in (self.const.bofh_email_create, self.const.bofh_email_convert): for r in br.get_requests(operation=op): if r['run_at'] < near_future: self.pending[int(r['entity_id'])] = True
def read_pending_moves(self): br = BofhdRequests(self._db, self.const) # We define near future as 15 minutes from now. near_future = mx.DateTime.now() + mx.DateTime.DateTimeDelta(0, 0, 15) for op in (self.const.bofh_email_create, self.const.bofh_email_move, self.const.bofh_email_convert): for r in br.get_requests(operation=op): if r['run_at'] < near_future: self.pending[int(r['entity_id'])] = True
def process_move_student_requests(): global fnr2move_student, autostud br = BofhdRequests(db, const) rows = br.get_requests(operation=const.bofh_move_student) if not rows: return logger.debug("Preparing autostud framework") autostud = AutoStud.AutoStud(db, logger, debug=False, cfg_file=studconfig_file, studieprogs_file=studieprogs_file, emne_info_file=emne_info_file, ou_perspective=ou_perspective) # Hent ut personens fødselsnummer + account_id fnr2move_student = {} account = Utils.Factory.get('Account')(db) person = Utils.Factory.get('Person')(db) for r in rows: if not is_valid_request(r['request_id']): continue account.clear() account.find(r['entity_id']) person.clear() person.find(account.owner_id) fnr = person.get_external_id(id_type=const.externalid_fodselsnr, source_system=const.system_fs) if not fnr: logger.warn("Not student fnr for: %i" % account.entity_id) br.delete_request(request_id=r['request_id']) db.commit() continue fnr = fnr[0]['external_id'] fnr2move_student.setdefault(fnr, []).append( (int(account.entity_id), int(r['request_id']), int(r['requestee_id']))) logger.debug("Starting callbacks to find: %s" % fnr2move_student) autostud.start_student_callbacks(student_info_file, move_student_callback) # Move remaining users to pending disk disk = Utils.Factory.get('Disk')(db) disk.find_by_path(cereconf.AUTOSTUD_PENDING_DISK) logger.debug(str(fnr2move_student.values())) for tmp_stud in fnr2move_student.values(): for account_id, request_id, requestee_id in tmp_stud: logger.debug("Sending %s to pending disk" % repr(account_id)) br.delete_request(request_id=request_id) br.add_request(requestee_id, br.batch_time, const.bofh_move_user, account_id, disk.entity_id, state_data=int(default_spread)) db.commit()
def process_requests(types): global max_requests operations = { 'sympa': [(const.bofh_sympa_create, proc_sympa_create, 2*60), (const.bofh_sympa_remove, proc_sympa_remove, 2*60)], } """Each type (or category) of requests consists of a list of which requests to process. The tuples are operation, processing function, and how long to delay the request (in minutes) if the function returns False. """ # TODO: There is no variable containing the default log directory # in cereconf reqlock = RequestLockHandler() br = BofhdRequests(db, const) for t in types: if t == 'move' and is_ok_batch_time(time.strftime("%H:%M")): # convert move_student into move_user requests process_move_student_requests() if t == 'email': process_email_move_requests() for op, process, delay in operations[t]: set_operator() start_time = time.time() for r in br.get_requests(operation=op, only_runnable=True): reqid = r['request_id'] logger.debug("Req: %s %d at %s, state %r", op, reqid, r['run_at'], r['state_data']) if time.time() - start_time > 30 * 60: break if r['run_at'] > mx.DateTime.now(): continue if not is_valid_request(reqid): continue if reqlock.grab(reqid): if max_requests <= 0: break max_requests -= 1 if process(r): br.delete_request(request_id=reqid) db.commit() else: db.rollback() if delay: br.delay_request(reqid, minutes=delay) db.commit() reqlock.release()
def process_email_move_requests(): # Easy round robin to balance load on real mail servers round_robin = {} for i in cereconf.PROC_BOFH_REQ_MOVE_SERVERS: round_robin[i] = 0 def get_srv(): srv = "" low = -1 for s in round_robin: if round_robin[s] < low or low == -1: srv = s low = round_robin[s] round_robin[srv] += 1 return srv br = BofhdRequests(db, const) rows = [r for r in br.get_requests(operation=const.bofh_email_move, only_runnable=True)] if len(rows) == 0: return procs = [] while len(rows) > 0 or len(procs) > 0: start = time.time() if len(procs) < 20 and len(rows) > 0: host = get_srv() r = rows.pop() pid = os.fork() if pid == 0: email_move_child(host, r) sys.exit(0) else: procs.append((pid, host)) for pid, host in procs: status = os.waitpid(pid, os.WNOHANG) # (X, Y) = waitpid # X = 0 - still running # X = pid - finished # Y = exit value if status[0] != 0: # process finished if status[1] != 0: logger.error("fork '%d' exited with code: %d", status[0], status[1]) procs.remove((pid, host)) round_robin[host] -= 1 # Don't hog the CPU while throttling if time.time() <= start + 1: time.sleep(0.5)
def _UiO_order_cyrus_action(self, action, destination, state_data=None): br = BofhdRequests(self._db, self.const) # If there are any registered BofhdRequests for this account # that would conflict with 'action', remove them. for anti_action in br.get_conflicts(action): for r in br.get_requests(entity_id=self.entity_id, operation=anti_action): self.logger.info("Removing BofhdRequest #%d: %r", r["request_id"], r) br.delete_request(request_id=r["request_id"]) # If the ChangeLog module knows who the user requesting this # change is, use that knowledge. Otherwise, set requestor to # None; it's the best we can do. requestor = getattr(self._db, "change_by", None) # Register a BofhdRequest to create the mailbox. reqid = br.add_request(requestor, br.now, action, self.entity_id, destination, state_data=state_data)
def _UiO_order_cyrus_action(self, action, destination, state_data=None): br = BofhdRequests(self._db, self.const) # If there are any registered BofhdRequests for this account # that would conflict with 'action', remove them. for anti_action in br.get_conflicts(action): for r in br.get_requests(entity_id=self.entity_id, operation=anti_action): self.logger.info("Removing BofhdRequest #%d: %r", r['request_id'], r) br.delete_request(request_id=r['request_id']) # If the ChangeLog module knows who the user requesting this # change is, use that knowledge. Otherwise, set requestor to # None; it's the best we can do. requestor = getattr(self._db, 'change_by', None) # Register a BofhdRequest to create the mailbox. reqid = br.add_request(requestor, br.now, action, self.entity_id, destination, state_data=state_data)
def process_requests(self, operations_map, op_types, max_requests): with closing(RequestLockHandler()) as reqlock: br = BofhdRequests(self.db, self.co) for t in op_types: for op in self.op_type_map[t]: if op not in operations_map: logger.info('Unable to process operation %r', op) continue func, settings = operations_map[op] delay = settings.get('delay', 0) set_operator(self.db) start_time = time.time() for r in br.get_requests(operation=op, only_runnable=True): reqid = r['request_id'] logger.debug("Req: %s %d at %s, state %r", op, reqid, r['run_at'], r['state_data']) if time.time() - start_time > 30 * 60: break if r['run_at'] > mx.DateTime.now(): continue # Moving users only at ok times if (op is self.co.bofh_move_user and not is_ok_batch_time()): break if not is_valid_request(br, reqid): continue if reqlock.grab(reqid): if max_requests <= 0: break max_requests -= 1 if func(r): br.delete_request(request_id=reqid) self.db.commit() else: self.db.rollback() if delay: br.delay_request(reqid, minutes=delay) self.db.commit()
class Quarantine2Request(EvtHandler): """When a quarantine has been added/updated/deleted, we register a bofh_quarantine_refresh bofhd_request on the apropriate start_date, end_date and disable_until dates. """ def __init__(self): self.br = BofhdRequests(db, const) self.eq = EntityQuarantine(db) def get_triggers(self): return ("quarantine_add", "quarantine_mod", "quarantine_del") def _get_quarantine(self, entity_id, q_type): self.eq.clear() try: self.eq.find(entity_id) except Errors.NotFoundError: return None qdata = self.eq.get_entity_quarantine(q_type) if not qdata: return None return qdata[0] def notify_quarantine_add(self, evt, params): # Register a bofh_quarantine_refresh on start, end and # disable_date qdata = self._get_quarantine(evt['subject_entity'], params['q_type']) if not qdata: return True for when in ('start_date', 'end_date', 'disable_until'): if qdata[when] is not None: self.br.add_request(None, qdata[when], const.bofh_quarantine_refresh, evt['subject_entity'], None, state_data=int(params['q_type'])) db.commit() return True def notify_quarantine_mod(self, evt, params): # Currently only disable_until is affected by quarantine_mod. qdata = self._get_quarantine(evt['subject_entity'], params['q_type']) if not qdata: return True if qdata['disable_until']: self.br.add_request(None, qdata['disable_until'], const.bofh_quarantine_refresh, evt['subject_entity'], None, state_data=int(params['q_type'])) self.br.add_request(None, self.br.now, const.bofh_quarantine_refresh, evt['subject_entity'], None, state_data=int(params['q_type'])) db.commit() return True def notify_quarantine_del(self, evt, params): # Remove existing requests for this entity_id/quarantine_type # combination as they are no longer needed for row in self.br.get_requests( entity_id=evt['subject_entity'], operation=int(const.bofh_quarantine_refresh)): if int(row['state_data']) == int(params['q_type']): self.br.delete_request(request_id=row['request_id']) self.br.add_request(None, self.br.now, const.bofh_quarantine_refresh, evt['subject_entity'], None, state_data=int(params['q_type'])) db.commit() return True
class MoveStudentProcessor(object): def __init__(self, db, co, ou_perspective, emne_info_file, studconfig_file, studieprogs_file, default_spread=None): self.db = db self.co = co self.br = BofhdRequests(self.db, self.co) self.default_spread = default_spread logger.debug("Preparing autostud framework") self.autostud = AutoStud.AutoStud(self.db, logger.getChild('autostud'), debug=False, cfg_file=studconfig_file, studieprogs_file=studieprogs_file, emne_info_file=emne_info_file, ou_perspective=ou_perspective) def process_requests(self, student_info_file): rows = self.br.get_requests(operation=self.co.bofh_move_student) if not rows: return # Set self.fnr2move_student self.set_fnr2move_student(rows) logger.debug("Starting callbacks to find: %s" % self.fnr2move_student) self.autostud.start_student_callbacks(student_info_file, self.move_student_callback) self.move_remaining_users() def move_remaining_users(self): # Move remaining users to pending disk disk = Factory.get('Disk')(self.db) disk.find_by_path(cereconf.AUTOSTUD_PENDING_DISK) logger.debug(str(self.fnr2move_student.values())) for tmp_stud in self.fnr2move_student.values(): for account_id, request_id, requestee_id in tmp_stud: logger.debug("Sending %s to pending disk" % repr(account_id)) self.br.delete_request(request_id=request_id) self.br.add_request(requestee_id, self.br.batch_time, self.co.bofh_move_user, account_id, disk.entity_id, state_data=int(self.default_spread)) self.db.commit() def set_fnr2move_student(self, rows): # Hent ut personens fodselsnummer + account_id self.fnr2move_student = {} account = Factory.get('Account')(self.db) person = Factory.get('Person')(self.db) for r in rows: if not is_valid_request(self.br, r['request_id']): continue account.clear() account.find(r['entity_id']) person.clear() person.find(account.owner_id) fnr = person.get_external_id( id_type=self.co.externalid_fodselsnr, source_system=self.co.system_fs ) if not fnr: logger.warn("Not student fnr for: %i" % account.entity_id) self.br.delete_request(request_id=r['request_id']) self.db.commit() continue fnr = fnr[0]['external_id'] self.fnr2move_student.setdefault(fnr, []).append( (int(account.entity_id), int(r['request_id']), int(r['requestee_id']))) def move_student_callback(self, person_info): """We will only move the student if it has a valid fnr from FS, and it is not currently on a student disk. If the new homedir cannot be determined, user will be moved to a pending disk. process_students moves users from this disk as soon as a proper disk can be determined. Currently we only operate on the disk whose spread is default_spread""" fnr = "%06d%05d" % (int(person_info['fodselsdato']), int(person_info['personnr'])) logger.debug("Callback for %s" % fnr) try: fodselsnr.personnr_ok(fnr) except Exception, e: logger.exception(e) return if fnr not in self.fnr2move_student: return account = Factory.get('Account')(self.db) group = Factory.get('Group')(self.db) for account_id, request_id, requestee_id in \ self.fnr2move_student.get(fnr, []): account.clear() account.find(account_id) groups = list(int(x["group_id"]) for x in group.search(member_id=account_id, indirect_members=False)) try: profile = self.autostud.get_profile(person_info, member_groups=groups) logger.debug(profile.matcher.debug_dump()) except AutostudError, msg: logger.debug("Error getting profile, using pending: %s" % msg) continue disks = self.determine_disks(account, request_id, profile, fnr) logger.debug(str((fnr, account_id, disks))) if disks: logger.debug("Destination %s" % repr(disks)) del (self.fnr2move_student[fnr]) for disk, spread in disks: self.br.delete_request(request_id=request_id) self.br.add_request(requestee_id, self.br.batch_time, self.co.bofh_move_user, account_id, disk, state_data=spread) self.db.commit()
def is_valid_request(req_id, local_db=db, local_co=const): # The request may have been canceled very recently br = BofhdRequests(local_db, local_co) for r in br.get_requests(request_id=req_id): return True return False
def process_delete_requests(): br = BofhdRequests(db, const) now = mx.DateTime.now() del_file = [] group = Factory.get('Group')(db) account = Factory.get('Account')(db) spreads = [] posix_home = '' pwd = '*' uid = '' gid = '' gecos = 'gecos' shell = '' line = '' for r in br.get_requests(operation=const.bofh_delete_user): if not is_valid_request(r['request_id']): continue if not keep_running(): break if r['run_at'] > now: continue try: account.clear() account.find(r['entity_id']) except Errors.NotFoundError: logger.error('Could not find account %s' % r['entity_id']) continue if account.is_deleted(): logger.warn("%s is already deleted" % account.account_name) br.delete_request(request_id=r['request_id']) db.commit() continue logger.info("Trying to delete account %s", account.account_name) blockers = account.get_delete_blockers(ignore_group_memberships=True) if blockers: logger.error('Manual cleaning required: ' 'Deleting account %s is blocked by: %s', account.account_name, ', '.join(blockers)) continue set_operator(r['requestee_id']) # Set expire_date (do not export data about this account) account.expire_date = br.now logger.debug("expire_date for %s registered as %s", account.account_name, br.now) account.write_db() # check for posix attrs posix_user = Factory.get('PosixUser')(db) posix_user.clear() try: posix_user.find(r['entity_id']) except Errors.NotFoundError: posix_user = None # Deal with all the systems that account data is exported to spreads = account.get_spread() logger.debug("Fetched all spreads for %s, %s", account.account_name, spreads) for row in spreads: # Account is valid in AD, remove account@ad spread if row['spread'] == const.spread_hia_ad_account: try: home = account.get_home(row['spread']) account.set_homedir(current_id=home['homedir_id'], status=const.home_status_archived) account.clear_home(row['spread']) except Errors.NotFoundError: logger.debug("No home in AD for %s, will remove spread to " "AD only.", account.account_name) account.delete_spread(row['spread']) logger.debug("Deleted account@ad spread for %s", account.account_name) # student-accounts usually have account@ldap, remove this elif row['spread'] == const.spread_ldap_account: account.delete_spread(row['spread']) logger.debug("Deleted account@ldap spread for %s", account.account_name) # An email account exists, remove account@imap spread, # register email account delete elif row['spread'] == const.spread_hia_email: et = Email.EmailTarget(db) try: et.find_by_target_entity(account.entity_id) except Errors.NotFoundError: logger.warn('No email target for %s, removing imap spread ' 'only.', account.account_name) logger.debug("Found e-mail target for %s", account.account_name) if et: es = Email.EmailServer(db) es.find(et.email_server_id) del_file.append('EMAIL:' + account.account_name + ':' + es.name) et.email_target_type = const.email_target_deleted et.write_db() account.delete_spread(row['spread']) elif row['spread'] in (const.spread_exchange_account, const.spread_exchange_acc_old): et = Email.EmailTarget(db) try: et.find_by_target_entity(account.entity_id) except Errors.NotFoundError: logger.warn('No email target for %s, will remove exchange ' 'spread.', account.account_name) if et: et.email_target_type = const.email_target_deleted et.write_db() logger.info("Deleted e-mail target for %s", account.account_name) account.delete_spread(row['spread']) # Account is valid in nis@hia, remove account@nis spread, # register nis-home delete elif row['spread'] == const.spread_nis_user: if not isinstance(posix_user, Factory.get('PosixUser')): logger.error("Manual intervention required, no posix " "account is found for account %s", account.account_name) continue posix_home = posix_user.get_posix_home(row['spread']) uid = posix_user.posix_uid gid = posix_user.gid_id shell = posix_user.shell line = string.join([account.account_name, pwd, str(uid), str(gid), gecos, posix_home, str(shell)], ':') line = 'NIS:' + line del_file.append(line) try: home = account.get_home(row['spread']) except Errors.NotFoundError: continue account.set_homedir(current_id=home['homedir_id'], status=const.home_status_archived) logger.debug("Set home to archived %s (%s)", home['homedir_id'], row['spread']) account.clear_home(row['spread']) logger.debug("clear_home in %s", row['spread']) account.delete_spread(row['spread']) # Account is valid in nisans@hia, remove account@nisans spread, # register nisans-home delete elif row['spread'] == const.spread_ans_nis_user: posix_home = posix_user.get_posix_home(row['spread']) uid = posix_user.posix_uid gid = posix_user.gid_id shell = posix_user.shell line = string.join([account.account_name, pwd, str(uid), str(gid), gecos, posix_home, str(shell)], ':') line = 'NISANS:' + line del_file.append(line) try: home = account.get_home(row['spread']) except Errors.NotFoundError: continue account.set_homedir(current_id=home['homedir_id'], status=const.home_status_archived) logger.debug("Set home to archived %s (%s)", home['homedir_id'], row['spread']) account.clear_home(row['spread']) account.delete_spread(row['spread']) else: account.delete_spread(row['spread']) if posix_user: posix_user.delete_posixuser() # Remove account from all groups for g in group.search(member_id=account.entity_id, indirect_members=False): group.clear() group.find(g['group_id']) group.remove_member(account.entity_id) # All done, remove request, commit results account.write_db() if posix_user: posix_user.write_db() br.delete_request(request_id=r['request_id']) db.commit() return del_file
def process_delete_requests(): br = BofhdRequests(db, const) now = mx.DateTime.now() del_file = [] group = Factory.get('Group')(db) account = Factory.get('Account')(db) spreads = [] posix_home = '' pwd = '*' uid = '' gid = '' gecos = 'gecos' shell = '' line = '' for r in br.get_requests(operation=const.bofh_delete_user): if not is_valid_request(r['request_id']): continue if not keep_running(): break if r['run_at'] > now: continue try: account.clear() account.find(r['entity_id']) except Errors.NotFoundError: logger.error('Could not find account %s' % r['entity_id']) continue if account.is_deleted(): logger.warn("%s is already deleted" % account.account_name) br.delete_request(request_id=r['request_id']) db.commit() continue logger.info("Trying to delete account %s", account.account_name) blockers = account.get_delete_blockers(ignore_group_memberships=True) if blockers: logger.error( 'Manual cleaning required: ' 'Deleting account %s is blocked by: %s', account.account_name, ', '.join(blockers)) continue set_operator(r['requestee_id']) # Set expire_date (do not export data about this account) account.expire_date = br.now logger.debug("expire_date for %s registered as %s", account.account_name, br.now) account.write_db() # check for posix attrs posix_user = Factory.get('PosixUser')(db) posix_user.clear() try: posix_user.find(r['entity_id']) except Errors.NotFoundError: posix_user = None # Deal with all the systems that account data is exported to spreads = account.get_spread() logger.debug("Fetched all spreads for %s, %s", account.account_name, spreads) for row in spreads: # Account is valid in AD, remove account@ad spread if row['spread'] == const.spread_hia_ad_account: try: home = account.get_home(row['spread']) account.set_homedir(current_id=home['homedir_id'], status=const.home_status_archived) account.clear_home(row['spread']) except Errors.NotFoundError: logger.debug( "No home in AD for %s, will remove spread to " "AD only.", account.account_name) account.delete_spread(row['spread']) logger.debug("Deleted account@ad spread for %s", account.account_name) # student-accounts usually have account@ldap, remove this elif row['spread'] == const.spread_ldap_account: account.delete_spread(row['spread']) logger.debug("Deleted account@ldap spread for %s", account.account_name) # An email account exists, remove account@imap spread, # register email account delete elif row['spread'] == const.spread_hia_email: et = Email.EmailTarget(db) try: et.find_by_target_entity(account.entity_id) except Errors.NotFoundError: logger.warn( 'No email target for %s, removing imap spread ' 'only.', account.account_name) logger.debug("Found e-mail target for %s", account.account_name) if et: es = Email.EmailServer(db) es.find(et.email_server_id) del_file.append('EMAIL:' + account.account_name + ':' + es.name) et.email_target_type = const.email_target_deleted et.write_db() account.delete_spread(row['spread']) elif row['spread'] in (const.spread_exchange_account, const.spread_exchange_acc_old): et = Email.EmailTarget(db) try: et.find_by_target_entity(account.entity_id) except Errors.NotFoundError: logger.warn( 'No email target for %s, will remove exchange ' 'spread.', account.account_name) if et: et.email_target_type = const.email_target_deleted et.write_db() logger.info("Deleted e-mail target for %s", account.account_name) account.delete_spread(row['spread']) # Account is valid in nis@hia, remove account@nis spread, # register nis-home delete elif row['spread'] == const.spread_nis_user: if not isinstance(posix_user, Factory.get('PosixUser')): logger.error( "Manual intervention required, no posix " "account is found for account %s", account.account_name) continue posix_home = posix_user.get_posix_home(row['spread']) uid = posix_user.posix_uid gid = posix_user.gid_id shell = posix_user.shell line = string.join([ account.account_name, pwd, str(uid), str(gid), gecos, posix_home, str(shell) ], ':') line = 'NIS:' + line del_file.append(line) try: home = account.get_home(row['spread']) except Errors.NotFoundError: continue account.set_homedir(current_id=home['homedir_id'], status=const.home_status_archived) logger.debug("Set home to archived %s (%s)", home['homedir_id'], row['spread']) account.clear_home(row['spread']) logger.debug("clear_home in %s", row['spread']) account.delete_spread(row['spread']) # Account is valid in nisans@hia, remove account@nisans spread, # register nisans-home delete elif row['spread'] == const.spread_ans_nis_user: posix_home = posix_user.get_posix_home(row['spread']) uid = posix_user.posix_uid gid = posix_user.gid_id shell = posix_user.shell line = string.join([ account.account_name, pwd, str(uid), str(gid), gecos, posix_home, str(shell) ], ':') line = 'NISANS:' + line del_file.append(line) try: home = account.get_home(row['spread']) except Errors.NotFoundError: continue account.set_homedir(current_id=home['homedir_id'], status=const.home_status_archived) logger.debug("Set home to archived %s (%s)", home['homedir_id'], row['spread']) account.clear_home(row['spread']) account.delete_spread(row['spread']) else: account.delete_spread(row['spread']) if posix_user: posix_user.delete_posixuser() # Remove account from all groups for g in group.search(member_id=account.entity_id, indirect_members=False): group.clear() group.find(g['group_id']) group.remove_member(account.entity_id) # All done, remove request, commit results account.write_db() if posix_user: posix_user.write_db() br.delete_request(request_id=r['request_id']) db.commit() return del_file