def handle_error(self, error_message): # for debugging purposes print(error_message) # don't send emails when running as dev if not TESTING: send_problem_report(error_message)
def _get_first_available_uid(): """Return the first available UID number. Searches our entire People ou in order to find it. It seems like there should be a better way to do this, but quick searches don't show any. We hard-code a value we know has already been reached and only select entries greater than that for performance. We then use a module-level dict to cache our output for the next function call. """ min_uid = _cache['known_uid'] with ldap_ocf() as c: c.search( constants.OCF_LDAP_PEOPLE, '(uidNumber>={KNOWN_MIN})'.format(KNOWN_MIN=min_uid), attributes=['uidNumber'], ) uids = [int(entry['attributes']['uidNumber'][0]) for entry in c.response] if len(uids) > 2500: send_problem_report(( 'Found {} accounts with UID >= {}, ' 'you should bump the constant for speed.' ).format(len(uids), min_uid)) if uids: max_uid = max(uids) _cache['known_uid'] = max_uid else: # If cached UID is later deleted, LDAP response will be empty. max_uid = min_uid return max_uid + 1
def sync(self): try: for ldap_group, dest_group in self.SYNC_PAIRS: ldap_members = set(list_staff(group=ldap_group)) dest_members = set( self.dest_service().list_members(dest_group)) to_add = ldap_members - dest_members missing = dest_members - ldap_members if missing: missing_header = '''The following users are in the {dest} destination group but are not in the {ldap} LDAP group:'''.format(dest=dest_group, ldap=ldap_group) self.logger.warning(missing_header) for m in missing: self.logger.warning(m) for username in to_add: if not self.args.dry_run: self.dest_service().add_to_group(username, dest_group) self.logger.info('Adding {} to group {}'.format( username, dest_group)) except Exception as e: self.logger.exception('Exception caught: {}'.format(e)) if not self.args.dry_run: mail.send_problem_report( 'An exception occurred in ldapsync: \n\n{}'.format(e))
def _get_first_available_uid(): """Return the first available UID number. Searches our entire People ou in order to find it. It seems like there should be a better way to do this, but quick searches don't show any. We hard-code a value we know has already been reached and only select entries greater than that for performance. We then use a module-level dict to cache our output for the next function call. """ min_uid = _cache['known_uid'] with ldap_ocf() as c: c.search( constants.OCF_LDAP_PEOPLE, '(uidNumber>={KNOWN_MIN})'.format(KNOWN_MIN=min_uid), attributes=['uidNumber'], ) uids = [ int(entry['attributes']['uidNumber'][0]) for entry in c.response ] if len(uids) > 2500: send_problem_report( ('Found {} accounts with UID >= {}, ' 'you should bump the constant for speed.').format( len(uids), min_uid)) if uids: max_uid = max(uids) _cache['known_uid'] = max_uid else: # If cached UID is later deleted, LDAP response will be empty. max_uid = min_uid return max_uid + 1
def handle_loop_error(err: Failure, loop_handler: LoopHandler) -> None: """Handle errors in a looping function and restart the loop.""" send_problem_report(err) err.printTraceback() # Sleep to avoid tight infinite loops spamming emails time.sleep(3) # Restart the given method loop_handler.start_loop()
def test_problem_report(self, mock_popen): send_problem_report('hellllo world') msg = self.get_message(mock_popen) assert msg['Subject'].startswith('[ocflib] Problem report') assert msg['From'] == MAIL_FROM assert msg['To'] == MAIL_ROOT assert 'hellllo world' in msg.get_payload()
def process_exception(self, request: HttpRequest, exception: Exception) -> Any: if isinstance(exception, ResponseException): return exception.response # maybe it's a real exception? if settings.DEBUG or settings.TESTING: return if isinstance(exception, Http404): # we don't care about reporting 404 errors return traceback = sanitize(format_exc()) headers = sanitize_wsgi_context(request.META) try: send_problem_report( dedent( """\ An exception occured in ocfweb: {traceback} Request: * Host: {host} * Path: {path} * Method: {request.method} * Secure: {is_secure} Request Headers: {headers} Session: {session} """, ).format( traceback=traceback, request=request, host=request.get_host(), path=request.get_full_path(), is_secure=request.is_secure(), session=pformat(dict(request.session)), headers=pformat(headers), ), ) except Exception as ex: print(ex) # just in case it errors again here send_problem_report( dedent( """\ An exception occured in ocfweb, but we errored trying to report it: {traceback} """, ).format(traceback=format_exc()), ) raise
def process_exception(self, request, exception): if isinstance(exception, ResponseException): return exception.response # maybe it's a real exception? if settings.DEBUG or settings.TESTING: return if isinstance(exception, Http404): # we don't care about reporting 404 errors return try: send_problem_report( dedent( """\ An exception occured in ocfweb: {traceback} Request: * Host: {host} * Path: {path} * Method: {request.method} * Secure: {is_secure} Request Headers: {headers} Session: {session} """ ).format( traceback=format_exc(), request=request, host=request.get_host(), path=request.get_full_path(), is_secure=request.is_secure(), session=pformat(dict(request.session)), headers=pformat(request.META), ) ) except Exception as ex: print(ex) # just in case it errors again here send_problem_report( dedent( """\ An exception occured in ocfweb, but we errored trying to report it: {traceback} """ ).format(traceback=format_exc()) ) raise
def _write_ldif(lines, dn, keytab, admin_principal): """Issue an update to LDAP via ldapmodify in the form of lines of an LDIF file. :param lines: ldif file as a sequence of lines """ cmd = 'kinit -t {keytab} {principal} ldapmodify'.format( keytab=keytab, principal=admin_principal, ) child = pexpect.spawn(cmd, timeout=10) child.expect('SASL data security layer installed.') for line in lines: child.sendline(line) child.sendeof() child.expect('entry "{}"'.format(dn)) child.expect(pexpect.EOF) output_after_adding = child.before.decode('utf8').strip() if 'Already exists (68)' in output_after_adding: raise ValueError('Tried to create duplicate entry.') elif 'No such object (32)' in output_after_adding: raise ValueError('Tried to modify nonexistent entry.') if output_after_adding != '': send_problem_report( dedent( '''\ Unknown problem occured when trying to write to LDAP; the code should be updated to handle this case. dn: {dn} keytab: {keytab} principal: {principal} Unexpected output: {output_after_adding} Lines passed to ldapadd: {lines} ''' ).format( dn=dn, keytab=keytab, principal=admin_principal, output_after_adding=output_after_adding, lines='\n'.join(' ' + line for line in lines) ) ) raise ValueError('Unknown LDAP failure was encountered.')
def _write_ldif(lines, dn, keytab, admin_principal): """Issue an update to LDAP via ldapmodify in the form of lines of an LDIF file. :param lines: ldif file as a sequence of lines """ cmd = 'kinit -t {keytab} {principal} ldapmodify'.format( keytab=keytab, principal=admin_principal, ) child = pexpect.spawn(cmd, timeout=10) child.expect('SASL data security layer installed.') for line in lines: child.sendline(line) child.sendeof() child.expect('entry "{}"'.format(dn)) child.expect(pexpect.EOF) output_after_adding = child.before.decode('utf8').strip() if 'Already exists (68)' in output_after_adding: raise ValueError('Tried to create duplicate entry.') elif 'No such object (32)' in output_after_adding: raise ValueError('Tried to modify nonexistent entry.') if output_after_adding != '': send_problem_report( dedent('''\ Unknown problem occured when trying to write to LDAP; the code should be updated to handle this case. dn: {dn} keytab: {keytab} principal: {principal} Unexpected output: {output_after_adding} Lines passed to ldapadd: {lines} ''').format(dn=dn, keytab=keytab, principal=admin_principal, output_after_adding=output_after_adding, lines='\n'.join(' ' + line for line in lines))) raise ValueError('Unknown LDAP failure was encountered.')
def process_exception(self, request, exception): if settings.DEBUG: return if isinstance(exception, Http404): # we don't care about reporting 404 errors return try: send_problem_report(dedent( """\ An exception occured in ocfweb: {traceback} Request: * Host: {host} * Path: {path} * Method: {request.method} * Secure: {is_secure} Request Headers: {headers} Session: {session} """ ).format( traceback=format_exc(), request=request, host=request.get_host(), path=request.get_full_path(), is_secure=request.is_secure(), session=pformat(dict(request.session)), headers=pformat(request.META), )) except Exception as ex: print(ex) # just in case it errors again here send_problem_report(dedent( """\ An exception occured in ocfweb, but we errored trying to report it: {traceback} """ ).format(traceback=format_exc())) raise
def error_report(request, new_request, response): print(bold(red('Error: Entered unexpected state.'))) print(bold('An email has been sent to OCF staff')) error_report = dedent("""\ Error encountered running approve! The request we submitted was: {request} The request we submitted after being flagged (if any) was: {new_request} The response we received was: {response} """).format(request=request, new_request=new_request, reponse=response) send_problem_report(error_report)
def process_exception(self, request, exception): if settings.DEBUG: return try: send_problem_report(dedent( """\ An exception occured in ocfweb: {traceback} Request: * Host: {host} * Path: {path} * Method: {request.method} * Secure: {is_secure} Request Headers: {headers} Session: {session} """ ).format( traceback=format_exc(), request=request, host=request.get_host(), path=request.get_full_path(), is_secure=request.is_secure(), session=pformat(dict(request.session)), headers=pformat(request.META), )) except Exception as ex: print(ex) # just in case it errors again here send_problem_report(dedent( """\ An exception occured in ocfweb, but we errored trying to report it: {traceback} """ ).format(traceback=format_exc())) raise
def failure_handler(exc, task_id, args, kwargs, einfo): """Handle errors in Celery tasks by reporting via ocflib. We want to report actual errors, not just validation errors. Unfortunately it's hard to pick them out. For now, we just ignore ValueErrors and report everything else. It's likely that we'll need to revisit that some time in the future. """ if isinstance(exc, ValueError): return try: send_problem_report(dedent( """\ An exception occured in create: {traceback} Task Details: * task_id: {task_id} Try `journalctl -u ocf-create` for more details.""" ).format( traceback=einfo, task_id=task_id, args=args, kwargs=kwargs, einfo=einfo, )) except Exception as ex: print(ex) # just in case it errors again here send_problem_report(dedent( """\ An exception occured in create, but we errored trying to report it: {traceback} """ ).format(traceback=format_exc())) raise
def timer(bot): last_date = None last_dsa_check = None while not bot.connection.connected: time.sleep(2) # TODO: timers should register as plugins like listeners do while True: try: last_date, old = date.today(), last_date if old and last_date != old: bot.bump_topic() if last_dsa_check is None or time.time( ) - last_dsa_check > 60 * DSA_FREQ: last_dsa_check = time.time() for line in debian_security.get_new_dsas(): bot.say('#rebuild', line) except Exception: error_msg = f'ircbot exception: {format_exc()}' bot.say('#rebuild', error_msg) # don't send emails when running as dev if not TESTING: send_problem_report( dedent(""" {error} {traceback} """).format( error=error_msg, traceback=format_exc(), ), ) time.sleep(1)
def error_report(request, new_request, response): print(bold(red('Error: Entered unexpected state.'))) print(bold('An email has been sent to OCF staff')) error_report = dedent( """\ Error encountered running approve! The request we submitted was: {request} The request we submitted after being flagged (if any) was: {new_request} The response we received was: {response} """ ).format( request=request, new_request=new_request, reponse=response ) send_problem_report(error_report)
def create_new_vhost(request): try: conn = rt_connection(credentials['rt'].username, credentials['rt'].password) ticket_number = RtTicket.create( conn, 'hostmaster', request.requestor, request.subject, request.message, ) pr_new_vhost( credentials['github'], request.username, request.aliases, request.docroot, request.flags, ticket_number, ) return True except Exception as e: send_problem_report(str(e)) return False
def create_ldap_entry_with_keytab( dn, attributes, keytab, admin_principal, ): """Creates an LDAP entry by shelling out to ldapadd. :param dn: distinguished name of the new entry :param attributes: dict mapping attribute name to list of values :param keytab: path to the admin keytab :param admin_principal: admin principal to use with the keytab :return: the password of the newly-created account """ # LDAP attributes can have multiple values, but commonly we don't consider # that. So, let's sanity check the types we've received. for v in attributes.values(): assert type(v) in (list, tuple), 'Value must be list or tuple.' def format_attr(key, values): # might be possible to have non-ASCII letters in keys, but don't think # it will happen to us. we can fix this if it ever does. assert all(c in ascii_letters for c in key), 'Key is ASCII letters.' # rather than try to carefully escape values, we just base64 encode return ( '{key}:: {value}'.format( key=key, value=b64encode(value.encode('utf8')).decode('ascii'), ) for value in values ) lines = list(chain( format_attr('dn', [dn]), *(format_attr(key, values) for key, values in attributes.items()) )) cmd = 'kinit -t {keytab} {principal} ldapadd'.format( keytab=keytab, principal=admin_principal, ) child = pexpect.spawn(cmd, timeout=10) child.expect('SASL data security layer installed.') for line in lines: child.sendline(line) child.sendeof() child.expect('adding new entry "{dn}"'.format(dn=dn)) child.expect(pexpect.EOF) output_after_adding = child.before.decode('utf8').strip() if 'Already exists (68)' in output_after_adding: raise ValueError('DN already exists, this is a duplicate.') if output_after_adding != '': send_problem_report( dedent( '''\ Unknown problem occured when trying to add LDAP entry; the code should be updated to handle this case. dn: {dn} keytab: {keytab} principal: {principal} Unexpected output: {output_after_adding} Lines passed to ldapadd: {lines} ''' ).format( dn=dn, keytab=keytab, principal=admin_principal, output_after_adding=output_after_adding, lines='\n'.join(' ' + line for line in lines) ) ) raise ValueError('Unknown LDAP failure was encountered.')
def _write_ldif(lines, dn, keytab=None, admin_principal=None): """Issue an update to LDAP via ldapmodify in the form of lines of an LDIF file. This could be a new addition to LDAP, a modification of an existing item, or even a deletion depending on the changetype attribute given as part of the sequence of lines. :param lines: ldif file as a sequence of lines A ldif file looks something like this: dn: uid=jvperrin,ou=People,dc=OCF,dc=Berkeley,dc=EDU changetype: modify replace: loginShell loginShell: /bin/zsh It specifies the record or records to change, the type of change, and the changes to make. To handle special characters (e.g. anything unprintable) we base64-encode the dn and the values we set to get something more like this (note the two colons instead of one to designate base64 data): dn:: dWlkPWp2cGVycmluLG91PVBlb3BsZSxkYz1PQ0YsZGM9QmVya2VsZXksZGM9RURV changetype: modify replace: loginShell loginShell:: L2Jpbi96c2g= """ # Authenticate if these options are given. Otherwise, assume that # authentication has already been done and that a valid kerberos ticket # for the current user already exists if keytab and admin_principal: command = ('/usr/bin/kinit', '-t', keytab, admin_principal, '/usr/bin/ldapmodify', '-Q') else: command = ('/usr/bin/ldapmodify', '-Q') try: subprocess.check_output( command, input='\n'.join(lines), universal_newlines=True, timeout=10, ) except subprocess.CalledProcessError as e: if e.returncode == 32: raise ValueError('Tried to modify nonexistent entry.') elif e.returncode == 68: raise ValueError('Tried to create duplicate entry.') else: send_problem_report( dedent('''\ Unknown problem occured when trying to write to LDAP; the code should be updated to handle this case. dn: {dn} keytab: {keytab} principal: {principal} Error code: {returncode} Unexpected output: {output} Lines passed to ldapmodify: {lines} ''').format(dn=dn, keytab=keytab, principal=admin_principal, returncode=e.returncode, output=e.output, lines='\n'.join(' ' + line for line in lines))) raise ValueError('Unknown LDAP failure was encountered.')
def on_pubmsg(self, conn, event): if event.target in self.channels: is_oper = False # event.source is like '[email protected]' assert event.source.count('!') == 1 user = NickMask(event.source).nick # Don't respond to other create bots to avoid loops if user.startswith('create'): return if user in self.channels[event.target].opers(): is_oper = True assert len(event.arguments) == 1 raw_text = event.arguments[0] def respond(raw_text, ping=True): fmt = '{user}: {raw_text}' if ping else '{raw_text}' full_raw_text = fmt.format(user=user, raw_text=raw_text) self.say(event.target, full_raw_text) was_mentioned = raw_text.lower().startswith( (IRC_NICKNAME.lower() + ' ', IRC_NICKNAME.lower() + ': ')) for listener in self.listeners: text = raw_text if listener.require_mention: if was_mentioned: # Chop off the bot nickname. text = text.split(' ', 1)[1] else: continue if ((listener.require_oper or listener.require_privileged_oper) and not is_oper): continue # Prevent people from creating a channel, becoming oper, # inviting the bot, and approving/rejecting accounts without # "real" oper privilege. if listener.require_privileged_oper and event.target not in IRC_CHANNELS_OPER: continue match = listener.pattern.search(text) if match is not None: msg = MatchedMessage( channel=event.target, text=text, raw_text=raw_text, match=match, is_oper=is_oper, nick=user, respond=respond, ) try: listener.fn(self, msg) except Exception as ex: error_msg = 'ircbot exception in {module}/{function}: {exception}'.format( module=listener.fn.__module__, function=listener.fn.__name__, exception=ex, ) msg.respond(error_msg, ping=False) # don't send emails when running as dev if not TESTING: send_problem_report( dedent(""" {error} {traceback} Message: * Channel: {channel} * Nick: {nick} * Oper?: {oper} * Text: {text} * Raw text: {raw_text} * Match groups: {groups} """).format( error=error_msg, traceback=format_exc(), channel=msg.channel, nick=msg.nick, oper=msg.is_oper, text=msg.text, raw_text=msg.raw_text, groups=msg.match.groups(), )) # everything gets logged except commands if raw_text[0] != '!': self.recent_messages[event.target].appendleft((user, raw_text))
def create_ldap_entry_with_keytab( dn, attributes, keytab, admin_principal, ): """Creates an LDAP entry by shelling out to ldapadd. :param dn: distinguished name of the new entry :param attributes: dict mapping attribute name to list of values :param keytab: path to the admin keytab :param admin_principal: admin principal to use with the keytab :return: the password of the newly-created account """ # LDAP attributes can have multiple values, but commonly we don't consider # that. So, let's sanity check the types we've received. for v in attributes.values(): assert type(v) in (list, tuple), 'Value must be list or tuple.' def format_attr(key, values): # might be possible to have non-ASCII letters in keys, but don't think # it will happen to us. we can fix this if it ever does. assert all(c in ascii_letters for c in key), 'Key is ASCII letters.' # rather than try to carefully escape values, we just base64 encode return ('{key}:: {value}'.format( key=key, value=b64encode(value.encode('utf8')).decode('ascii'), ) for value in values) lines = list( chain( format_attr('dn', [dn]), *(format_attr(key, values) for key, values in attributes.items()))) cmd = 'kinit -t {keytab} {principal} ldapadd'.format( keytab=keytab, principal=admin_principal, ) child = pexpect.spawn(cmd, timeout=10) child.expect('SASL data security layer installed.') for line in lines: child.sendline(line) child.sendeof() child.expect('adding new entry "{dn}"'.format(dn=dn)) child.expect(pexpect.EOF) output_after_adding = child.before.decode('utf8').strip() if 'Already exists (68)' in output_after_adding: raise ValueError('DN already exists, this is a duplicate.') if output_after_adding != '': send_problem_report( dedent('''\ Unknown problem occured when trying to add LDAP entry; the code should be updated to handle this case. dn: {dn} keytab: {keytab} principal: {principal} Unexpected output: {output_after_adding} Lines passed to ldapadd: {lines} ''').format(dn=dn, keytab=keytab, principal=admin_principal, output_after_adding=output_after_adding, lines='\n'.join(' ' + line for line in lines))) raise ValueError('Unknown LDAP failure was encountered.')
def run_periodic_functions() -> None: global delay_on_error # First, import urls so that views are imported, decorators are run, and # our periodic functions get registered. import ocfweb.urls # noqa was_error = False for pf in periodic_functions: if pf.seconds_since_last_update() >= pf.period: _logger.info(bold(green(f'Updating periodic function: {pf}'))) try: pf.update() except Exception as ex: was_error = True if isinstance(ex, KeyboardInterrupt) or settings.DEBUG: raise try: send_problem_report( dedent("""\ An exception occurred in an ocfweb periodic function: {traceback} Periodic function: * Key: {pf.function_call_key} * Last Update: {last_update} ({seconds_since_last_update} seconds ago) * Period: {pf.period} * TTL: {pf.ttl} The background process will now pause for {delay} seconds. """).format( traceback=format_exc(), pf=pf, last_update=pf.last_update(), seconds_since_last_update=pf. seconds_since_last_update(), delay=delay_on_error, ), ) _logger.error(format_exc()) except Exception as ex: print(ex) # just in case it errors again here send_problem_report( dedent("""\ An exception occured in ocfweb, but we errored trying to report it: {traceback} """).format(traceback=format_exc()), ) raise else: _logger.debug(bold( yellow(f'Not updating periodic function: {pf}'))) if was_error: delay_on_error = min(DELAY_ON_ERROR_MAX, delay_on_error * 2) time.sleep(delay_on_error) else: delay_on_error = max(DELAY_ON_ERROR_MIN, delay_on_error / 2)
def run_periodic_functions(): global delay_on_error # First, import urls so that views are imported, decorators are run, and # our periodic functions get registered. import ocfweb.urls # noqa was_error = False for pf in periodic_functions: if pf.seconds_since_last_update() >= pf.period: _logger.info(bold(green('Updating periodic function: {}'.format(pf)))) try: pf.update() except Exception as ex: was_error = True if isinstance(ex, KeyboardInterrupt) or settings.DEBUG: raise try: send_problem_report(dedent( """\ An exception occurred in an ocfweb periodic function: {traceback} Periodic function: * Key: {pf.function_call_key} * Last Update: {last_update} ({seconds_since_last_update} seconds ago) * Period: {pf.period} * TTL: {pf.ttl} The background process will now pause for {delay} seconds. """ ).format( traceback=format_exc(), pf=pf, last_update=pf.last_update(), seconds_since_last_update=pf.seconds_since_last_update(), delay=delay_on_error, )) _logger.error(format_exc()) except Exception as ex: print(ex) # just in case it errors again here send_problem_report(dedent( """\ An exception occured in ocfweb, but we errored trying to report it: {traceback} """ ).format(traceback=format_exc())) raise else: _logger.debug(bold(yellow('Not updating periodic function: {}'.format(pf)))) if was_error: delay_on_error = min(DELAY_ON_ERROR_MAX, delay_on_error * 2) time.sleep(delay_on_error) else: delay_on_error = max(DELAY_ON_ERROR_MIN, delay_on_error / 2)