def handle_error(self, error_message):
# for debugging purposes
print(error_message)
# don't send emails when running as dev
if not TESTING:
send_problem_report(error_message)
def _get_first_available_uid():
"""Return the first available UID number.
Searches our entire People ou in order to find it. It seems like there
should be a better way to do this, but quick searches don't show any.
We hard-code a value we know has already been reached and only select
entries greater than that for performance. We then use a module-level
dict to cache our output for the next function call.
"""
min_uid = _cache['known_uid']
with ldap_ocf() as c:
c.search(
constants.OCF_LDAP_PEOPLE,
'(uidNumber>={KNOWN_MIN})'.format(KNOWN_MIN=min_uid),
attributes=['uidNumber'],
)
uids = [int(entry['attributes']['uidNumber'][0]) for entry in c.response]
if len(uids) > 2500:
send_problem_report((
'Found {} accounts with UID >= {}, '
'you should bump the constant for speed.'
).format(len(uids), min_uid))
if uids:
max_uid = max(uids)
_cache['known_uid'] = max_uid
else:
# If cached UID is later deleted, LDAP response will be empty.
max_uid = min_uid
return max_uid + 1
def sync(self):
try:
for ldap_group, dest_group in self.SYNC_PAIRS:
ldap_members = set(list_staff(group=ldap_group))
dest_members = set(
self.dest_service().list_members(dest_group))
to_add = ldap_members - dest_members
missing = dest_members - ldap_members
if missing:
missing_header = '''The following users are in the {dest}
destination group but are not in the {ldap}
LDAP group:'''.format(dest=dest_group,
ldap=ldap_group)
self.logger.warning(missing_header)
for m in missing:
self.logger.warning(m)
for username in to_add:
if not self.args.dry_run:
self.dest_service().add_to_group(username, dest_group)
self.logger.info('Adding {} to group {}'.format(
username, dest_group))
except Exception as e:
self.logger.exception('Exception caught: {}'.format(e))
if not self.args.dry_run:
mail.send_problem_report(
'An exception occurred in ldapsync: \n\n{}'.format(e))
def _get_first_available_uid():
"""Return the first available UID number.
Searches our entire People ou in order to find it. It seems like there
should be a better way to do this, but quick searches don't show any.
We hard-code a value we know has already been reached and only select
entries greater than that for performance. We then use a module-level
dict to cache our output for the next function call.
"""
min_uid = _cache['known_uid']
with ldap_ocf() as c:
c.search(
constants.OCF_LDAP_PEOPLE,
'(uidNumber>={KNOWN_MIN})'.format(KNOWN_MIN=min_uid),
attributes=['uidNumber'],
)
uids = [
int(entry['attributes']['uidNumber'][0]) for entry in c.response
]
if len(uids) > 2500:
send_problem_report(
('Found {} accounts with UID >= {}, '
'you should bump the constant for speed.').format(
len(uids), min_uid))
if uids:
max_uid = max(uids)
_cache['known_uid'] = max_uid
else:
# If cached UID is later deleted, LDAP response will be empty.
max_uid = min_uid
return max_uid + 1
def handle_loop_error(err: Failure, loop_handler: LoopHandler) -> None:
"""Handle errors in a looping function and restart the loop."""
send_problem_report(err)
err.printTraceback()
# Sleep to avoid tight infinite loops spamming emails
time.sleep(3)
# Restart the given method
loop_handler.start_loop()
def test_problem_report(self, mock_popen):
send_problem_report('hellllo world')
msg = self.get_message(mock_popen)
assert msg['Subject'].startswith('[ocflib] Problem report')
assert msg['From'] == MAIL_FROM
assert msg['To'] == MAIL_ROOT
assert 'hellllo world' in msg.get_payload()
def process_exception(self, request: HttpRequest,
exception: Exception) -> Any:
if isinstance(exception, ResponseException):
return exception.response
# maybe it's a real exception?
if settings.DEBUG or settings.TESTING:
return
if isinstance(exception, Http404):
# we don't care about reporting 404 errors
return
traceback = sanitize(format_exc())
headers = sanitize_wsgi_context(request.META)
try:
send_problem_report(
dedent(
"""\
An exception occured in ocfweb:
{traceback}
Request:
* Host: {host}
* Path: {path}
* Method: {request.method}
* Secure: {is_secure}
Request Headers:
{headers}
Session:
{session}
""", ).format(
traceback=traceback,
request=request,
host=request.get_host(),
path=request.get_full_path(),
is_secure=request.is_secure(),
session=pformat(dict(request.session)),
headers=pformat(headers),
), )
except Exception as ex:
print(ex) # just in case it errors again here
send_problem_report(
dedent(
"""\
An exception occured in ocfweb, but we errored trying to report it:
{traceback}
""", ).format(traceback=format_exc()), )
raise
def process_exception(self, request, exception):
if isinstance(exception, ResponseException):
return exception.response
# maybe it's a real exception?
if settings.DEBUG or settings.TESTING:
return
if isinstance(exception, Http404):
# we don't care about reporting 404 errors
return
try:
send_problem_report(
dedent(
"""\
An exception occured in ocfweb:
{traceback}
Request:
* Host: {host}
* Path: {path}
* Method: {request.method}
* Secure: {is_secure}
Request Headers:
{headers}
Session:
{session}
"""
).format(
traceback=format_exc(),
request=request,
host=request.get_host(),
path=request.get_full_path(),
is_secure=request.is_secure(),
session=pformat(dict(request.session)),
headers=pformat(request.META),
)
)
except Exception as ex:
print(ex) # just in case it errors again here
send_problem_report(
dedent(
"""\
An exception occured in ocfweb, but we errored trying to report it:
{traceback}
"""
).format(traceback=format_exc())
)
raise
def _write_ldif(lines, dn, keytab, admin_principal):
"""Issue an update to LDAP via ldapmodify in the form of lines of an LDIF
file.
:param lines: ldif file as a sequence of lines
"""
cmd = 'kinit -t {keytab} {principal} ldapmodify'.format(
keytab=keytab,
principal=admin_principal,
)
child = pexpect.spawn(cmd, timeout=10)
child.expect('SASL data security layer installed.')
for line in lines:
child.sendline(line)
child.sendeof()
child.expect('entry "{}"'.format(dn))
child.expect(pexpect.EOF)
output_after_adding = child.before.decode('utf8').strip()
if 'Already exists (68)' in output_after_adding:
raise ValueError('Tried to create duplicate entry.')
elif 'No such object (32)' in output_after_adding:
raise ValueError('Tried to modify nonexistent entry.')
if output_after_adding != '':
send_problem_report(
dedent(
'''\
Unknown problem occured when trying to write to LDAP; the code
should be updated to handle this case.
dn: {dn}
keytab: {keytab}
principal: {principal}
Unexpected output:
{output_after_adding}
Lines passed to ldapadd:
{lines}
'''
).format(
dn=dn,
keytab=keytab,
principal=admin_principal,
output_after_adding=output_after_adding,
lines='\n'.join(' ' + line for line in lines)
)
)
raise ValueError('Unknown LDAP failure was encountered.')
def _write_ldif(lines, dn, keytab, admin_principal):
"""Issue an update to LDAP via ldapmodify in the form of lines of an LDIF
file.
:param lines: ldif file as a sequence of lines
"""
cmd = 'kinit -t {keytab} {principal} ldapmodify'.format(
keytab=keytab,
principal=admin_principal,
)
child = pexpect.spawn(cmd, timeout=10)
child.expect('SASL data security layer installed.')
for line in lines:
child.sendline(line)
child.sendeof()
child.expect('entry "{}"'.format(dn))
child.expect(pexpect.EOF)
output_after_adding = child.before.decode('utf8').strip()
if 'Already exists (68)' in output_after_adding:
raise ValueError('Tried to create duplicate entry.')
elif 'No such object (32)' in output_after_adding:
raise ValueError('Tried to modify nonexistent entry.')
if output_after_adding != '':
send_problem_report(
dedent('''\
Unknown problem occured when trying to write to LDAP; the code
should be updated to handle this case.
dn: {dn}
keytab: {keytab}
principal: {principal}
Unexpected output:
{output_after_adding}
Lines passed to ldapadd:
{lines}
''').format(dn=dn,
keytab=keytab,
principal=admin_principal,
output_after_adding=output_after_adding,
lines='\n'.join(' ' + line for line in lines)))
raise ValueError('Unknown LDAP failure was encountered.')
def process_exception(self, request, exception):
if settings.DEBUG:
return
if isinstance(exception, Http404):
# we don't care about reporting 404 errors
return
try:
send_problem_report(dedent(
"""\
An exception occured in ocfweb:
{traceback}
Request:
* Host: {host}
* Path: {path}
* Method: {request.method}
* Secure: {is_secure}
Request Headers:
{headers}
Session:
{session}
"""
).format(
traceback=format_exc(),
request=request,
host=request.get_host(),
path=request.get_full_path(),
is_secure=request.is_secure(),
session=pformat(dict(request.session)),
headers=pformat(request.META),
))
except Exception as ex:
print(ex) # just in case it errors again here
send_problem_report(dedent(
"""\
An exception occured in ocfweb, but we errored trying to report it:
{traceback}
"""
).format(traceback=format_exc()))
raise
def error_report(request, new_request, response):
print(bold(red('Error: Entered unexpected state.')))
print(bold('An email has been sent to OCF staff'))
error_report = dedent("""\
Error encountered running approve!
The request we submitted was:
{request}
The request we submitted after being flagged (if any) was:
{new_request}
The response we received was:
{response}
""").format(request=request, new_request=new_request, reponse=response)
send_problem_report(error_report)
def process_exception(self, request, exception):
if settings.DEBUG:
return
try:
send_problem_report(dedent(
"""\
An exception occured in ocfweb:
{traceback}
Request:
* Host: {host}
* Path: {path}
* Method: {request.method}
* Secure: {is_secure}
Request Headers:
{headers}
Session:
{session}
"""
).format(
traceback=format_exc(),
request=request,
host=request.get_host(),
path=request.get_full_path(),
is_secure=request.is_secure(),
session=pformat(dict(request.session)),
headers=pformat(request.META),
))
except Exception as ex:
print(ex) # just in case it errors again here
send_problem_report(dedent(
"""\
An exception occured in ocfweb, but we errored trying to report it:
{traceback}
"""
).format(traceback=format_exc()))
raise
def failure_handler(exc, task_id, args, kwargs, einfo):
"""Handle errors in Celery tasks by reporting via ocflib.
We want to report actual errors, not just validation errors. Unfortunately
it's hard to pick them out. For now, we just ignore ValueErrors and report
everything else.
It's likely that we'll need to revisit that some time in the future.
"""
if isinstance(exc, ValueError):
return
try:
send_problem_report(dedent(
"""\
An exception occured in create:
{traceback}
Task Details:
* task_id: {task_id}
Try `journalctl -u ocf-create` for more details."""
).format(
traceback=einfo,
task_id=task_id,
args=args,
kwargs=kwargs,
einfo=einfo,
))
except Exception as ex:
print(ex) # just in case it errors again here
send_problem_report(dedent(
"""\
An exception occured in create, but we errored trying to report it:
{traceback}
"""
).format(traceback=format_exc()))
raise
def failure_handler(exc, task_id, args, kwargs, einfo):
"""Handle errors in Celery tasks by reporting via ocflib.
We want to report actual errors, not just validation errors. Unfortunately
it's hard to pick them out. For now, we just ignore ValueErrors and report
everything else.
It's likely that we'll need to revisit that some time in the future.
"""
if isinstance(exc, ValueError):
return
try:
send_problem_report(dedent(
"""\
An exception occured in create:
{traceback}
Task Details:
* task_id: {task_id}
Try `journalctl -u ocf-create` for more details."""
).format(
traceback=einfo,
task_id=task_id,
args=args,
kwargs=kwargs,
einfo=einfo,
))
except Exception as ex:
print(ex) # just in case it errors again here
send_problem_report(dedent(
"""\
An exception occured in create, but we errored trying to report it:
{traceback}
"""
).format(traceback=format_exc()))
raise
def timer(bot):
last_date = None
last_dsa_check = None
while not bot.connection.connected:
time.sleep(2)
# TODO: timers should register as plugins like listeners do
while True:
try:
last_date, old = date.today(), last_date
if old and last_date != old:
bot.bump_topic()
if last_dsa_check is None or time.time(
) - last_dsa_check > 60 * DSA_FREQ:
last_dsa_check = time.time()
for line in debian_security.get_new_dsas():
bot.say('#rebuild', line)
except Exception:
error_msg = f'ircbot exception: {format_exc()}'
bot.say('#rebuild', error_msg)
# don't send emails when running as dev
if not TESTING:
send_problem_report(
dedent("""
{error}
{traceback}
""").format(
error=error_msg,
traceback=format_exc(),
), )
time.sleep(1)
def error_report(request, new_request, response):
print(bold(red('Error: Entered unexpected state.')))
print(bold('An email has been sent to OCF staff'))
error_report = dedent(
"""\
Error encountered running approve!
The request we submitted was:
{request}
The request we submitted after being flagged (if any) was:
{new_request}
The response we received was:
{response}
"""
).format(
request=request,
new_request=new_request,
reponse=response
)
send_problem_report(error_report)
def create_new_vhost(request):
try:
conn = rt_connection(credentials['rt'].username,
credentials['rt'].password)
ticket_number = RtTicket.create(
conn,
'hostmaster',
request.requestor,
request.subject,
request.message,
)
pr_new_vhost(
credentials['github'],
request.username,
request.aliases,
request.docroot,
request.flags,
ticket_number,
)
return True
except Exception as e:
send_problem_report(str(e))
return False
def create_ldap_entry_with_keytab(
dn,
attributes,
keytab,
admin_principal,
):
"""Creates an LDAP entry by shelling out to ldapadd.
:param dn: distinguished name of the new entry
:param attributes: dict mapping attribute name to list of values
:param keytab: path to the admin keytab
:param admin_principal: admin principal to use with the keytab
:return: the password of the newly-created account
"""
# LDAP attributes can have multiple values, but commonly we don't consider
# that. So, let's sanity check the types we've received.
for v in attributes.values():
assert type(v) in (list, tuple), 'Value must be list or tuple.'
def format_attr(key, values):
# might be possible to have non-ASCII letters in keys, but don't think
# it will happen to us. we can fix this if it ever does.
assert all(c in ascii_letters for c in key), 'Key is ASCII letters.'
# rather than try to carefully escape values, we just base64 encode
return (
'{key}:: {value}'.format(
key=key,
value=b64encode(value.encode('utf8')).decode('ascii'),
) for value in values
)
lines = list(chain(
format_attr('dn', [dn]),
*(format_attr(key, values) for key, values in attributes.items())
))
cmd = 'kinit -t {keytab} {principal} ldapadd'.format(
keytab=keytab,
principal=admin_principal,
)
child = pexpect.spawn(cmd, timeout=10)
child.expect('SASL data security layer installed.')
for line in lines:
child.sendline(line)
child.sendeof()
child.expect('adding new entry "{dn}"'.format(dn=dn))
child.expect(pexpect.EOF)
output_after_adding = child.before.decode('utf8').strip()
if 'Already exists (68)' in output_after_adding:
raise ValueError('DN already exists, this is a duplicate.')
if output_after_adding != '':
send_problem_report(
dedent(
'''\
Unknown problem occured when trying to add LDAP entry; the code
should be updated to handle this case.
dn: {dn}
keytab: {keytab}
principal: {principal}
Unexpected output:
{output_after_adding}
Lines passed to ldapadd:
{lines}
'''
).format(
dn=dn,
keytab=keytab,
principal=admin_principal,
output_after_adding=output_after_adding,
lines='\n'.join(' ' + line for line in lines)
)
)
raise ValueError('Unknown LDAP failure was encountered.')
def _write_ldif(lines, dn, keytab=None, admin_principal=None):
"""Issue an update to LDAP via ldapmodify in the form of lines of an LDIF
file. This could be a new addition to LDAP, a modification of an existing
item, or even a deletion depending on the changetype attribute given as
part of the sequence of lines.
:param lines: ldif file as a sequence of lines
A ldif file looks something like this:
dn: uid=jvperrin,ou=People,dc=OCF,dc=Berkeley,dc=EDU
changetype: modify
replace: loginShell
loginShell: /bin/zsh
It specifies the record or records to change, the type of change, and the
changes to make. To handle special characters (e.g. anything unprintable)
we base64-encode the dn and the values we set to get something more like
this (note the two colons instead of one to designate base64 data):
dn:: dWlkPWp2cGVycmluLG91PVBlb3BsZSxkYz1PQ0YsZGM9QmVya2VsZXksZGM9RURV
changetype: modify
replace: loginShell
loginShell:: L2Jpbi96c2g=
"""
# Authenticate if these options are given. Otherwise, assume that
# authentication has already been done and that a valid kerberos ticket
# for the current user already exists
if keytab and admin_principal:
command = ('/usr/bin/kinit', '-t', keytab, admin_principal,
'/usr/bin/ldapmodify', '-Q')
else:
command = ('/usr/bin/ldapmodify', '-Q')
try:
subprocess.check_output(
command,
input='\n'.join(lines),
universal_newlines=True,
timeout=10,
)
except subprocess.CalledProcessError as e:
if e.returncode == 32:
raise ValueError('Tried to modify nonexistent entry.')
elif e.returncode == 68:
raise ValueError('Tried to create duplicate entry.')
else:
send_problem_report(
dedent('''\
Unknown problem occured when trying to write to LDAP; the
code should be updated to handle this case.
dn: {dn}
keytab: {keytab}
principal: {principal}
Error code: {returncode}
Unexpected output:
{output}
Lines passed to ldapmodify:
{lines}
''').format(dn=dn,
keytab=keytab,
principal=admin_principal,
returncode=e.returncode,
output=e.output,
lines='\n'.join(' ' + line
for line in lines)))
raise ValueError('Unknown LDAP failure was encountered.')
def on_pubmsg(self, conn, event):
if event.target in self.channels:
is_oper = False
# event.source is like '[email protected]'
assert event.source.count('!') == 1
user = NickMask(event.source).nick
# Don't respond to other create bots to avoid loops
if user.startswith('create'):
return
if user in self.channels[event.target].opers():
is_oper = True
assert len(event.arguments) == 1
raw_text = event.arguments[0]
def respond(raw_text, ping=True):
fmt = '{user}: {raw_text}' if ping else '{raw_text}'
full_raw_text = fmt.format(user=user, raw_text=raw_text)
self.say(event.target, full_raw_text)
was_mentioned = raw_text.lower().startswith(
(IRC_NICKNAME.lower() + ' ', IRC_NICKNAME.lower() + ': '))
for listener in self.listeners:
text = raw_text
if listener.require_mention:
if was_mentioned:
# Chop off the bot nickname.
text = text.split(' ', 1)[1]
else:
continue
if ((listener.require_oper or listener.require_privileged_oper)
and not is_oper):
continue
# Prevent people from creating a channel, becoming oper,
# inviting the bot, and approving/rejecting accounts without
# "real" oper privilege.
if listener.require_privileged_oper and event.target not in IRC_CHANNELS_OPER:
continue
match = listener.pattern.search(text)
if match is not None:
msg = MatchedMessage(
channel=event.target,
text=text,
raw_text=raw_text,
match=match,
is_oper=is_oper,
nick=user,
respond=respond,
)
try:
listener.fn(self, msg)
except Exception as ex:
error_msg = 'ircbot exception in {module}/{function}: {exception}'.format(
module=listener.fn.__module__,
function=listener.fn.__name__,
exception=ex,
)
msg.respond(error_msg, ping=False)
# don't send emails when running as dev
if not TESTING:
send_problem_report(
dedent("""
{error}
{traceback}
Message:
* Channel: {channel}
* Nick: {nick}
* Oper?: {oper}
* Text: {text}
* Raw text: {raw_text}
* Match groups: {groups}
""").format(
error=error_msg,
traceback=format_exc(),
channel=msg.channel,
nick=msg.nick,
oper=msg.is_oper,
text=msg.text,
raw_text=msg.raw_text,
groups=msg.match.groups(),
))
# everything gets logged except commands
if raw_text[0] != '!':
self.recent_messages[event.target].appendleft((user, raw_text))
def create_ldap_entry_with_keytab(
dn,
attributes,
keytab,
admin_principal,
):
"""Creates an LDAP entry by shelling out to ldapadd.
:param dn: distinguished name of the new entry
:param attributes: dict mapping attribute name to list of values
:param keytab: path to the admin keytab
:param admin_principal: admin principal to use with the keytab
:return: the password of the newly-created account
"""
# LDAP attributes can have multiple values, but commonly we don't consider
# that. So, let's sanity check the types we've received.
for v in attributes.values():
assert type(v) in (list, tuple), 'Value must be list or tuple.'
def format_attr(key, values):
# might be possible to have non-ASCII letters in keys, but don't think
# it will happen to us. we can fix this if it ever does.
assert all(c in ascii_letters for c in key), 'Key is ASCII letters.'
# rather than try to carefully escape values, we just base64 encode
return ('{key}:: {value}'.format(
key=key,
value=b64encode(value.encode('utf8')).decode('ascii'),
) for value in values)
lines = list(
chain(
format_attr('dn', [dn]),
*(format_attr(key, values) for key, values in attributes.items())))
cmd = 'kinit -t {keytab} {principal} ldapadd'.format(
keytab=keytab,
principal=admin_principal,
)
child = pexpect.spawn(cmd, timeout=10)
child.expect('SASL data security layer installed.')
for line in lines:
child.sendline(line)
child.sendeof()
child.expect('adding new entry "{dn}"'.format(dn=dn))
child.expect(pexpect.EOF)
output_after_adding = child.before.decode('utf8').strip()
if 'Already exists (68)' in output_after_adding:
raise ValueError('DN already exists, this is a duplicate.')
if output_after_adding != '':
send_problem_report(
dedent('''\
Unknown problem occured when trying to add LDAP entry; the code
should be updated to handle this case.
dn: {dn}
keytab: {keytab}
principal: {principal}
Unexpected output:
{output_after_adding}
Lines passed to ldapadd:
{lines}
''').format(dn=dn,
keytab=keytab,
principal=admin_principal,
output_after_adding=output_after_adding,
lines='\n'.join(' ' + line for line in lines)))
raise ValueError('Unknown LDAP failure was encountered.')
def run_periodic_functions() -> None:
global delay_on_error
# First, import urls so that views are imported, decorators are run, and
# our periodic functions get registered.
import ocfweb.urls # noqa
was_error = False
for pf in periodic_functions:
if pf.seconds_since_last_update() >= pf.period:
_logger.info(bold(green(f'Updating periodic function: {pf}')))
try:
pf.update()
except Exception as ex:
was_error = True
if isinstance(ex, KeyboardInterrupt) or settings.DEBUG:
raise
try:
send_problem_report(
dedent("""\
An exception occurred in an ocfweb periodic function:
{traceback}
Periodic function:
* Key: {pf.function_call_key}
* Last Update: {last_update} ({seconds_since_last_update} seconds ago)
* Period: {pf.period}
* TTL: {pf.ttl}
The background process will now pause for {delay} seconds.
""").format(
traceback=format_exc(),
pf=pf,
last_update=pf.last_update(),
seconds_since_last_update=pf.
seconds_since_last_update(),
delay=delay_on_error,
), )
_logger.error(format_exc())
except Exception as ex:
print(ex) # just in case it errors again here
send_problem_report(
dedent("""\
An exception occured in ocfweb, but we errored trying to report it:
{traceback}
""").format(traceback=format_exc()), )
raise
else:
_logger.debug(bold(
yellow(f'Not updating periodic function: {pf}')))
if was_error:
delay_on_error = min(DELAY_ON_ERROR_MAX, delay_on_error * 2)
time.sleep(delay_on_error)
else:
delay_on_error = max(DELAY_ON_ERROR_MIN, delay_on_error / 2)
def run_periodic_functions():
global delay_on_error
# First, import urls so that views are imported, decorators are run, and
# our periodic functions get registered.
import ocfweb.urls # noqa
was_error = False
for pf in periodic_functions:
if pf.seconds_since_last_update() >= pf.period:
_logger.info(bold(green('Updating periodic function: {}'.format(pf))))
try:
pf.update()
except Exception as ex:
was_error = True
if isinstance(ex, KeyboardInterrupt) or settings.DEBUG:
raise
try:
send_problem_report(dedent(
"""\
An exception occurred in an ocfweb periodic function:
{traceback}
Periodic function:
* Key: {pf.function_call_key}
* Last Update: {last_update} ({seconds_since_last_update} seconds ago)
* Period: {pf.period}
* TTL: {pf.ttl}
The background process will now pause for {delay} seconds.
"""
).format(
traceback=format_exc(),
pf=pf,
last_update=pf.last_update(),
seconds_since_last_update=pf.seconds_since_last_update(),
delay=delay_on_error,
))
_logger.error(format_exc())
except Exception as ex:
print(ex) # just in case it errors again here
send_problem_report(dedent(
"""\
An exception occured in ocfweb, but we errored trying to report it:
{traceback}
"""
).format(traceback=format_exc()))
raise
else:
_logger.debug(bold(yellow('Not updating periodic function: {}'.format(pf))))
if was_error:
delay_on_error = min(DELAY_ON_ERROR_MAX, delay_on_error * 2)
time.sleep(delay_on_error)
else:
delay_on_error = max(DELAY_ON_ERROR_MIN, delay_on_error / 2)
