def test_E_CallsOnException(self): fail_counter = [] on_exception_counter = [] class TestException(Exception): pass def failer(): fail_counter.append(1) raise TestException() def on_exception(e, delay): print e if delay is not None: print delay.total_seconds() on_exception_counter.append(1) numDelays = 4 delays = [datetime.timedelta() for _ in range(0, numDelays)] try: phlsys_tryloop.try_loop_delay( failer, delays, onException=on_exception) except TestException: pass else: raise Exception("did not receive TestException") self.assertEqual(1 + numDelays, len(fail_counter)) self.assertEqual(len(fail_counter), len(on_exception_counter))
def test_E_CallsOnException(self): fail_counter = [] on_exception_counter = [] class TestException(Exception): pass def failer(): fail_counter.append(1) raise TestException() def on_exception(e, delay): print(e) if delay is not None: print(delay.total_seconds()) on_exception_counter.append(1) numDelays = 4 delays = [datetime.timedelta() for _ in range(0, numDelays)] try: phlsys_tryloop.try_loop_delay(failer, delays, onException=on_exception) except TestException: pass else: raise Exception("did not receive TestException") self.assertEqual(1 + numDelays, len(fail_counter)) self.assertEqual(len(fail_counter), len(on_exception_counter))
def test_D_RaiseThrough(self): class TestException(Exception): pass counter = [] def failer(): counter.append(1) raise TypeError() numDelays = 4 delays = [datetime.timedelta() for _ in range(0, numDelays)] try: phlsys_tryloop.try_loop_delay(failer, delays, TestException) except TypeError: pass else: raise Exception("did not receive TypeError") self.assertEqual(1, len(counter))
def test_C_RetriesEachDelay(self): class TestException(Exception): pass counter = [] def failer(): counter.append(1) raise TestException() numDelays = 4 delays = [datetime.timedelta() for _ in range(0, numDelays)] try: phlsys_tryloop.try_loop_delay(failer, delays, TestException) except TestException: pass else: raise Exception("did not receive TestException") self.assertEqual(1 + numDelays, len(counter))
def tryloop(f, identifier, detail): """Return the result of the supplied operation 'f', retry on exception. Will retry operation 'f' after a short delay if it raises, gives up after a few attempts. :f: a callable that can be invoked like so 'f()' :identifier: a short string identifier for the calling line of code :detail: a string of the variables associated with the call, e.g. repo name :returns: the return value of operation 'f', if successful """ def on_tryloop_exception(e, delay): abdt_logging.on_retry_exception(identifier, detail, e, delay) delays = phlsys_tryloop.make_default_short_retry() return phlsys_tryloop.try_loop_delay( f, delays, onException=on_tryloop_exception)
def critical_tryloop(f, identifier, detail): """Return the result of the supplied operation 'f', retry on exception. Will retry operation 'f' indefinitely until it does not raise, the delays used gradually increase so that whichever system is being tried is not overly stressed. :f: a callable that can be invoked like so 'f()' :identifier: a short string identifier for the calling line of code :detail: a string of the variables associated with the call, e.g. repo name :returns: the return value of operation 'f', if successful """ def on_tryloop_exception(e, delay): abdt_logging.on_retry_exception(identifier, detail, e, delay) delays = phlsys_tryloop.make_default_endless_retry() return phlsys_tryloop.try_loop_delay( f, delays, onException=on_tryloop_exception)
def test_B_ReturnsResult(self): self.assertEqual(1, phlsys_tryloop.try_loop_delay(lambda: 1, [])) self.assertEqual("hi", phlsys_tryloop.try_loop_delay(lambda: "hi", []))
def run_once(args, out): sender = phlmail_sender.MailSender( phlsys_sendmail.Sendmail(), args.arcyd_email) mailer = abdmail_mailer.Mailer( sender, [args.admin_email], args.repo_desc, args.instance_uri) # TODO: this should be a URI for users not conduit # prepare delays in the event of trouble when fetching or connecting # TODO: perhaps this policy should be decided higher-up delays = [ datetime.timedelta(seconds=1), datetime.timedelta(seconds=1), datetime.timedelta(seconds=10), datetime.timedelta(seconds=10), datetime.timedelta(seconds=100), datetime.timedelta(seconds=100), datetime.timedelta(seconds=1000), ] # log.error if we get an exception when fetching def on_exception(e, delay): logging.error(str(e) + "\nwill wait " + str(delay)) if args.try_touch_path: try: # TODO: don't rely on the touch command phlsys_subprocess.run("touch", args.try_touch_path) except Exception: pass # XXX: we don't care atm, later log this with phlsys_fs.chdir_context(args.repo_path): out.display("fetch (" + args.repo_desc + "): ") phlsys_tryloop.try_loop_delay( lambda: phlsys_subprocess.run_commands("git fetch -p"), delays, onException=on_exception) # XXX: until conduit refreshes the connection, we'll suffer from # timeouts; reduce the probability of this by using a new # conduit each time. # create an array so that the 'connect' closure binds to the 'conduit' # variable as we'd expect, otherwise it'll just modify a local variable # and this 'conduit' will remain 'None' # XXX: we can do better in python 3.x conduit = [None] def connect(): #nonlocal conduit # XXX: we'll rebind in python 3.x, instead of array conduit[0] = phlsys_conduit.Conduit( args.instance_uri, args.arcyd_user, args.arcyd_cert, https_proxy=args.https_proxy) phlsys_tryloop.try_loop_delay(connect, delays, onException=on_exception) out.display("process (" + args.repo_desc + "): ") abdi_processrepo.processUpdatedRepo( conduit[0], args.repo_path, "origin", mailer) if args.ok_touch_path: try: # TODO: don't rely on the touch command phlsys_subprocess.run("touch", args.ok_touch_path) except Exception: pass # XXX: we don't care atm, later log this
def _run_once(args, out, reporter): sender = phlmail_sender.MailSender( phlsys_sendmail.Sendmail(), args.arcyd_email) mailer = abdmail_mailer.Mailer( sender, [args.admin_email], args.repo_desc, args.instance_uri) # TODO: this should be a URI for users not conduit pluginManager = phlsys_pluginmanager.PluginManager( args.plugins, args.trusted_plugins) # prepare delays in the event of trouble when fetching or connecting # TODO: perhaps this policy should be decided higher-up delays = [ datetime.timedelta(seconds=1), datetime.timedelta(seconds=1), datetime.timedelta(seconds=10), datetime.timedelta(seconds=10), datetime.timedelta(seconds=100), datetime.timedelta(seconds=100), datetime.timedelta(seconds=1000), ] # log.error if we get an exception when fetching def on_tryloop_exception(e, delay): reporter.on_tryloop_exception(e, delay) logging.error(str(e) + "\nwill wait " + str(delay)) def prune_and_fetch(): phlsys_subprocess.run_commands("git remote prune origin") phlsys_subprocess.run_commands("git fetch") with phlsys_fs.chdir_context(args.repo_path): out.display("fetch (" + args.repo_desc + "): ") phlsys_tryloop.try_loop_delay( prune_and_fetch, delays, onException=on_tryloop_exception) # XXX: until conduit refreshes the connection, we'll suffer from # timeouts; reduce the probability of this by using a new # conduit each time. # create an array so that the 'connect' closure binds to the 'conduit' # variable as we'd expect, otherwise it'll just modify a local variable # and this 'conduit' will remain 'None' # XXX: we can do better in python 3.x (nonlocal?) conduit = [None] def connect(): # nonlocal conduit # XXX: we'll rebind in python 3.x, instead of array conduit[0] = phlsys_conduit.Conduit( args.instance_uri, args.arcyd_user, args.arcyd_cert, https_proxy=args.https_proxy) phlsys_tryloop.try_loop_delay( connect, delays, onException=on_tryloop_exception) out.display("process (" + args.repo_desc + "): ") arcyd_conduit = abdt_conduit.Conduit(conduit[0]) branch_url_callable = None if args.branch_url_format: def make_branch_url(branch_name): return args.branch_url_format.format(branch=branch_name) branch_url_callable = make_branch_url arcyd_clone = abdt_git.Clone( args.repo_path, "origin", args.repo_desc, branch_url_callable) branches = arcyd_clone.get_managed_branches() try: abdi_processrepo.process_branches( branches, arcyd_conduit, mailer, pluginManager, reporter) except Exception: reporter.on_traceback(traceback.format_exc()) raise reporter.on_completed()