def test_sequence(self): """ The function generates a packer configuration file, runs packer build and uploads the AMI ids to a given S3 bucket. """ options = PublishInstallerImagesOptions() options.parseOptions( [b'--source-ami-map', b'{"us-west-1": "ami-1234"}'] ) configuration_path = self.make_temporary_directory() ami_map = PACKER_OUTPUT_US_ALL.output perform_sequence( seq=[ (PackerConfigure( build_region=options["build_region"], publish_regions=options["regions"], source_ami_map=options["source-ami-map"], template=options["template"], ), lambda intent: configuration_path), (PackerBuild( configuration_path=configuration_path, ), lambda intent: ami_map), (StandardOut( content=json.dumps( thaw(ami_map), encoding='utf-8', ) + b"\n", ), lambda intent: None), ], eff=publish_installer_images_effects(options=options) )
def test_record_recently_converged(self): """ After converging, the group is added to ``recently_converged`` -- but *before* being removed from ``currently_converging``, to avoid race conditions. """ currently = Reference(pset()) recently = Reference(pmap()) remove_from_currently = match_func(pset([self.group_id]), pset([])) sequence = [ (ReadReference(currently), lambda i: pset()), add_to_currently(currently, self.group_id), (('ec', self.tenant_id, self.group_id, 3600), lambda i: (StepResult.SUCCESS, ScalingGroupStatus.ACTIVE)), (Func(time.time), lambda i: 100), add_to_recently(recently, self.group_id, 100), (ModifyReference(currently, remove_from_currently), noop), (DeleteNode(path='/groups/divergent/tenant-id_g1', version=self.version), noop), (Log('mark-clean-success', {}), noop) ] eff = converge_one_group( currently, recently, self.tenant_id, self.group_id, self.version, 3600, execute_convergence=self._execute_convergence) perform_sequence(sequence, eff)
def test_sequence(self): """ The function generates a packer configuration file, runs packer build and uploads the AMI ids to a given S3 bucket. """ options = default_options() configuration_path = self.make_temporary_directory() ami_map = PACKER_OUTPUT_US_ALL.output perform_sequence(seq=[ (PackerConfigure( build_region=options["build_region"], publish_regions=options["regions"], source_ami=options["source_ami"], template=options["template"], distribution=options["distribution"], ), lambda intent: configuration_path), (PackerBuild(configuration_path=configuration_path, ), lambda intent: ami_map), (WriteToS3( content=json.dumps( thaw(ami_map), encoding='utf-8', ), target_bucket=options["target_bucket"], target_key=options["template"], ), lambda intent: None), ], eff=publish_installer_images_effects(options=options))
def test_multiple_errors(self): """ If bulk add returns 409 then multiple errors returned are collected and raised as a single `BulkErrors` """ errors = { "errors": [ lb_inactive(self.lbs[0]), "Load Balancer Pool {} does not exist".format(self.lbs[1]), "Cloud Server {} is unprocessable".format(self.nodes[2]) ] } seq = [(self.svc_req_intent(self.data), const(stub_json_response(errors, 409))), (log_intent("request-rcv3-bulk", errors, req_body=("jsonified", self.data)), noop)] with self.assertRaises(r.BulkErrors) as ec: perform_sequence(seq, r.bulk_add(self.pairs)) self.assertEqual( ec.exception.errors, pset([ r.LBInactive(self.lbs[0]), r.NoSuchLBError(self.lbs[1]), r.ServerUnprocessableError(self.nodes[2]) ]))
def test_multiple_errors(self): """ If bulk add returns 409 then multiple errors returned are collected and raised as a single `BulkErrors` """ errors = { "errors": [ lb_inactive(self.lbs[0]), "Load Balancer Pool {} does not exist".format(self.lbs[1]), "Cloud Server {} is unprocessable".format(self.nodes[2]) ] } seq = [ (self.svc_req_intent(self.data), const(stub_json_response(errors, 409))), (log_intent( "request-rcv3-bulk", errors, req_body=("jsonified", self.data)), noop) ] with self.assertRaises(r.BulkErrors) as ec: perform_sequence(seq, r.bulk_add(self.pairs)) self.assertEqual( ec.exception.errors, pset([r.LBInactive(self.lbs[0]), r.NoSuchLBError(self.lbs[1]), r.ServerUnprocessableError(self.nodes[2])]) )
def test_sequence(self): """ The function generates a packer configuration file, runs packer build and uploads the AMI ids to a given S3 bucket. """ options = PublishInstallerImagesOptions() options.parseOptions( [b'--source-ami-map', b'{"us-west-1": "ami-1234"}']) configuration_path = self.make_temporary_directory() ami_map = PACKER_OUTPUT_US_ALL.output perform_sequence(seq=[ (PackerConfigure( build_region=options["build_region"], publish_regions=options["regions"], source_ami_map=options["source-ami-map"], template=options["template"], distribution=options["distribution"], ), lambda intent: configuration_path), (PackerBuild(configuration_path=configuration_path, ), lambda intent: ami_map), (StandardOut(content=json.dumps( thaw(ami_map), encoding='utf-8', ) + b"\n", ), lambda intent: None), ], eff=publish_installer_images_effects(options=options))
def test_create_client_cert(): seq = [(kubecert.CreateDirectory( path='/cert', create_parents=True, ), return_none), (kubecert.GenerateRSAKey(path='/cert/key.pem'), return_none), (kubecert.GenerateOpenSSLConfig(path='/cert/openssl.conf', kind='client'), return_none), (kubecert.GenerateCSR(output_path='/cert/csr.pem', config_path='/cert/openssl.conf', common_name='common.name', key_path='/cert/key.pem'), return_none), (kubecert.SignCertificate( csr_path='/cert/csr.pem', ca_cert_path='/ca/ca-crt.pem', ca_key_path='/ca/ca-key.pem', output_path='/cert/crt.pem', config_path='/cert/openssl.conf', ), return_none)] eff = kubecert.generate_cert( ca_path='/ca', outpath='/cert', common_name='common.name', kind='client', ) perform_sequence(seq, eff)
def test_sequence(self): """ The function generates a packer configuration file, runs packer build and uploads the AMI ids to a given S3 bucket. """ options = default_options() configuration_path = self.make_temporary_directory() ami_map = PACKER_OUTPUT_US_ALL.output perform_sequence( seq=[ ( PackerConfigure( build_region=options["build_region"], publish_regions=options["regions"], source_ami=options["source_ami"], template=options["template"], distribution=options["distribution"], ), lambda intent: configuration_path, ), (PackerBuild(configuration_path=configuration_path), lambda intent: ami_map), ( WriteToS3( content=json.dumps(thaw(ami_map), encoding="utf-8"), target_bucket=options["target_bucket"], target_key=options["template"], ), lambda intent: None, ), ], eff=publish_installer_images_effects(options=options), )
def test_fold_effect_str(): """str()ing a FoldError returns useful traceback/exception info.""" effs = [Effect("a"), Effect(Error(ZeroDivisionError("foo"))), Effect("c")] dispatcher = [("a", lambda i: "Ei")] eff = fold_effect(operator.add, "Nil", effs) with raises(FoldError) as excinfo: perform_sequence(dispatcher, eff) assert str(excinfo.value).startswith("<FoldError after accumulating 'NilEi'> Original traceback follows:\n") assert str(excinfo.value).endswith("ZeroDivisionError: foo")
def test_quit_game(): for exc in (KeyboardInterrupt(), EOFError()): expected_effects = [ (Display(render(initial_state)), noop), (Prompt("> "), lambda i: raise_(exc)), (Display("\nThanks for playing!"), noop), ] eff = step(initial_state) with raises(SystemExit): perform_sequence(expected_effects, eff)
def _verify_sequence(self, sequence, converging=Reference(pset()), recent=Reference(pmap()), allow_refs=True): """ Verify that sequence is executed """ eff = converge_one_group( converging, recent, self.tenant_id, self.group_id, self.version, 3600, execute_convergence=self._execute_convergence) fb_dispatcher = _get_dispatcher() if allow_refs else base_dispatcher perform_sequence(sequence, eff, fallback_dispatcher=fb_dispatcher)
def test_fold_effect_str(): """str()ing a FoldError returns useful traceback/exception info.""" effs = [Effect("a"), Effect(Error(ZeroDivisionError("foo"))), Effect("c")] dispatcher = [("a", lambda i: "Ei")] eff = fold_effect(operator.add, "Nil", effs) with raises(FoldError) as excinfo: perform_sequence(dispatcher, eff) assert str(excinfo.value).startswith( "<FoldError after accumulating 'NilEi'> Original traceback follows:\n") assert str(excinfo.value).endswith("ZeroDivisionError: foo")
def test_mainloop(): expected_effects = [(Display(render(initial_state)), noop), (Prompt("> "), lambda i: "move east"), (Display("Okay."), noop), (SaveGame(state=in_street), noop), (Display(render(in_street)), noop), (Prompt("> "), lambda i: raise_(KeyboardInterrupt())), (Display("\nThanks for playing!"), noop)] eff = mainloop(initial_state) with raises(SystemExit): perform_sequence(expected_effects, eff)
def test_create_ca(): seq = [ (kubecert.CreateDirectory('/test', create_parents=True), return_none), (kubecert.GenerateRSAKey(path='/test/ca-key.pem'), return_none), ( kubecert.GenerateCACertificate( path='/test/ca-crt.pem', key_path='/test/ca-key.pem', common_name='common.name', ), return_none, ), ] eff = kubecert.generate_ca(output_path='/test', common_name='common.name') perform_sequence(seq, eff)
def test_success(self): """ Gets LB contents with drained_at correctly """ node11 = node('11', 'a11', condition='DRAINING') node12 = node('12', 'a12') node21 = node('21', 'a21', weight=3) node22 = node('22', 'a22', weight=None, condition='DRAINING') seq = [ lb_req('loadbalancers', True, {'loadBalancers': [{ 'id': 1 }, { 'id': 2 }]}), parallel_sequence([[nodes_req(1, [node11, node12])], [nodes_req(2, [node21, node22])], [lb_hm_req(1, {"type": "CONNECT"})], [lb_hm_req(2, {})]]), parallel_sequence([[node_feed_req('1', '11', '11feed')], [node_feed_req('2', '22', '22feed')]]), ] eff = get_clb_contents() self.assertEqual(perform_sequence(seq, eff), ([ attr.assoc(CLBNode.from_node_json(1, node11), _drained_at=1.0), CLBNode.from_node_json(1, node12), CLBNode.from_node_json(2, node21), attr.assoc(CLBNode.from_node_json(2, node22), _drained_at=2.0) ], { '1': CLB(True), '2': CLB(False) }))
def test_no_exponential_backoff(self): """ If ``False`` is passed for the ``backoff`` parameter, the effect is always retried with the same delay. """ divisors = [0, 0, 0, 1] def tester(): x = divisors.pop(0) return 1 / x seq = [ (Delay(5), lambda ignore: None), (Delay(5), lambda ignore: None), (Delay(5), lambda ignore: None), ] retrier = retry_effect_with_timeout( Effect(Func(tester)), timeout=1, retry_wait=timedelta(seconds=5), backoff=False, ) result = perform_sequence(seq, retrier) self.assertEqual(result, 1)
def test_all_retries(self): """ If bulk_delete returns "server not a member", lb or server deleted for all attempted pairs then there is no retry and returns None """ errors = { "errors": [ server_not_member(self.lbs[0].upper(), self.nodes[0]), "Cloud Server {} does not exist".format(self.nodes[1]), "Load Balancer Pool {} does not exist".format( self.lbs[2].upper()) ] } pairs = pset([ (self.lbs[0], self.nodes[1]), # test same server pairs (self.lbs[2], self.nodes[0]) # test same lb pairs ]) pairs = self.pairs | pairs data = r._sorted_data(pairs) seq = [(self.svc_req_intent(data), const(stub_json_response(errors, 409))), (log_intent("request-rcv3-bulk", errors, req_body=("jsonified", data)), noop)] self.assertIsNone(perform_sequence(seq, r.bulk_delete(pairs)))
def test_no_draining(self): """ Doesnt fetch feeds if all nodes are ENABLED """ seq = [ lb_req('loadbalancers', True, {'loadBalancers': [{ 'id': 1 }, { 'id': 2 }]}), parallel_sequence([[nodes_req(1, [node('11', 'a11')])], [nodes_req(2, [node('21', 'a21')])], [lb_hm_req(1, {})], [lb_hm_req(2, {})]]), parallel_sequence([]) # No nodes to fetch ] make_desc = partial(CLBDescription, port=20, weight=2, condition=CLBNodeCondition.ENABLED, type=CLBNodeType.PRIMARY) eff = get_clb_contents() self.assertEqual(perform_sequence(seq, eff), ([ CLBNode( node_id='11', address='a11', description=make_desc(lb_id='1')), CLBNode( node_id='21', address='a21', description=make_desc(lb_id='2')) ], { '1': CLB(False), '2': CLB(False) }))
def test_returns_as_servers(self): """ Returns servers with AS metadata in it grouped by scaling group ID """ as_servers = ([{ 'metadata': { 'rax:auto_scaling_group_id': 'a' }, 'id': i } for i in range(5)] + [{ 'metadata': { 'rax:auto_scaling_group_id': 'b' }, 'id': i } for i in range(5, 8)] + [{ 'metadata': { 'rax:auto_scaling_group_id': 'a' }, 'id': 10 }]) servers = as_servers + [{'metadata': 'junk'}] * 3 eff = get_all_scaling_group_servers() body = {'servers': servers} sequence = [(service_request(*self.req).intent, lambda i: (StubResponse(200, None), body)), (Log(mock.ANY, mock.ANY), lambda i: None)] result = perform_sequence(sequence, eff) self.assertEqual(result, { 'a': as_servers[:5] + [as_servers[-1]], 'b': as_servers[5:8] })
def test_sequence_error(): """ Allows :obj:`FoldError` to be raised when an Effect fails. The list accumulated so far is the `accumulator` value in the :obj:`FoldError`. """ effs = [Effect("a"), Effect(Error(ZeroDivisionError("foo"))), Effect("c")] dispatcher = [("a", lambda i: "Ei")] eff = sequence(effs) with raises(FoldError) as excinfo: perform_sequence(dispatcher, eff) assert excinfo.value.accumulator == ["Ei"] assert excinfo.value.wrapped_exception[0] is ZeroDivisionError assert str(excinfo.value.wrapped_exception[1]) == "foo"
def _test_deleting_group(self, step_result, with_delete, group_status): def _plan(dsg, *a, **kwargs): self.dsg = dsg return [TestStep(Effect("step"))] self.state.status = ScalingGroupStatus.DELETING sequence = [ parallel_sequence([]), (Log('execute-convergence', mock.ANY), noop), parallel_sequence([ [("step", lambda i: (step_result, []))] ]), (Log('execute-convergence-results', mock.ANY), noop), ] if with_delete: sequence.append((DeleteGroup(tenant_id=self.tenant_id, group_id=self.group_id), noop)) self.assertEqual( # skipping cache update intents returned in get_seq() perform_sequence(self.get_seq(False) + sequence, self._invoke(_plan)), (step_result, group_status)) # desired capacity was changed to 0 self.assertEqual(self.dsg.capacity, 0)
def test_acquire_blocking_no_timeout(self): """ When acquire_eff is called without timeout, it creates child, realizes its not the smallest, tries again every 0.1 seconds without checking time and succeeds if its the smallest node """ seq = [ (Constant(None), noop), (zk.CreateNode("/testlock"), const("/testlock")), (Func(uuid.uuid4), const("prefix")), (zk.CreateNode( "/testlock/prefix", value="id", ephemeral=True, sequence=True), const("/testlock/prefix0000000001")), (GetChildren("/testlock"), const(["prefix0000000000", "prefix0000000001"])), (Func(time.time), const(0)), (Delay(0.1), noop), (GetChildren("/testlock"), const(["prefix0000000000", "prefix0000000001"])), (Delay(0.1), noop), (GetChildren("/testlock"), const(["prefix0000000001"])) ] self.assertTrue( perform_sequence(seq, self.lock.acquire_eff(True, None)))
def test_no_steps(self): """ If state of world matches desired, no steps are executed, but the `active` servers are still updated, and SUCCESS is the return value. """ for serv in self.servers: serv.desired_lbs = pset() sequence = [ parallel_sequence([]), (Log('execute-convergence', mock.ANY), noop), (Log('execute-convergence-results', {'results': [], 'worst_status': 'SUCCESS'}), noop), (UpdateServersCache( "tenant-id", "group-id", self.now, [thaw(self.servers[0].json.set('_is_as_active', True)), thaw(self.servers[1].json.set("_is_as_active", True))]), noop) ] self.state_active = { 'a': {'id': 'a', 'links': [{'href': 'link1', 'rel': 'self'}]}, 'b': {'id': 'b', 'links': [{'href': 'link2', 'rel': 'self'}]} } self.cache[0]["_is_as_active"] = True self.cache[1]["_is_as_active"] = True self.assertEqual( perform_sequence(self.get_seq() + sequence, self._invoke()), (StepResult.SUCCESS, ScalingGroupStatus.ACTIVE))
def test_log_steps(self): """The steps to be executed are logged to cloud feeds.""" step = CreateServer(server_config=pmap({"foo": "bar"})) step.as_effect = lambda: Effect("create-server") def plan(*args, **kwargs): return pbag([step]) sequence = [ parallel_sequence([ [parallel_sequence([ [(Log('convergence-create-servers', {'num_servers': 1, 'server_config': {'foo': 'bar'}, 'cloud_feed': True}), noop)] ])] ]), (Log(msg='execute-convergence', fields=mock.ANY), noop), parallel_sequence([ [("create-server", lambda i: (StepResult.RETRY, []))] ]), (Log(msg='execute-convergence-results', fields=mock.ANY), noop) ] self.assertEqual( perform_sequence(self.get_seq() + sequence, self._invoke(plan)), (StepResult.RETRY, ScalingGroupStatus.ACTIVE))
def test_ignore_disappearing_divergent_flag(self): """ When the divergent flag disappears just as we're starting to converge, the group does not get converged and None is returned as its result. This happens when a concurrent convergence iteration is just finishing up. """ eff = self._converge_all_groups(['00_g1']) def get_bound_sequence(tid, gid): # since this GetStat is going to return None, no more effects will # be run. This is the crux of what we're testing. znode = '/groups/divergent/{}_{}'.format(tid, gid) return [ (GetStat(path=znode), noop), (Log('converge-divergent-flag-disappeared', fields={'znode': znode}), noop)] sequence = [ (ReadReference(ref=self.currently_converging), lambda i: pset()), (Log('converge-all-groups', dict(group_infos=[self.group_infos[0]], currently_converging=[])), noop), (ReadReference(ref=self.recently_converged), lambda i: pmap()), (Func(time.time), lambda i: 100), parallel_sequence([ [(BoundFields(mock.ANY, fields={'tenant_id': '00', 'scaling_group_id': 'g1'}), nested_sequence(get_bound_sequence('00', 'g1')))], ]), ] self.assertEqual(perform_sequence(sequence, eff), [None])
def test_dont_filter_out_non_recently_converged(self): """ If a group was converged in the past but not recently, it will be cleaned from the ``recently_converged`` map, and it will be converged. """ # g1: converged a while ago; divergent -> removed and converged # g2: converged recently; not divergent -> not converged # g3: converged a while ago; not divergent -> removed and not converged eff = self._converge_all_groups(['00_g1']) sequence = [ (ReadReference(ref=self.currently_converging), lambda i: pset([])), (Log('converge-all-groups', dict(group_infos=[self.group_infos[0]], currently_converging=[])), noop), (ReadReference(ref=self.recently_converged), lambda i: pmap({'g1': 4, 'g2': 10, 'g3': 0})), (Func(time.time), lambda i: 20), (ModifyReference(self.recently_converged, match_func("literally anything", pmap({'g2': 10}))), noop), parallel_sequence([[self._expect_group_converged('00', 'g1')]]) ] self.assertEqual(perform_sequence(sequence, eff), ['converged g1!'])
def test_failure_unknown_reasons(self): """ The group is put into ERROR state if any step returns FAILURE, and unknown error is defaulted to fixed reason """ exc_info = raise_to_exc_info(ValueError('wat')) def plan(*args, **kwargs): return [TestStep(Effect("fail"))] sequence = [ parallel_sequence([]), (Log(msg='execute-convergence', fields=mock.ANY), noop), parallel_sequence([ [("fail", lambda i: (StepResult.FAILURE, [ErrorReason.Exception(exc_info)]))] ]), (Log(msg='execute-convergence-results', fields=mock.ANY), noop), (UpdateGroupStatus(scaling_group=self.group, status=ScalingGroupStatus.ERROR), noop), (Log('group-status-error', dict(isError=True, cloud_feed=True, status='ERROR', reasons=['Unknown error occurred'])), noop), (UpdateGroupErrorReasons(self.group, ['Unknown error occurred']), noop) ] self.assertEqual( perform_sequence(self.get_seq() + sequence, self._invoke(plan)), (StepResult.FAILURE, ScalingGroupStatus.ERROR))
def test_lb_disappeared_during_node_fetch(self): """ If a load balancer gets deleted while fetching nodes, no nodes will be returned for it. """ seq = [ lb_req('loadbalancers', True, {'loadBalancers': [{'id': 1}, {'id': 2}]}), parallel_sequence([ [nodes_req(1, [node('11', 'a11')])], [lb_req('loadbalancers/2/nodes', True, CLBNotFoundError(lb_id=u'2'))], [lb_hm_req(1, {"type": "CONNECT"})], [lb_req('loadbalancers/2/healthmonitor', True, CLBNotFoundError(lb_id=u'2'))] ]), parallel_sequence([]) # No node feeds to fetch ] make_desc = partial(CLBDescription, port=20, weight=2, condition=CLBNodeCondition.ENABLED, type=CLBNodeType.PRIMARY) eff = get_clb_contents() self.assertEqual( perform_sequence(seq, eff), ([CLBNode(node_id='11', address='a11', description=make_desc(lb_id='1'))], {'1': CLB(True)}))
def test_added(self): """ total desired, pending and actual are added to cloud metrics """ metrics = [GroupMetrics('t1', 'g1', 3, 2, 0), GroupMetrics('t2', 'g1', 4, 4, 1), GroupMetrics('t2', 'g', 100, 20, 0)] m = {'collectionTime': 100000, 'ttlInSeconds': 5 * 24 * 60 * 60} md = merge(m, {'metricValue': 107, 'metricName': 'ord.desired'}) ma = merge(m, {'metricValue': 26, 'metricName': 'ord.actual'}) mp = merge(m, {'metricValue': 1, 'metricName': 'ord.pending'}) mt = merge(m, {'metricValue': 2, 'metricName': 'ord.tenants'}) mg = merge(m, {'metricValue': 3, 'metricName': 'ord.groups'}) mt1d = merge(m, {'metricValue': 3, 'metricName': 'ord.t1.desired'}) mt1a = merge(m, {'metricValue': 2, 'metricName': 'ord.t1.actual'}) mt1p = merge(m, {'metricValue': 0, 'metricName': 'ord.t1.pending'}) mt2d = merge(m, {'metricValue': 104, 'metricName': 'ord.t2.desired'}) mt2a = merge(m, {'metricValue': 24, 'metricName': 'ord.t2.actual'}) mt2p = merge(m, {'metricValue': 1, 'metricName': 'ord.t2.pending'}) req_data = [md, ma, mp, mt, mg, mt1d, mt1a, mt1p, mt2d, mt2a, mt2p] log = mock_log() seq = [ (Func(time.time), const(100)), (service_request( ServiceType.CLOUD_METRICS_INGEST, "POST", "ingest", data=req_data, log=log).intent, noop) ] eff = add_to_cloud_metrics(m['ttlInSeconds'], 'ord', metrics, 2, log) self.assertIsNone(perform_sequence(seq, eff)) log.msg.assert_called_once_with( 'total desired: {td}, total_actual: {ta}, total pending: {tp}', td=107, ta=26, tp=1)
def test_prefetch_github_actual_prefetch(pypi2nix_list_remote): seq = [ ( nix_prefetch_github.GetListRemote(owner="seppeljordan", repo="pypi2nix"), lambda i: pypi2nix_list_remote, ), ( nix_prefetch_github.CalculateSha256Sum( owner="seppeljordan", repo="pypi2nix", revision=pypi2nix_list_remote.branch("master"), ), lambda i: "TEST_ACTUALHASH", ), ( nix_prefetch_github.TryPrefetch( owner="seppeljordan", repo="pypi2nix", rev=pypi2nix_list_remote.branch("master"), sha256="TEST_ACTUALHASH", ), lambda i: None, ), ] eff = nix_prefetch_github.prefetch_github( owner="seppeljordan", repo="pypi2nix", prefetch=True ) prefetch_result = perform_sequence(seq, eff) assert prefetch_result["rev"] == pypi2nix_list_remote.branch("master") assert prefetch_result["sha256"] == "TEST_ACTUALHASH"
def test_added(self): """ total desired, pending and actual are added to cloud metrics """ td = 10 ta = 20 tp = 3 tt = 7 tg = 13 m = {'collectionTime': 100000, 'ttlInSeconds': 5 * 24 * 60 * 60} md = merge(m, {'metricValue': td, 'metricName': 'ord.desired'}) ma = merge(m, {'metricValue': ta, 'metricName': 'ord.actual'}) mp = merge(m, {'metricValue': tp, 'metricName': 'ord.pending'}) mt = merge(m, {'metricValue': tt, 'metricName': 'ord.tenants'}) mg = merge(m, {'metricValue': tg, 'metricName': 'ord.groups'}) req_data = [md, ma, mp, mt, mg] log = object() seq = [ (Func(time.time), const(100)), (service_request( ServiceType.CLOUD_METRICS_INGEST, "POST", "ingest", data=req_data, log=log).intent, noop) ] eff = add_to_cloud_metrics( m['ttlInSeconds'], 'ord', td, ta, tp, tt, tg, log=log) self.assertIsNone(perform_sequence(seq, eff))
def _check_retries(self, get_pairs_data): errors = { "errors": [ server_not_member(self.lbs[0].upper(), self.nodes[0]), "Cloud Server {} does not exist".format(self.nodes[1]), "Load Balancer Pool {} does not exist".format( self.lbs[2].upper()) ] } lbr1 = "d6d3aa7c-dfa5-4e61-96ee-1d54ac1075d2" noder1 = "a95ae0c4-6ab8-4873-b82f-f8433840cff2" lbr2 = "e6d3aa7c-dfa5-4e61-96ee-1d54ac1075d2" noder2 = "e95ae0c4-6ab8-4873-b82f-f8433840cff2" pairs, data = get_pairs_data(lbr1, noder1, lbr2, noder2) retried_data = r._sorted_data([(lbr1, noder1), (lbr2, noder2)]) success_resp = {"good": "response"} seq = [ (self.svc_req_intent(data), const(stub_json_response(errors, 409))), (log_intent( "request-rcv3-bulk", errors, req_body=("jsonified", data)), noop), (self.svc_req_intent(retried_data), const(stub_json_response(success_resp, 204))), (log_intent( "request-rcv3-bulk", success_resp, req_body=("jsonified", retried_data)), noop) ] self.assertEqual( perform_sequence(seq, r.bulk_delete(pairs)), success_resp)
def test_list_servers_details_all_gets_until_no_next_link(self): """ :func:`list_servers_details_all` follows the servers links until there are no more links, and returns a list of servers as the result. It ignores any non-next links. """ bodies = [ {'servers': ['1', '2'], 'servers_links': [{'href': 'doesnt_matter_url?marker=3', 'rel': 'next'}]}, {'servers': ['3', '4'], 'servers_links': [{'href': 'doesnt_matter_url?marker=5', 'rel': 'next'}, {'href': 'doesnt_matter_url?marker=1', 'rel': 'prev'}]}, {'servers': ['5', '6'], 'servers_links': [{'href': 'doesnt_matter_url?marker=3', 'rel': 'prev'}]} ] resps = [json.dumps(d) for d in bodies] eff = list_servers_details_all({'marker': ['1']}) seq = [ (self._list_server_details_intent({'marker': ['1']}), service_request_eqf(stub_pure_response(resps[0], 200))), (self._list_server_details_log_intent(bodies[0]), lambda _: None), (self._list_server_details_intent({'marker': ['3']}), service_request_eqf(stub_pure_response(resps[1], 200))), (self._list_server_details_log_intent(bodies[1]), lambda _: None), (self._list_server_details_intent({'marker': ['5']}), service_request_eqf(stub_pure_response(resps[2], 200))), (self._list_server_details_log_intent(bodies[2]), lambda _: None) ] result = perform_sequence(seq, eff) self.assertEqual(result, ['1', '2', '3', '4', '5', '6'])
def test_from_cache(self): """ If cache is there then servers returned are updated with servers not found in current list marked as deleted """ asmetakey = "rax:autoscale:group:id" cache = [ {'id': 'a', 'metadata': {asmetakey: "gid"}}, # gets updated {'id': 'b', 'metadata': {asmetakey: "gid"}}, # deleted {'id': 'd', 'metadata': {asmetakey: "gid"}}, # meta removed {'id': 'c', 'metadata': {asmetakey: "gid"}}] # same current = [ {'id': 'a', 'b': 'c', 'metadata': {asmetakey: "gid"}}, {'id': 'z', 'z': 'w', 'metadata': {asmetakey: "gid"}}, # new {'id': 'd', 'metadata': {"changed": "yes"}}, {'id': 'c', 'metadata': {asmetakey: "gid"}}] last_update = datetime(2010, 5, 20) sequence = [ (("cachegstidgid", False), lambda i: (cache, last_update)), (("alls",), lambda i: current)] del_cache_server = deepcopy(cache[1]) del_cache_server["status"] = "DELETED" self.assertEqual( self.freeze(perform_sequence(sequence, self._invoke())), self.freeze([del_cache_server, cache[-1]] + current[0:2]))
def perform_retry_without_delay(actual_retry_intent): should_retry = actual_retry_intent.should_retry if isinstance(should_retry, ShouldDelayAndRetry): def should_retry(exc_info): exc_type, exc_value, exc_traceback = exc_info failure = Failure(exc_value, exc_type, exc_traceback) return Effect( Constant( actual_retry_intent.should_retry.can_retry(failure))) new_retry_effect = Effect( Retry(effect=actual_retry_intent.effect, should_retry=should_retry)) _dispatchers = [ TypeDispatcher({Retry: perform_retry}), base_dispatcher ] if fallback_dispatcher is not None: _dispatchers.append(fallback_dispatcher) seq = [(expected_retry_intent.effect.intent, performer) for performer in performers] return perform_sequence(seq, new_retry_effect, ComposedDispatcher(_dispatchers))
def test_reactivate_group_on_success_after_steps(self): """ When the group started in ERROR state, and convergence succeeds, the group is put back into ACTIVE. """ self.manifest['state'].status = ScalingGroupStatus.ERROR def plan(*args, **kwargs): return pbag([TestStep(Effect("step"))]) sequence = [ parallel_sequence([]), (Log(msg='execute-convergence', fields=mock.ANY), noop), parallel_sequence([ [("step", lambda i: (StepResult.SUCCESS, []))] ]), (Log(msg='execute-convergence-results', fields=mock.ANY), noop), (UpdateGroupStatus(scaling_group=self.group, status=ScalingGroupStatus.ACTIVE), noop), (Log('group-status-active', dict(cloud_feed=True, status='ACTIVE')), noop), (UpdateServersCache( "tenant-id", "group-id", self.now, [thaw(self.servers[0].json.set('_is_as_active', True)), thaw(self.servers[1].json.set('_is_as_active', True))]), noop), ] self.assertEqual( perform_sequence(self.get_seq() + sequence, self._invoke(plan)), (StepResult.SUCCESS, ScalingGroupStatus.ACTIVE))
def test_sequence_error(): """ Allows :obj:`FoldError` to be raised when an Effect fails. The list accumulated so far is the `accumulator` value in the :obj:`FoldError`. """ effs = [Effect('a'), Effect(Error(ZeroDivisionError('foo'))), Effect('c')] dispatcher = [('a', lambda i: 'Ei')] eff = sequence(effs) with raises(FoldError) as excinfo: perform_sequence(dispatcher, eff) assert excinfo.value.accumulator == ['Ei'] assert excinfo.value.wrapped_exception[0] is ZeroDivisionError assert str(excinfo.value.wrapped_exception[1]) == 'foo'
def test_lb_disappeared_during_node_fetch(self): """ If a load balancer gets deleted while fetching nodes, no nodes will be returned for it. """ seq = [ lb_req('loadbalancers', True, {'loadBalancers': [{ 'id': 1 }, { 'id': 2 }]}), parallel_sequence([[nodes_req(1, [node('11', 'a11')])], [ lb_req('loadbalancers/2/nodes', True, CLBNotFoundError(lb_id=u'2')) ], [lb_hm_req(1, {"type": "CONNECT"})], [ lb_req('loadbalancers/2/healthmonitor', True, CLBNotFoundError(lb_id=u'2')) ]]), parallel_sequence([]) # No node feeds to fetch ] make_desc = partial(CLBDescription, port=20, weight=2, condition=CLBNodeCondition.ENABLED, type=CLBNodeType.PRIMARY) eff = get_clb_contents() self.assertEqual(perform_sequence(seq, eff), ([ CLBNode( node_id='11', address='a11', description=make_desc(lb_id='1')) ], { '1': CLB(True) }))
def test_set_metadata_item(self): """ :obj:`SetMetadataItemOnServer.as_effect` produces a request for setting a metadata item on a particular server. It succeeds if successful, but does not fail for any errors. """ server_id = u'abc123' meta = SetMetadataItemOnServer(server_id=server_id, key='metadata_key', value='teapot') eff = meta.as_effect() seq = [ (eff.intent, lambda i: (StubResponse(202, {}), {})), (Log(ANY, ANY), lambda _: None) ] self.assertEqual( perform_sequence(seq, eff), (StepResult.SUCCESS, [])) exceptions = (NoSuchServerError("msg", server_id=server_id), ServerMetadataOverLimitError("msg", server_id=server_id), NovaRateLimitError("msg"), APIError(code=500, body="", headers={})) for exception in exceptions: self.assertRaises( type(exception), perform_sequence, [(eff.intent, lambda i: raise_(exception))], eff)
def test_step(): expected_effects = [(Display(render(initial_state)), noop), (Prompt("> "), lambda i: "move east"), (Display("Okay."), noop)] eff = step(initial_state) result = perform_sequence(expected_effects, eff) assert result == in_street
def _test_no_cache(self, empty): current = [] if empty else [{'id': 'a', 'a': 'b'}, {'id': 'b', 'b': 'c'}] sequence = [ (("cachegstidgid", False), lambda i: (object(), None)), (("all-as",), lambda i: {} if empty else {"gid": current})] self.assertEqual(perform_sequence(sequence, self._invoke()), current)
def test_filters_on_user_criteria(self): """ Considers user provided filter if provided """ as_servers = ([{ 'metadata': { 'rax:auto_scaling_group_id': 'a' }, 'id': i } for i in range(5)] + [{ 'metadata': { 'rax:auto_scaling_group_id': 'b' }, 'id': i } for i in range(5, 8)]) servers = as_servers + [{'metadata': 'junk'}] * 3 eff = get_all_scaling_group_servers( server_predicate=lambda s: s['id'] % 3 == 0) body = {'servers': servers} sequence = [(service_request(*self.req).intent, lambda i: (StubResponse(200, None), body)), (Log(mock.ANY, mock.ANY), lambda i: None)] result = perform_sequence(sequence, eff) self.assertEqual(result, { 'a': [as_servers[0], as_servers[3]], 'b': [as_servers[6]] })
def _perform_add_event(self, response_sequence): """ Given a sequence of functions that take an intent and returns a response (or raises an exception), perform :func:`add_event` and return the result. """ log = object() eff = add_event(self.event, 'tid', 'ord', log) uid = '00000000-0000-0000-0000-000000000000' svrq = service_request( ServiceType.CLOUD_FEEDS, 'POST', 'autoscale/events', headers={'content-type': ['application/vnd.rackspace.atom+json']}, data=self._get_request('INFO', uid, 'tid'), log=log, success_pred=has_code(201), json_response=False) seq = [ (TenantScope(mock.ANY, 'tid'), nested_sequence([ retry_sequence( Retry(effect=svrq, should_retry=ShouldDelayAndRetry( can_retry=mock.ANY, next_interval=exponential_backoff_interval(2))), response_sequence) ])) ] return perform_sequence(seq, eff)
def test_success(self): """ Gets LB contents with drained_at correctly """ node11 = node('11', 'a11', condition='DRAINING') node12 = node('12', 'a12') node21 = node('21', 'a21', weight=3) node22 = node('22', 'a22', weight=None, condition='DRAINING') seq = [ lb_req('loadbalancers', True, {'loadBalancers': [{'id': 1}, {'id': 2}]}), parallel_sequence([[nodes_req(1, [node11, node12])], [nodes_req(2, [node21, node22])], [lb_hm_req(1, {"type": "CONNECT"})], [lb_hm_req(2, {})]]), parallel_sequence([[node_feed_req('1', '11', '11feed')], [node_feed_req('2', '22', '22feed')]]), ] eff = get_clb_contents() self.assertEqual( perform_sequence(seq, eff), ([attr.assoc(CLBNode.from_node_json(1, node11), _drained_at=1.0), CLBNode.from_node_json(1, node12), CLBNode.from_node_json(2, node21), attr.assoc(CLBNode.from_node_json(2, node22), _drained_at=2.0)], {'1': CLB(True), '2': CLB(False)}))
def test_fold_effect_errors(): """ When one of the effects in the folding list fails, a FoldError is raised with the accumulator so far. """ effs = [Effect("a"), Effect(Error(ZeroDivisionError("foo"))), Effect("c")] dispatcher = [("a", lambda i: "Ei")] eff = fold_effect(operator.add, "Nil", effs) with raises(FoldError) as excinfo: perform_sequence(dispatcher, eff) assert excinfo.value.accumulator == "NilEi" assert_that(excinfo.value.wrapped_exception, MatchesException(ZeroDivisionError("foo")))
def test_lb_disappeared_during_feed_fetch(self): """ If a load balancer gets deleted while fetching feeds, no nodes will be returned for it. """ node21 = node('21', 'a21', condition='DRAINING', weight=None) seq = [ lb_req('loadbalancers', True, {'loadBalancers': [{'id': 1}, {'id': 2}]}), parallel_sequence([ [nodes_req(1, [node('11', 'a11', condition='DRAINING'), node('12', 'a12')])], [nodes_req(2, [node21])], [lb_hm_req(1, {"type": "CONNECT"})], [lb_hm_req(2, {"type": "CONNECT"})] ]), parallel_sequence([ [node_feed_req('1', '11', CLBNotFoundError(lb_id=u'1'))], [node_feed_req('2', '21', '22feed')]]), ] eff = get_clb_contents() self.assertEqual( perform_sequence(seq, eff), ([attr.assoc(CLBNode.from_node_json(2, node21), _drained_at=2.0)], {'2': CLB(True)}))
def test_fold_effect_errors(): """ When one of the effects in the folding list fails, a FoldError is raised with the accumulator so far. """ effs = [Effect('a'), Effect(Error(ZeroDivisionError('foo'))), Effect('c')] dispatcher = [('a', lambda i: 'Ei')] eff = fold_effect(operator.add, 'Nil', effs) with raises(FoldError) as excinfo: perform_sequence(dispatcher, eff) assert excinfo.value.accumulator == 'NilEi' assert excinfo.value.wrapped_exception[0] is ZeroDivisionError assert str(excinfo.value.wrapped_exception[1]) == 'foo'
def test_acquire_blocking_success(self): """ acquire_eff creates child, realizes its not the smallest. Tries again every 0.01 seconds until it succeeds """ seq = [ (Constant(None), noop), (zk.CreateNode("/testlock"), const("/testlock")), (Func(uuid.uuid4), const("prefix")), (zk.CreateNode( "/testlock/prefix", value="id", ephemeral=True, sequence=True), const("/testlock/prefix0000000001")), (GetChildren("/testlock"), const(["prefix0000000000", "prefix0000000001"])), (Func(time.time), const(0)), (Delay(0.1), noop), (GetChildren("/testlock"), const(["prefix0000000000", "prefix0000000001"])), (Func(time.time), const(0.2)), (Delay(0.1), noop), (GetChildren("/testlock"), const(["prefix0000000001"])) ] self.assertTrue( perform_sequence(seq, self.lock.acquire_eff(True, 1)))
def test_sequence_error(): """ Allows :obj:`FoldError` to be raised when an Effect fails. The list accumulated so far is the `accumulator` value in the :obj:`FoldError`. """ effs = [Effect("a"), Effect(Error(ZeroDivisionError("foo"))), Effect("c")] dispatcher = [("a", lambda i: "Ei")] eff = sequence(effs) with raises(FoldError) as excinfo: perform_sequence(dispatcher, eff) assert excinfo.value.accumulator == ["Ei"] assert_that(excinfo.value.wrapped_exception, MatchesException(ZeroDivisionError("foo")))
def test_is_acquired_no_children(self): """ is_acquired_eff returns False if there are no children """ self.lock._node = "/testlock/prefix000000000" seq = [(GetChildren("/testlock"), const([]))] self.assertFalse(perform_sequence(seq, self.lock.is_acquired_eff()))
def test_fold_effect_errors(): """ When one of the effects in the folding list fails, a FoldError is raised with the accumulator so far. """ effs = [Effect("a"), Effect(Error(ZeroDivisionError("foo"))), Effect("c")] dispatcher = [("a", lambda i: "Ei")] eff = fold_effect(operator.add, "Nil", effs) with raises(FoldError) as excinfo: perform_sequence(dispatcher, eff) assert excinfo.value.accumulator == "NilEi" assert excinfo.value.wrapped_exception[0] is ZeroDivisionError assert str(excinfo.value.wrapped_exception[1]) == "foo"
def test_all_retries(self): """ If bulk_delete returns "server not a member", lb or server deleted for all attempted pairs then there is no retry and returns None """ errors = { "errors": [ server_not_member(self.lbs[0].upper(), self.nodes[0]), "Cloud Server {} does not exist".format(self.nodes[1]), "Load Balancer Pool {} does not exist".format( self.lbs[2].upper()) ] } pairs = pset([ (self.lbs[0], self.nodes[1]), # test same server pairs (self.lbs[2], self.nodes[0]) # test same lb pairs ]) pairs = self.pairs | pairs data = r._sorted_data(pairs) seq = [ (self.svc_req_intent(data), const(stub_json_response(errors, 409))), (log_intent( "request-rcv3-bulk", errors, req_body=("jsonified", data)), noop) ] self.assertIsNone(perform_sequence(seq, r.bulk_delete(pairs)))
def test_lb_disappeared_during_feed_fetch(self): """ If a load balancer gets deleted while fetching feeds, no nodes will be returned for it. """ node21 = node('21', 'a21', condition='DRAINING', weight=None) seq = [ lb_req('loadbalancers', True, {'loadBalancers': [{ 'id': 1 }, { 'id': 2 }]}), parallel_sequence([[ nodes_req(1, [ node('11', 'a11', condition='DRAINING'), node('12', 'a12') ]) ], [nodes_req(2, [node21])], [lb_hm_req(1, {"type": "CONNECT"})], [lb_hm_req(2, {"type": "CONNECT"})]]), parallel_sequence( [[node_feed_req('1', '11', CLBNotFoundError(lb_id=u'1'))], [node_feed_req('2', '21', '22feed')]]), ] eff = get_clb_contents() self.assertEqual( perform_sequence(seq, eff), ([attr.assoc(CLBNode.from_node_json(2, node21), _drained_at=2.0) ], { '2': CLB(True) }))
def _check_retries(self, get_pairs_data): errors = { "errors": [ server_not_member(self.lbs[0].upper(), self.nodes[0]), "Cloud Server {} does not exist".format(self.nodes[1]), "Load Balancer Pool {} does not exist".format( self.lbs[2].upper()) ] } lbr1 = "d6d3aa7c-dfa5-4e61-96ee-1d54ac1075d2" noder1 = "a95ae0c4-6ab8-4873-b82f-f8433840cff2" lbr2 = "e6d3aa7c-dfa5-4e61-96ee-1d54ac1075d2" noder2 = "e95ae0c4-6ab8-4873-b82f-f8433840cff2" pairs, data = get_pairs_data(lbr1, noder1, lbr2, noder2) retried_data = r._sorted_data([(lbr1, noder1), (lbr2, noder2)]) success_resp = {"good": "response"} seq = [(self.svc_req_intent(data), const(stub_json_response(errors, 409))), (log_intent("request-rcv3-bulk", errors, req_body=("jsonified", data)), noop), (self.svc_req_intent(retried_data), const(stub_json_response(success_resp, 204))), (log_intent("request-rcv3-bulk", success_resp, req_body=("jsonified", retried_data)), noop)] self.assertEqual(perform_sequence(seq, r.bulk_delete(pairs)), success_resp)
def test_reactivate_group_on_success_with_no_steps(self): """ When the group started in ERROR state, and convergence succeeds, the group is put back into ACTIVE, even if there were no steps to execute. """ self.manifest['state'].status = ScalingGroupStatus.ERROR for serv in self.servers: serv.desired_lbs = pset() sequence = [ parallel_sequence([]), (Log(msg='execute-convergence', fields=mock.ANY), noop), (Log(msg='execute-convergence-results', fields=mock.ANY), noop), (UpdateGroupStatus(scaling_group=self.group, status=ScalingGroupStatus.ACTIVE), noop), (Log('group-status-active', dict(cloud_feed=True, status='ACTIVE')), noop), (UpdateServersCache( "tenant-id", "group-id", self.now, [thaw(self.servers[0].json.set("_is_as_active", True)), thaw(self.servers[1].json.set("_is_as_active", True))]), noop) ] self.state_active = { 'a': {'id': 'a', 'links': [{'href': 'link1', 'rel': 'self'}]}, 'b': {'id': 'b', 'links': [{'href': 'link2', 'rel': 'self'}]} } self.cache[0]["_is_as_active"] = True self.cache[1]["_is_as_active"] = True self.assertEqual( perform_sequence(self.get_seq() + sequence, self._invoke()), (StepResult.SUCCESS, ScalingGroupStatus.ACTIVE))
def test_is_acquired_not_first_child(self): """ is_acquired_eff returns False if its not is not the first child """ self.lock._node = "/testlock/prefix0000000001" seq = [(GetChildren("/testlock"), const(["prefix0000000000", "prefix0000000001"]))] self.assertFalse(perform_sequence(seq, self.lock.is_acquired_eff()))