Beispiel #1
0
    def test_unique_values_from_query(self):
        """Athena - Unique Values from Query"""
        query = {
            'ResultSet': {
                'Rows': [
                    {
                        'Data': [{
                            'VarCharValue': 'foobar'
                        }]
                    },
                    {
                        'Data': [{
                            'VarCharValue': 'barfoo'
                        }]
                    },
                    {
                        'Data': [{
                            'VarCharValue': 'barfoo'
                        }]
                    },
                    {
                        'Data': [{
                            'VarCharValue': 'foobarbaz'
                        }]
                    },
                ]
            }
        }
        expected_result = {'foobar', 'barfoo', 'foobarbaz'}

        result = self.client._unique_values_from_query(query)
        assert_count_equal(result, expected_result)
Beispiel #2
0
def test_get_tasks_config():
    tc = api.get_tasks_config()
    nt.assert_is_instance(tc, TasksConfigBaseMapping)
    nt.assert_count_equal(tc, [
        'test_stolos/test_get_tasks_config/depends_on2',
        'test_stolos/test_get_tasks_config/custom_job_id1',
        'test_stolos/test_get_tasks_config/app2',
        'test_stolos/test_get_tasks_config/topological_sort1',
        'test_stolos/test_get_tasks_config/fanout1',
        'test_stolos/test_get_tasks_config/depends_on1',
        'test_stolos/test_get_tasks_config/bash2',
        'test_stolos/test_get_tasks_config/app3',
        'test_stolos/test_get_tasks_config/app1',
        'test_stolos/test_get_tasks_config/bash1',
        'test_stolos/test_get_tasks_config/app4',
        'test_stolos/test_get_tasks_config/valid1',
        'test_stolos/test_get_tasks_config/valid2',
        'test_stolos/test_get_tasks_config/valid3',
        'test_stolos/test_get_tasks_config/valid3b',
        'test_stolos/test_get_tasks_config/valid4',
        'test_stolos/test_get_tasks_config/all_test1',
        'test_stolos/test_get_tasks_config/all_test2',
        'test_stolos/test_get_tasks_config/all_test3',
        'test_stolos/test_get_tasks_config/all_test4',
        'test_stolos/test_get_tasks_config/all_test4b',
        'test_stolos/test_get_tasks_config/all_test5',
        'test_stolos/test_get_tasks_config/autofill1',
        'test_stolos/test_get_tasks_config/autofill2',
        'test_stolos/test_get_tasks_config/autofill3',
        'test_stolos/test_get_tasks_config/autofill_getparents',
    ])
Beispiel #3
0
    def test_get_table_partitions(self):
        """Athena - Get Table Partitions"""
        self.client._client.results = [
            {
                'Data': [{
                    'VarCharValue': 'dt=2018-12-10-10'
                }]
            },
            {
                'Data': [{
                    'VarCharValue': 'dt=2018-12-09-10'
                }]
            },
            {
                'Data': [{
                    'VarCharValue': 'dt=2018-12-09-10'
                }]
            },
            {
                'Data': [{
                    'VarCharValue': 'dt=2018-12-11-10'
                }]
            },
        ]

        expected_result = {
            'dt=2018-12-10-10', 'dt=2018-12-09-10', 'dt=2018-12-11-10'
        }

        result = self.client.get_table_partitions('test_table')
        assert_count_equal(result, expected_result)
 def test_expand_action_with_permission_only_action(self):
     # There are 17 privileges list as "logs.CreateLogDelivery [permission only]"
     expanded_actions = expand_action("logs:GetLogDelivery")
     assert_count_equal(expanded_actions, [{
         "service": "logs",
         "action": "GetLogDelivery"
     }])
Beispiel #5
0
 def test_tojson(self):
     serialise.JSON().write('test', db)
     expect = {
         'Town': [
             {'name': 'down', '_uid': self.t1._uid},
             {'name': 'up', '_uid': self.t2._uid}
         ],
         'Address': [
             {'street': 'easy', 'town': self.t1._uid, '_uid': self.a1._uid},
             {'street': 'some', 'town': self.t2._uid, '_uid': self.a2._uid}
         ],
         'Person': [
             {'custno': 1, 'name': 'matt', 'age': 43,
                 'address': self.a1._uid, '_uid': self.p1._uid},
             {'custno': 2, 'name': 'bob', 'age': 3,
                 'address': self.a1._uid, '_uid': self.p2._uid},
             {'custno': 3, 'name': 'peter', 'age': 29,
                 'address': self.a2._uid, '_uid': self.p3._uid}
         ]
     }
     with open('test', 'rt') as fh:
         got = json.load(fh)
     assert set(got.keys()) == set(expect.keys())
     for table, data in got.items():
         assert_count_equal(data, expect[table])
Beispiel #6
0
def test_get_tasks_config():
    tc = api.get_tasks_config()
    nt.assert_is_instance(tc, TasksConfigBaseMapping)
    nt.assert_count_equal(
        tc,
        ['test_stolos/test_get_tasks_config/depends_on2',
         'test_stolos/test_get_tasks_config/custom_job_id1',
         'test_stolos/test_get_tasks_config/app2',
         'test_stolos/test_get_tasks_config/topological_sort1',
         'test_stolos/test_get_tasks_config/fanout1',
         'test_stolos/test_get_tasks_config/depends_on1',
         'test_stolos/test_get_tasks_config/bash2',
         'test_stolos/test_get_tasks_config/app3',
         'test_stolos/test_get_tasks_config/app1',
         'test_stolos/test_get_tasks_config/bash1',
         'test_stolos/test_get_tasks_config/app4',
         'test_stolos/test_get_tasks_config/valid1',
         'test_stolos/test_get_tasks_config/valid2',
         'test_stolos/test_get_tasks_config/valid3',
         'test_stolos/test_get_tasks_config/valid3b',
         'test_stolos/test_get_tasks_config/valid4',
         'test_stolos/test_get_tasks_config/all_test1',
         'test_stolos/test_get_tasks_config/all_test2',
         'test_stolos/test_get_tasks_config/all_test3',
         'test_stolos/test_get_tasks_config/all_test4',
         'test_stolos/test_get_tasks_config/all_test4b',
         'test_stolos/test_get_tasks_config/all_test5',
         'test_stolos/test_get_tasks_config/autofill1',
         'test_stolos/test_get_tasks_config/autofill2',
         'test_stolos/test_get_tasks_config/autofill3',
         'test_stolos/test_get_tasks_config/autofill_getparents',
         ])
Beispiel #7
0
 def test_load_include():
     """Shared - Config Loading - Include"""
     config = load_config(include={'clusters', 'logs.json'})
     expected_keys = ['clusters', 'logs']
     expected_clusters_keys = ['prod', 'dev']
     assert_count_equal(list(config.keys()), expected_keys)
     assert_count_equal(list(config['clusters'].keys()),
                        expected_clusters_keys)
Beispiel #8
0
def then_compare_tables(context):
    expected = context.table
    testing.assert_true(expected, "Please ensure table is provided")
    observed = context.result
    testing.assert_count_equal([str(x) for x in observed.columns], expected.headings)
    testing.assert_equals(len(expected.rows), len(observed))
    for i in range(len(expected.rows)):
        testing.assert_count_equal([str(x) for x in observed.loc[i].tolist()], 
                                     list(expected.rows[i]))
Beispiel #9
0
    def test_query_result_paginator(self):
        """Athena - Query Result Paginator"""
        data = {'Data': [{'VarCharValue': 'result'}]}
        self.client._client.results = [
            data,
        ]

        items = list(self.client.query_result_paginator('test query'))
        assert_count_equal(items, [{'ResultSet': {'Rows': [data]}}] * 4)
Beispiel #10
0
    def test_iter(self):
        context = {
            'd': 13,
            'str': 'string_Test',
        }

        exec_context = ExecutionContext.from_ns(context)
        correct = list(exec_context.keys())
        test = list(exec_context)
        nt.assert_count_equal(test, correct)
Beispiel #11
0
def test_hansle_typelets():
    """
    hansel typelets should be a complete list. this means
    it will contain superclass typelets as well.
    """
    nt.assert_count_equal(Parent._earthdragon_typelets, ['id'])
    nt.assert_count_equal(Child._earthdragon_merged_typelets, ['id', 'id2'])
    nt.assert_count_equal(Child._earthdragon_typelets, [ 'id2'])
    nt.assert_count_equal(GrandChild._earthdragon_merged_typelets, ['id', 'id2', 'id3'])
    nt.assert_count_equal(GrandChild._earthdragon_typelets, ['id3'])
Beispiel #12
0
    def test_iter(self):
        context = {
            'd': 13,
            'str': 'string_Test',
        }

        exec_context = ExecutionContext.from_ns(context)
        correct = list(exec_context.keys())
        test = list(exec_context)
        nt.assert_count_equal(test, correct)
Beispiel #13
0
 def test_multigraph_with_keys(self):
     G = nx.MultiGraph()
     nx.add_cycle(G, [0, 1, 2, 3])
     G.add_edge(1, 2)
     G.add_edge(1, 2)
     edges = list(eulerian_circuit(G, source=0, keys=True))
     nodes = [u for u, v, k in edges]
     assert_equal(nodes, [0, 3, 2, 1, 2, 1])
     assert_equal(edges[:2], [(0, 3, 0), (3, 2, 0)])
     assert_count_equal(edges[2:5], [(2, 1, 0), (1, 2, 1), (2, 1, 2)])
     assert_equal(edges[5:], [(1, 0, 0)])
Beispiel #14
0
def subs_eq(got, expected, any_order=False):
    """
    Compare Subtitle objects using vars() so that differences are easy to
    identify.
    """
    got_vars = [vars(sub) for sub in got]
    expected_vars = [vars(sub) for sub in expected]
    if any_order:
        assert_count_equal(got_vars, expected_vars)
    else:
        eq(got_vars, expected_vars)
Beispiel #15
0
def subs_eq(got, expected, any_order=False):
    '''
    Compare Subtitle objects using vars() so that differences are easy to
    identify.
    '''
    got_vars = [vars(sub) for sub in got]
    expected_vars = [vars(sub) for sub in expected]
    if any_order:
        assert_count_equal(got_vars, expected_vars)
    else:
        eq(got_vars, expected_vars)
Beispiel #16
0
def test_output_loading():
    """OutputDispatcher - Loading Output Classes"""
    loaded_outputs = set(StreamAlertOutput.get_all_outputs())
    # Add new outputs to this list to make sure they're loaded properly
    expected_outputs = {
        'aws-firehose', 'aws-lambda', 'aws-s3', 'aws-ses', 'aws-sns',
        'aws-sqs', 'aws-cloudwatch-log', 'carbonblack', 'demisto', 'github',
        'jira', 'komand', 'pagerduty', 'pagerduty-v2', 'pagerduty-incident',
        'phantom', 'slack', 'teams'
    }
    assert_count_equal(loaded_outputs, expected_outputs)
Beispiel #17
0
 def test(self, parse):
     zk_dict = {
         'zk0': {'Addr': 'zkv4', 'L4Port': 2181},
         'zk1': {'Addr': 'zkv6', 'L4Port': 2182},
     }
     inst = Topology()
     parse.side_effect = lambda x: x
     # Call
     inst._parse_zk_dicts({"ZookeeperService": zk_dict})
     # Tests
     ntools.assert_count_equal(inst.zookeepers,
                               ["[zkv4]:2181", "[zkv6]:2182"])
Beispiel #18
0
    def test(self, router):
        def _mk_router(type_):
            m = create_mock(["interface"])
            m.interface = create_mock(["link_type"])
            m.interface.link_type = type_
            routers[type_].append(m)
            return m

        routers = defaultdict(list)
        router_dict = {
            "br-parent": "PARENT",
            "br-child": "CHILD",
            "br-peer": "PEER",
            "br-core0": "CORE",
            "br-core1": "CORE",
        }
        inst = Topology()
        router.side_effect = lambda v, k: _mk_router(v)
        # Call
        inst._parse_router_dicts({"BorderRouters": router_dict})
        # Tests
        ntools.assert_count_equal(inst.parent_border_routers,
                                  routers["PARENT"])
        ntools.assert_count_equal(inst.child_border_routers, routers["CHILD"])
        ntools.assert_count_equal(inst.peer_border_routers, routers["PEER"])
        ntools.assert_count_equal(inst.core_border_routers, routers["CORE"])
Beispiel #19
0
 def test(self, router):
     def _mk_router(type_):
         m = create_mock(["interfaces"])
         m.interfaces = {0: create_mock(["link_type"])}
         m.interfaces[0].link_type = type_
         routers[type_].append(m)
         return m
     routers = defaultdict(list)
     router_dict = {"br-parent": "parent"}
     inst = Topology()
     router.side_effect = lambda v, k: _mk_router(v)
     # Call
     inst._parse_router_dicts({"BorderRouters": router_dict})
     # Tests
     ntools.assert_count_equal(inst.border_routers, routers["parent"])
Beispiel #20
0
def test_topological_sort(topological_sort1, app1, app2, depends_on1, bash2,
                          depends_on_job_id1, func_name):
    nt.assert_count_equal(
        list(dag_tools.topological_sort(dag_tools.get_parents(
            topological_sort1, depends_on_job_id1, True,))),
        [
            (app1, '20140601_101_profile-%s' % func_name, u'dep1'),
            (app1, '20140601_102_profile-%s' % func_name, u'dep1'),
            (app2, '20140601_101_profile-%s' % func_name, u'dep1'),
            (app2, '20140601_102_profile-%s' % func_name, u'dep1'),
            (depends_on1, u'20140601_testID1-%s' % func_name, u'dep1'),
            (bash2, '20140601_101_profile-%s' % func_name, u'dep1'),
            (bash2, '20140601_102_profile-%s' % func_name, u'dep1')
        ]
    )
Beispiel #21
0
    def test_eval(self):
        """L5PC: test evaluation of l5pc evaluator"""

        result = self.l5pc_evaluator.evaluate_with_dicts(
            param_dict=release_parameters)

        expected_results = load_from_json('expected_results.json')

        # Use two lines below to update expected result
        # expected_results['TestL5PCEvaluator.test_eval'] = result
        # dump_to_json(expected_results, 'expected_results.json')

        try:
            nt.assert_count_equal(
                result, expected_results['TestL5PCEvaluator.test_eval'])
        except AttributeError:
            nt.assert_items_equal(
                result, expected_results['TestL5PCEvaluator.test_eval'])
Beispiel #22
0
    def test_auto_finder(self):

        data = self.data
        tr = self.gen_repo(data)

        # test get
        for id in data:
            obj = tr.get(id)
            self.check_obj_against_data(obj)

        # check by user
        user_res = tr.by_user_id(22)
        for id, obj in user_res.items():
            self.check_obj_against_data(obj)

        nt.assert_count_equal(user_res, [1, 2])

        user_res = tr.by_user_id(25)
        nt.assert_count_equal(user_res, [4])
Beispiel #23
0
    def test_eval(self):
        """L5PC: test evaluation of l5pc evaluator"""

        result = self.l5pc_evaluator.evaluate_with_dicts(
            param_dict=release_parameters)

        expected_results = load_from_json('expected_results.json')

        # Use two lines below to update expected result
        # expected_results['TestL5PCEvaluator.test_eval'] = result
        # dump_to_json(expected_results, 'expected_results.json')

        try:
            nt.assert_count_equal(
                result,
                expected_results['TestL5PCEvaluator.test_eval'])
        except AttributeError:
            nt.assert_items_equal(
                result,
                expected_results['TestL5PCEvaluator.test_eval'])
Beispiel #24
0
def test_merge_required_outputs_dne():
    """Shared - Merge Required Outputs, Does Not Exist"""
    # A simple user config that will be merged with required outputs
    users_config = {
        'aws-s3': {
            'bucket': 'my.s3.bucket'
        },
        'aws-sns': {
            'topic': 'my-sns-topic'
        },
        'slack': ['slack_output']
    }

    outputs = resources.merge_required_outputs(users_config, "test")

    assert_equal(len(outputs), 4)

    expected_fh = {'alerts': 'test_streamalert_alert_delivery'}

    assert_count_equal(outputs['aws-firehose'], expected_fh)
Beispiel #25
0
def test_expanded_multi_nested_partial():
    # we are expecting these to match by execution context
    ns = {'test1':0, 'test2': 1}
    leaf = _manifest("(test1 + test2)", ns)

    ns = {'x':1, 'y': leaf}
    xy = _manifest("(x + y)", ns)

    ns = {'a': 1, 'b': xy}
    sub = _manifest("(a + b)", ns)

    parent_ns = {'e': 3, 'a': sub}
    parent = _manifest("e + a", parent_ns)

    expanded = parent.expand()
    nt.assert_count_equal(expanded.context.keys(),
                          ['a', 'e', 'x', 'test1', 'test2'])
    nt.assert_equal(expanded.expression.get_source(),
                    "(e + (a + (x + (test1 + test2))))")
    nt.assert_equal(expanded.eval(), 6)
    async def test(self) -> None:
        for i in range(0, 8, 4):
            for j in range(2):
                await self.add_chunks((j, i))
        chunk_info = await self.r.get_chunk_info()

        expected_calls = []
        for i in range(0, 8, 4):
            for j in range(2):
                for k in range(i, i + 4, 2):
                    expected_calls.append(mock.call(
                        'prefix/weights', np.s_[j:j+1, k:k+2, 0:2], mock.ANY))
        for i in range(0, 8, 2):
            expected_calls.append(mock.call(
                'prefix/weights_channel', np.s_[0:2, i:i+2], mock.ANY))
        assert_count_equal(expected_calls, self.chunk_store.put_chunk.mock_calls)
        # Check the array values. assert_count_equal doesn't work well for this
        # because of how equality operators are implemented in numpy.
        for call in self.chunk_store.put_chunk.mock_calls:
            name, slices, value = call[1]
            if name == 'prefix/weights':
                np.testing.assert_array_equal(self.weights[slices], value)
            else:
                np.testing.assert_array_equal(self.weights_channel[slices], value)

        assert_equal(
            chunk_info,
            {
                'weights': {
                    'prefix': 'prefix',
                    'chunks': ((1, 1), (2, 2, 2, 2), (2,)),
                    'shape': (2, 8, 2),
                    'dtype': '|u1'
                },
                'weights_channel': {
                    'prefix': 'prefix',
                    'chunks': ((2,), (2, 2, 2, 2)),
                    'shape': (2, 8),
                    'dtype': np.dtype(np.float32).str
                }
            })
 def test_expand_action_with_expansion_for_prefix_used_multiple_times(self):
     expanded_actions = expand_action("ses:Describe*")
     assert_count_equal(
         expanded_actions,
         [
             {
                 "service": "ses",
                 "action": "DescribeActiveReceiptRuleSet"
             },
             {
                 "service": "ses",
                 "action": "DescribeConfigurationSet"
             },
             {
                 "service": "ses",
                 "action": "DescribeReceiptRule"
             },
             {
                 "service": "ses",
                 "action": "DescribeReceiptRuleSet"
             },
         ],
     )
Beispiel #28
0
    def test_expression(self):
        source = """
        arr = np.arange(20)
        res = np.sum(arr)
        """
        source = dedent(source)
        lines = source.strip().split('\n')
        load_names = [['np'], ['np', 'arr']]
        for i, line in enumerate(lines):
            code = ast.parse(line, '<>', 'exec')

            # expression must be evaluable, assignments are not
            with nt.assert_raises(Exception):
                Expression(code.body[0])

            extracted_expr = grab_expression_from_assign(code)
            # skip the assign
            base_expr = ast.parse(line.split('=')[1].strip(), mode='eval')
            exp1 = Expression(extracted_expr)
            exp2 = Expression(base_expr)
            nt.assert_equal(exp1, exp2)
            nt.assert_is_not(exp1, exp2)
            nt.assert_count_equal(exp1.load_names(), load_names[i])
Beispiel #29
0
def test_get_children(func_name, app1, app2, app4, depends_on1,
                      depends_on2, bash1, bash2):

    nt.assert_count_equal(
        list(dag_tools.get_children(
            depends_on2, '20140601_876_profile-%s' % func_name)),
        [(depends_on1, u'20140601_testID2-%s' % func_name, u'depgrp2')])

    nt.assert_count_equal(
        list(dag_tools.get_children(bash2, '20140601_9899_purchase')),
        []
    )

    nt.assert_count_equal(
        list(dag_tools.get_children(bash1, '20140601_9899_purchase')),
        [(bash2, '20140601_9899_purchase', 'default')]
    )

    nt.assert_count_equal(
        list(dag_tools.get_children(
            app1, '20140601_999_purchase-%s' % func_name)),
        [
            (depends_on1, u'20140601_testID1-%s' % func_name, u'depgrp1'),
            (app2, '20140601_999_purchase-%s' % func_name, 'default'),
            (app4, '20140601_999_purchase-%s' % func_name, 'default'),
        ]
    )

    nt.assert_count_equal(
        list(dag_tools.get_children(
            app1, '20140601_876_purchase-%s' % func_name)),
        [
            (depends_on1, u'20140601_testID1-%s' % func_name, u'depgrp1'),
            (app2, '20140601_876_purchase-%s' % func_name, 'default'),
            (app4, '20140601_876_purchase-%s' % func_name, 'default'),
        ]
    )
Beispiel #30
0
    def test_save(self):
        indexer = RepoFinder(None, {'user_id':{}, 'product_id': {}})
        indexer.save(Obj(1, 123, 4))
        indexer.save(Obj(3, 123, 5))
        # only saving objects for one user
        user_id_index = indexer.indexers['user_id']
        nt.assert_count_equal(user_id_index, [123])
        # get obj for user, check ids
        obj_for_user = user_id_index[123]
        nt.assert_count_equal(obj_for_user, [1,3])

        # saved obj for two products
        product_id_index = indexer.indexers['product_id']
        nt.assert_count_equal(product_id_index, [4,5])

        objs_for_product = product_id_index[4]
        nt.assert_count_equal(objs_for_product, [1])
Beispiel #31
0
def test_merge_required_outputs_exists():
    """Shared - Merge Required Outputs, Has Existing"""
    # A simple user config with an exist aws-firehose output
    # that will be merged with required outputs
    users_config = {
        'aws-firehose': {
            'notalerts': 'resource_name'
        },
        'aws-sns': {
            'topic': 'my-sns-topic'
        },
        'slack': ['slack_output']
    }

    outputs = resources.merge_required_outputs(users_config, "test")

    assert_equal(len(outputs), 3)

    expected_fh = {
        'notalerts': 'resource_name',
        'alerts': 'test_streamalert_alert_delivery'
    }

    assert_count_equal(outputs['aws-firehose'], expected_fh)
Beispiel #32
0
def test_fan_out_tasks(app1, app2, app4, fanout1, func_name):
    # test for Many-to-Many relationships between parent and child tasks
    nt.assert_count_equal(
        list(dag_tools.get_parents(
            'test_stolos/test_fan_out_tasks/fanout1', '20140715_8')),
        [])

    nt.assert_count_equal(
        list(dag_tools.get_parents(
            'test_stolos/test_fan_out_tasks/fanout1',
            '20140715_testID5-%s' % func_name, True)),
        [
            (app1, '20140714_555_profile-%s' % func_name, u'dep2'),
            (app1, '20140715_555_profile-%s' % func_name, u'dep2'),
        ])

    nt.assert_count_equal(
        list(dag_tools.get_children(
            'test_stolos/test_fan_out_tasks/app1',
            '20140715_9_profile-%s' % func_name, True,)),
        [(app2, '20140715_9_profile-%s' % func_name, 'default'),
         (app4, '20140715_9_profile-%s' % func_name, 'default'),
         (fanout1, '20140715_testID1-%s' % func_name, u'dep1'),
         (fanout1, '20140715_testID2-%s' % func_name, u'dep1'),
         (fanout1, '20140715_testID3-%s' % func_name, u'dep1'),
         ])

    nt.assert_count_equal(
        list(dag_tools.get_children(
            app1, '20140715_555_profile-%s' % func_name, True,)),
        [
            (app2, '20140715_555_profile-%s' % func_name, 'default'),
            (app4, '20140715_555_profile-%s' % func_name, 'default'),
            (fanout1, u'20140714_testID5-%s' % func_name, u'dep2'),
            (fanout1, u'20140714_testID6-%s' % func_name, u'dep2'),
            (fanout1, u'20140715_testID1-%s' % func_name, u'dep1'),
            (fanout1, u'20140715_testID2-%s' % func_name, u'dep1'),
            (fanout1, u'20140715_testID3-%s' % func_name, u'dep1'),
            (fanout1, u'20140715_testID5-%s' % func_name, u'dep2'),
            (fanout1, u'20140715_testID6-%s' % func_name, u'dep2'),
        ])
 def test_expand_action_with_casing(self):
     expanded_actions = expand_action("iAm:li*sTuS*rs")
     assert_count_equal(expanded_actions, [{
         "service": "iam",
         "action": "ListUsers"
     }])
 def test_expand_action_with_expansion(self):
     expanded_actions = expand_action("s3:listallmybucke*")
     assert_count_equal(expanded_actions, [{
         "service": "s3",
         "action": "ListAllMyBuckets"
     }])
Beispiel #35
0
 def test_required_auth_info(self):
     """SalesforceApp - Required Auth Info"""
     assert_count_equal(list(self._app.required_auth_info().keys()), {
         'client_id', 'client_secret', 'username', 'password',
         'security_token'
     })
def test_discover_fields():
    # can find simple list of properties
    assert_count_equal(panoptes._discover_fields({}), [])
    assert_count_equal(panoptes._discover_fields({'f1': 'v1'}), ['f1'])
    assert_count_equal(panoptes._discover_fields({
        'f1': 'v1',
        'f2': 'v2'
    }), ['f1', 'f2'])
    assert_count_equal(
        panoptes._discover_fields({
            'f1': 'v1',
            'destination': 'd1',
            'f2': 'v2'
        }), ['f1', 'f2'])

    # ignores known fields
    assert_count_equal(panoptes._discover_fields({'destination': 'v1'}), [])
    assert_count_equal(
        panoptes._discover_fields({
            'destination': 'v1',
            'f2': 'v2'
        }), ['f2'])
Beispiel #37
0
def test_build_dag():
    dag = api.build_dag()
    nt.assert_is_instance(dag, MultiDiGraph)
    tc = api.get_tasks_config()
    nt.assert_count_equal(tc.keys(), dag.node.keys())
Beispiel #38
0
def test_get_parents(app1, app2, depends_on1, depends_on2, bash1, bash2,
                     depends_on_job_id1, func_name):

    # test case with no parents
    nt.assert_equal(
        list(dag_tools.get_parents(app1, '20140101_876_purchase', True)),
        []
    )

    # test the basic inheritance scenario
    nt.assert_count_equal(
        list(dag_tools.get_parents(bash2, '20140501_876_profile', True)),
        [(bash1, '20140501_876_profile', 'default')]
    )

    # test invalid job_id
    nt.assert_count_equal(
        list(dag_tools.get_parents(depends_on1, '20140101_999999', True)),
        []
    )

    # test invalid metadata in job_id
    nt.assert_count_equal(
        list(dag_tools.get_parents(depends_on1, '20140601_999', True)),
        []
    )

    # test depends_on for one of the dependency groups
    nt.assert_count_equal(
        list(dag_tools.get_parents(
            depends_on1, '20140601_testID2-%s' % func_name, True)),
        [
            (depends_on2, '20140601_1011_profile-%s' % func_name, u'depgrp2'),
            (depends_on2, '20140601_9020_profile-%s' % func_name, u'depgrp2'),
            (depends_on2, '20140601_876_profile-%s' % func_name, u'depgrp2')
        ])

    # test depends_on for one of the dependency groups
    # also tests that get_parents returns a stable ordering
    nt.assert_count_equal(
        list(dag_tools.get_parents(depends_on1, depends_on_job_id1, True)),
        [
            (app1, '20140601_1011_profile-%s' % func_name, u'depgrp1'),
            (app1, '20140601_1011_purchase-%s' % func_name, u'depgrp1'),
            (app1, '20140601_9020_profile-%s' % func_name, u'depgrp1'),
            (app1, '20140601_9020_purchase-%s' % func_name, u'depgrp1'),
            (app1, '20140601_876_profile-%s' % func_name, u'depgrp1'),
            (app1, '20140601_876_purchase-%s' % func_name, u'depgrp1'),
            (app1, '20140601_999_purchase-%s' % func_name, u'depgrp1'),
            (app2, '20140601_1011_profile-%s' % func_name, u'depgrp1'),
            (app2, '20140601_1011_purchase-%s' % func_name, u'depgrp1'),
            (app2, '20140601_9020_profile-%s' % func_name, u'depgrp1'),
            (app2, '20140601_9020_purchase-%s' % func_name, u'depgrp1'),
            (app2, '20140601_876_profile-%s' % func_name, u'depgrp1'),
            (app2, '20140601_876_purchase-%s' % func_name, u'depgrp1')
        ]
    )

    # test depends_on when multiple dependency groups map to the same job_id
    # I guess it's okay if they map to the same id?
    nt.assert_count_equal(
        list(dag_tools.get_parents(
            depends_on1, '20140601_testID3-%s' % func_name, True)),
        [(app1, '20140601_444_profile-%s' % func_name, u'depgrp4'),
         (app1, '20140601_876_profile-%s' % func_name, u'depgrp3'),
         ]
    )

    # test the filter_deps option
    nt.assert_count_equal(
        list(dag_tools.get_parents(
            depends_on1, '20140601_testID3-%s' % func_name, True,
            filter_deps=['depgrp4'])),
        [(app1, '20140601_444_profile-%s' % func_name, u'depgrp4')]
    )

    with nt.assert_raises(exceptions.DAGMisconfigured):
        list(dag_tools.get_parents(
            depends_on1, '20140601_testID3-%s' % func_name, True,
            filter_deps=['depgrp99999']))
Beispiel #39
0
 def test_required_auth_info(self):
     """AliyunApp - Required Auth Info"""
     assert_count_equal(list(self._app.required_auth_info().keys()),
                        {'access_key_id', 'access_key_secret', 'region_id'})
def test_flatten():
    # makes any combination of lists and not-lists into a single flat list
    assert_count_equal(list(panoptes._flatten([])), [])
    assert_count_equal(list(panoptes._flatten([1])), [1])
    assert_count_equal(list(panoptes._flatten([1, 2])), [1, 2])
    assert_count_equal(list(panoptes._flatten([1, [2]])), [1, 2])
    assert_count_equal(list(panoptes._flatten([[1], 2])), [1, 2])
    assert_count_equal(list(panoptes._flatten([[1], [2]])), [1, 2])
    assert_count_equal(list(panoptes._flatten([1, [2], 3])), [1, 2, 3])
    assert_count_equal(list(panoptes._flatten([1, [2], 2])), [1, 2, 2])
    assert_count_equal(list(panoptes._flatten([[1], 2, []])), [1, 2])
    assert_count_equal(list(panoptes._flatten([[1], [], 2])), [1, 2])