Пример #1
0
def sagemaker_session_stopped():
    boto_mock = Mock(name='boto_session')
    boto_mock.client('logs').describe_log_streams.return_value = DEFAULT_LOG_STREAMS
    boto_mock.client('logs').get_log_events.side_effect = DEFAULT_LOG_EVENTS
    ims = sagemaker.Session(boto_session=boto_mock, sagemaker_client=Mock())
    ims.sagemaker_client.describe_training_job.return_value = STOPPED_DESCRIBE_JOB_RESULT
    return ims
Пример #2
0
def test_ecr_login_needed(check_output):
    session_mock = Mock()

    token = 'very-secure-token'
    token_response = 'AWS:%s' % token
    b64_token = base64.b64encode(token_response.encode('utf-8'))
    response = {
        u'authorizationData':
            [
                {
                    u'authorizationToken': b64_token,
                    u'proxyEndpoint': u'https://520713654638.dkr.ecr.us-east-1.amazonaws.com'
                }
            ],
        'ResponseMetadata':
            {
                'RetryAttempts': 0,
                'HTTPStatusCode': 200,
                'RequestId': '25b2ac63-36bf-11e8-ab6a-e5dc597d2ad9',
            }
    }
    session_mock.client('ecr').get_authorization_token.return_value = response
    image = '520713654638.dkr.ecr.us-east-1.amazonaws.com/image-i-need:1.1'
    sagemaker.local.image._ecr_login_if_needed(session_mock, image)

    expected_command = 'docker login -u AWS -p %s https://520713654638.dkr.ecr.us-east-1.amazonaws.com' % token

    check_output.assert_called_with(expected_command, shell=True)
    session_mock.client('ecr').get_authorization_token.assert_called_with(registryIds=['520713654638'])
Пример #3
0
def sagemaker_session_full_lifecycle():
    boto_mock = Mock(name='boto_session')
    boto_mock.client('logs').describe_log_streams.side_effect = LIFECYCLE_LOG_STREAMS
    boto_mock.client('logs').get_log_events.side_effect = STREAM_LOG_EVENTS
    ims = sagemaker.Session(boto_session=boto_mock, sagemaker_client=Mock())
    ims.sagemaker_client.describe_training_job.side_effect = [IN_PROGRESS_DESCRIBE_JOB_RESULT,
                                                              IN_PROGRESS_DESCRIBE_JOB_RESULT,
                                                              COMPLETED_DESCRIBE_JOB_RESULT]
    return ims
    def test_should_not_include_files_outside_config_directory(self):

        mock_svn_service = Mock(SvnService)
        mock_svn_service.config_url = 'svn://url/for/configuration/repository'
        mock_svn_service.path_to_config = '/config'
        mock_svn_service.client = Mock()
        mock_info = Mock()

        mock_path_object_1 = Mock()
        mock_path_object_1.path = '/config/foo'
        mock_path_object_1.action = 'A'

        mock_path_object_2 = Mock()
        mock_path_object_2.path = '/XXXXXX/bar'
        mock_path_object_2.action = 'A'

        mock_path_object_3 = Mock()
        mock_path_object_3.path = '/XXX/foobar'
        mock_path_object_3.action = 'A'

        mock_info.changed_paths = [mock_path_object_1, mock_path_object_2, mock_path_object_3]
        mock_svn_service.get_logs_for_revision.return_value = [mock_info]

        actual = SvnService.get_changed_paths_with_action(mock_svn_service, '1980')

        self.assertEqual([('foo', 'A')], actual)
    def test_sendEvent_should_use_kwargs_as_event_items(self):
        mock_broadcaster = Mock(WampBroadcaster)
        mock_broadcaster.target = "broadcaster-target"
        mock_broadcaster.logger = Mock()
        mock_broadcaster.client = Mock()

        WampBroadcaster._sendEvent(
            mock_broadcaster,
            "event-id",
            "event-data",
            tracking_id="tracking-id",
            target="target",
            state="foobar",
            bar="baz",
        )

        actual_call = mock_broadcaster.client.publish.call_args
        self.assertEqual(
            call(
                "target",
                {
                    "payload": "event-data",
                    "type": "event",
                    "id": "event-id",
                    "tracking_id": "tracking-id",
                    "target": "target",
                    "state": "foobar",
                    "bar": "baz",
                },
            ),
            actual_call,
        )
Пример #6
0
    def test_readFromBackingErrorHandling(self):
        client = Object
        vos_VOFILE = Object()
        vos_VOFILE.URLs = ["url0", "URL1"]
        vos_VOFILE.urlIndex = 0
        vos_VOFILE.open = Mock()
        mock_resp1 = Mock()
        mock_resp1.iter_content = Mock(return_value="bar")
        mock_resp2 = Mock()
        mock_resp2.iter_content = Mock(return_value="foo")
        returns = [mock_resp1, mock_resp2]

        def side_effect(*args, **kwds):
            return returns.pop(0)
        vos_VOFILE.read = MagicMock(side_effect=side_effect)
        vos_VOFILE.close = Mock()
        client.open = Mock(return_value=vos_VOFILE)
        myVofs = Mock()
        myVofs.client = client
        testProxy = vofs.MyIOProxy(myVofs, None)
        testProxy.writeToCache = Mock()
        path = "/dir1/dir2/file"
        cacheFile = Object()
        cacheFile.path = path
        cacheFile.gotHeader = True
        cacheFile.cache = Object()
        testProxy.setCacheFile(cacheFile)
        testProxy.readFromBacking()
    def test_check_connection_should_return_true_when_link_is_up(self):
        mock_broadcaster = Mock(WampBroadcaster)
        mock_broadcaster.logger = Mock()
        mock_broadcaster.url = "ws://broadcaster"
        mock_broadcaster.client = Mock()

        self.assertEqual(WampBroadcaster._check_connection(mock_broadcaster), True)
        self.assertFalse(hasattr(mock_broadcaster, "not_connected_warning_sent"))
    def test_should_set_the_client_to_none(self, mock_log):
        mock_receiver = Mock(Receiver)
        mock_broadcaster = Mock()
        mock_broadcaster.client = 'Test client'
        mock_receiver.broadcaster = mock_broadcaster

        Receiver.onConnectionLost(mock_receiver, 'Spam eggs.')

        self.assertEquals(None, mock_broadcaster.client)
Пример #9
0
 def test_successfully_adds_kinesis_subscription(self, mocked_session):
     _mocked_lambda = Mock()
     _mocked_session = Mock()
     _mocked_session.client = Mock()
     _mocked_session.client.return_value = _mocked_lambda
     mocked_session.return_value = _mocked_session
     conf = config.Config(path.dirname(__file__),
                          config_file=path.join(EX_CONFIG, 'lambda-with-subscription.json'))
     subscribers.create_subscriptions(conf, None)
     nt.assert_equals(True, _mocked_lambda.create_event_source_mapping.called)
Пример #10
0
    def test_should_raise_exception_when_pysvn_client_fails_to_log(self):

        mock_svn_service = Mock(SvnService)
        mock_svn_service.config_url = 'svn://url/for/configuration/repository/config'
        mock_svn_service.base_url = 'svn://url/for/configuration/repository'
        mock_svn_service.path_to_config = '/config'
        mock_svn_service.client = Mock()
        mock_svn_service.client.log.side_effect = Exception("Aaarrrgggghh...")

        self.assertRaises(SvnServiceException, SvnService.get_logs_for_revision, mock_svn_service, '1980')
Пример #11
0
 def test___init__(self):
     parser = common.CliOptions().create_optparser(False)
     common.AuthedCommandsBase.debug = True
     dbaas = Mock()
     dbaas.authenticate = Mock(return_value=None)
     dbaas.client = Mock()
     dbaas.client.auth_token = Mock()
     dbaas.client.service_url = Mock()
     dbaas.client.authenticate_with_token = Mock()
     common.AuthedCommandsBase._get_client = Mock(return_value=dbaas)
     authed_cmd = common.AuthedCommandsBase(parser)
Пример #12
0
def test_download_file():
    boto_mock = Mock(name='boto_session')
    boto_mock.client('sts').get_caller_identity.return_value = {'Account': '123'}
    bucket_mock = Mock()
    boto_mock.resource('s3').Bucket.return_value = bucket_mock
    session = sagemaker.Session(boto_session=boto_mock, sagemaker_client=Mock())

    sagemaker_container = _SageMakerContainer('local', 2, 'my-image', sagemaker_session=session)
    sagemaker_container._download_file(BUCKET_NAME, '/prefix/path/file.tar.gz', '/tmp/file.tar.gz')

    bucket_mock.download_file.assert_called_with('prefix/path/file.tar.gz', '/tmp/file.tar.gz')
Пример #13
0
    def test_should_return_logs_for_revision(self):
        mock_svn_service = Mock(SvnService)
        mock_svn_service.base_url = 'svn://url/for/configuration/repository'
        mock_svn_service.path_to_config = '/config'
        mock_svn_service.client = Mock()
        mock_logs = Mock()
        mock_svn_service.client.log.return_value = mock_logs

        actual = SvnService.get_logs_for_revision(mock_svn_service, '1980')

        self.assertEqual(mock_logs, actual)
Пример #14
0
def sagemaker_session():
    boto_mock = Mock(name='boto_session', region_name=REGION)
    boto_mock.client('sts').get_caller_identity.return_value = {'Account': '123'}
    boto_mock.resource('s3').Bucket(BUCKET_NAME).objects.filter.return_value = []

    sms = sagemaker.Session(boto_session=boto_mock, sagemaker_client=Mock())

    sms.default_bucket = Mock(name='default_bucket', return_value=BUCKET_NAME)
    sms.expand_role = Mock(return_value=EXPANDED_ROLE)

    return sms
Пример #15
0
    def test_attach(self):
        i = BaseIterator(name="test123", priority=111, classname="com.test.Class")

        it_s = i.get_iterator_setting()
        scopes = set([IteratorScope.SCAN, IteratorScope.MINC, IteratorScope.MAJC])

        conn = Mock()
        conn.client = Mock()
        conn.client.attachIterator = Mock()
        conn.login = "Login"

        i.attach(conn, "mytable123", scopes)
        conn.client.attachIterator.assert_called_with("Login", "mytable123", it_s, scopes)
Пример #16
0
 def test_successfully_updates_kinesis_subscription(self, mocked_session):
     resonse = {"Error": {"Code": "ResourceConflictException", "Message": ""}}
     err = botocore.exceptions.ClientError(resonse, "create_event_source_mapping")
     _mocked_lambda = Mock()
     _mocked_lambda.create_event_source_mapping.side_effect = err
     _mocked_lambda.list_event_source_mappings.return_value = {
         'EventSourceMappings': [{'UUID': 'myuuid'}]
     }
     _mocked_session = Mock()
     _mocked_session.client = Mock()
     _mocked_session.client.return_value = _mocked_lambda
     mocked_session.return_value = _mocked_session
     conf = config.Config(path.dirname(__file__),
                          config_file=path.join(EX_CONFIG, 'lambda-with-subscription.json'))
     subscribers.create_subscriptions(conf, None)
     nt.assert_equals(True, _mocked_lambda.update_event_source_mapping.called)
Пример #17
0
    def test_should_ensure_that_all_returned_host_names_are_ordinary_strings(self):

        mock_svn_service = Mock(SvnService)
        mock_svn_service.config_url = '/path to repository/config'
        mock_svn_service.client = Mock()
        item0 = Mock()
        item0.repos_path = "get_hosts removes the first element - so this will never show up"
        item1 = Mock()
        item1.repos_path = "bar"
        item2 = Mock()
        item2.repos_path = u"spam"
        mock_svn_service.client.list.return_value = [(item0,), (item1,), (item2,)]

        actual_host_names = SvnService.get_hosts(mock_svn_service, 123)

        self.assert_is_ordinary_string(actual_host_names[0])
        self.assert_is_ordinary_string(actual_host_names[1])
Пример #18
0
    def test_login(self):
        self.auth.username = "username"
        self.auth.apikey = "apikey"
        self.auth.tenant_id = "tenant_id"
        self.auth.auth_url = "auth_url"
        dbaas = Mock()
        dbaas.authenticate = Mock(return_value=None)
        dbaas.client = Mock()
        dbaas.client.auth_token = Mock()
        dbaas.client.service_url = Mock()
        self.auth._get_client = Mock(return_value=dbaas)
        self.auth.login()

        self.auth.debug = True
        self.auth._get_client = Mock(side_effect=ValueError)
        self.assertRaises(ValueError, self.auth.login)

        self.auth.debug = False
        self.auth.login()
Пример #19
0
 def testWriteToBacking(self):
     # Submit a write request for the whole file.
     with Cache(TestVOFS.testCacheDir, 100, timeout=1) as testCache:
         client = Object
         client.copy = Mock()
         client.copy = Mock(return_value=12345)
         vofsObj = Mock()
         vofsObj.client = client
         node = Object
         node.uri = "vos:/dir1/dir2/file"
         node.props = {"MD5": 12345}
         vofsObj.get_node = Mock(return_value=node)
         testProxy = vofs.MyIOProxy(vofsObj, None)
         path = "/dir1/dir2/file"
         with FileHandle(path, testCache, testProxy) as testFileHandle:
             testProxy.cacheFile = testFileHandle
             self.assertEqual(testProxy.writeToBacking(), 12345)
         client.copy.assert_called_once_with(
             testCache.dataDir + "/dir1/dir2/file", node.uri, send_md5=True)
Пример #20
0
    def test_should_return_list_with_directory_name_and_action_for_path_to_file_when_a_file_has_been_added(self):

        mock_svn_service = Mock(SvnService)
        mock_svn_service.config_url = 'svn://url/for/configuration/repository'
        mock_svn_service.path_to_config = '/config'
        mock_svn_service.client = Mock()
        mock_info = Mock()
        mock_path_object_1 = Mock()
        mock_path_object_1.path = '/config/'
        mock_path_object_1.action = 'A'
        mock_info.changed_paths = [mock_path_object_1]
        mock_path_object_2 = Mock()
        mock_path_object_2.path = '/config/spam.egg'
        mock_path_object_2.action = 'A'
        mock_info.changed_paths = [mock_path_object_1, mock_path_object_2]
        mock_svn_service.get_logs_for_revision.return_value = [mock_info]

        actual = SvnService.get_changed_paths_with_action(mock_svn_service, '1980')

        self.assertEqual([('', 'A'), ('spam.egg', 'A')], actual)
    def test_sendEvent_should_publish_expected_event_on_default_target(self):
        mock_broadcaster = Mock(WampBroadcaster)
        mock_broadcaster.target = "broadcaster-target"
        mock_broadcaster.logger = Mock()
        mock_broadcaster.client = Mock()

        WampBroadcaster._sendEvent(mock_broadcaster, "event-id", "event-data", tracking_id="tracking-id")

        actual_call = mock_broadcaster.client.publish.call_args
        self.assertEqual(
            call(
                "broadcaster-target",
                {
                    "payload": "event-data",
                    "type": "event",
                    "id": "event-id",
                    "tracking_id": "tracking-id",
                    "target": "broadcaster-target",
                },
            ),
            actual_call,
        )
Пример #22
0
    def test_should_return_list_with_tuples_including_one_tuple_which_has_a_delete_action(self):

        mock_svn_service = Mock(SvnService)
        mock_svn_service.config_url = 'svn://url/for/configuration/repository'
        mock_svn_service.path_to_config = '/config'
        mock_svn_service.client = Mock()
        mock_info = Mock()
        mock_path_object_1 = Mock()
        mock_path_object_1.path = '/config/'
        mock_path_object_1.action = 'A'
        mock_info.changed_paths = [mock_path_object_1]
        mock_path_object_2 = Mock()
        mock_path_object_2.path = '/config/spam.egg'
        mock_path_object_2.action = 'A'
        mock_path_object_3 = Mock()
        mock_path_object_3.path = '/config/foo.bar'
        mock_path_object_3.action = 'D'
        mock_info.changed_paths = [mock_path_object_1, mock_path_object_2, mock_path_object_3]
        mock_svn_service.get_logs_for_revision.return_value = [mock_info]

        actual = SvnService.get_changed_paths_with_action(mock_svn_service, '1980')

        self.assertEqual([('', 'A'), ('spam.egg', 'A'), ('foo.bar', 'D')], actual)
Пример #23
0
def test_download_folder(makedirs):
    boto_mock = Mock(name='boto_session')
    boto_mock.client('sts').get_caller_identity.return_value = {'Account': '123'}

    session = sagemaker.Session(boto_session=boto_mock, sagemaker_client=Mock())

    train_data = Mock()
    validation_data = Mock()

    train_data.bucket_name.return_value = BUCKET_NAME
    train_data.key = '/prefix/train/train_data.csv'
    validation_data.bucket_name.return_value = BUCKET_NAME
    validation_data.key = '/prefix/train/validation_data.csv'

    s3_files = [train_data, validation_data]
    boto_mock.resource('s3').Bucket(BUCKET_NAME).objects.filter.return_value = s3_files

    obj_mock = Mock()
    boto_mock.resource('s3').Object.return_value = obj_mock

    sagemaker_container = _SageMakerContainer('local', 2, 'my-image', sagemaker_session=session)
    sagemaker_container._download_folder(BUCKET_NAME, '/prefix', '/tmp')

    obj_mock.download_file.assert_called()
    calls = [call(os.path.join('/tmp', 'train/train_data.csv')),
             call(os.path.join('/tmp', 'train/validation_data.csv'))]
    obj_mock.download_file.assert_has_calls(calls)
    obj_mock.reset_mock()

    # Testing with a trailing slash for the prefix.
    sagemaker_container._download_folder(BUCKET_NAME, '/prefix/', '/tmp')
    obj_mock.download_file.assert_called()
    calls = [call(os.path.join('/tmp', 'train/train_data.csv')),
             call(os.path.join('/tmp', 'train/validation_data.csv'))]

    obj_mock.download_file.assert_has_calls(calls)
Пример #24
0
    def testReadFromBacking(self):
        callCount = [0]

        def mock_read(block_size):
            callCount[0] += 1
            if callCount[0] == 1:
                return "1234"
            else:
                return None

        with Cache(TestVOFS.testCacheDir, 100, timeout=1) as testCache:
            client = Object
            vos_VOFILE = Object()
            vos_VOFILE.URLs = ["url0", "URL1"]
            vos_VOFILE.urlIndex = 0
            vos_VOFILE.open = Mock()
            vos_VOFILE.read = Mock(side_effect=mock_read)
            vos_VOFILE.close = Mock()
            client.open = Mock(return_value=vos_VOFILE)
            path = "/dir1/dir2/file"
            myVofs = Mock()
            myVofs.cacheFile = Mock()
            myVofs.cacheFile.path = path
            myVofs.client = client
            client.getFileInfo = Mock(return_value=(123, "456", 45))
            testProxy = vofs.MyIOProxy(myVofs, None)

            with FileHandle(path, testCache, testProxy) as \
                    testFileHandle:
                testProxy.writeToCache = Mock(return_value=4)
                testProxy.cacheFile = testFileHandle
                testProxy.cacheFile.readThread = Mock()
                testProxy.cacheFile.readThread.aborted = False
                try:

                    # Submit a request for the whole file
                    testProxy.readFromBacking()
                    client.open.assert_called_once_with(path, mode=os.O_RDONLY,
                                                        view="data", size=None, range=None)
                    self.assertEqual(vos_VOFILE.close.call_count, 1)
                    self.assertEqual(vos_VOFILE.read.call_count, 2)

                    # Submit a range request
                    vos_VOFILE.close.reset_mock()
                    vos_VOFILE.read.reset_mock()
                    callCount[0] = 0
                    testProxy.readFromBacking(100, 200)
                    self.assertEqual(client.open.call_count, 1)
                    vos_VOFILE.open.assert_called_once_with("url0",
                                                            bytes="bytes=200-299")
                    self.assertEqual(vos_VOFILE.close.call_count, 1)
                    self.assertEqual(vos_VOFILE.read.call_count, 2)

                    # Submit a request which gets aborted.
                    vos_VOFILE.open.reset_mock()
                    vos_VOFILE.close.reset_mock()
                    vos_VOFILE.read.reset_mock()
                    callCount[0] = 0
                    testProxy.writeToCache.side_effect = CacheAborted(
                        "aborted")
                    testProxy.readFromBacking(150, 200)
                    self.assertEqual(client.open.call_count, 1)
                    vos_VOFILE.open.assert_called_once_with("url0",
                                                            bytes="bytes=200-349")
                    self.assertEqual(vos_VOFILE.close.call_count, 1)
                    self.assertEqual(vos_VOFILE.read.call_count, 1)

                    # Submit a request with size = None and offset > 0
                    vos_VOFILE.open.reset_mock()
                    vos_VOFILE.close.reset_mock()
                    vos_VOFILE.read.reset_mock()
                    callCount[0] = 0
                    testProxy.readFromBacking(None, 1)
                    self.assertEqual(client.open.call_count, 1)
                    vos_VOFILE.open.assert_called_once_with("url0",
                                                            bytes="bytes=1-")
                    self.assertEqual(vos_VOFILE.close.call_count, 1)
                    self.assertEqual(vos_VOFILE.read.call_count, 1)

                    # Do a read which fails. This should result in a
                    # renegotiation.
                    vos_VOFILE.open.reset_mock()
                    vos_VOFILE.close.reset_mock()
                    vos_VOFILE.read.reset_mock()
                    callCount[0] = 0
                    self.assertTrue(testProxy.lastVOFile is not None)
                    testProxy.lastVOFile.read = Mock(side_effect=OSError)
                    # This throws an exception because read will be called
                    # twice, the first is caught and error handling occurs, the
                    # second is not caught.
                    with self.assertRaises(OSError):
                        testProxy.readFromBacking(None, 1)
                    self.assertEqual(client.open.call_count, 2)
                    vos_VOFILE.open.assert_called_with("url0",
                                                       bytes="bytes=1-")
                    self.assertEqual(vos_VOFILE.close.call_count, 1)
                    self.assertEqual(vos_VOFILE.read.call_count, 2)
                    self.assertTrue(type(testProxy.exception) is OSError)
                    testProxy.exception = None

                except Exception as e:
                    print("unexpected exception", e)

                finally:
                    testProxy.cacheFile.readThread = None
Пример #25
0
def sagemaker_session():
    boto_mock = Mock(name='boto_session')
    boto_mock.client('sts').get_caller_identity.return_value = {'Account': '123'}
    ims = sagemaker.Session(boto_session=boto_mock, sagemaker_client=Mock())
    ims.expand_role = Mock(return_value=EXPANDED_ROLE)
    return ims
def sagemaker_session():
    boto_mock = Mock(name='boto_session', region_name=REGION)
    boto_mock.client('sts').get_caller_identity.return_value = {'Account': ACCOUNT_ID}
    ims = sagemaker.Session(boto_session=boto_mock)
    return ims
import sys
import mock
import unittest
import os
import types
from mock import patch, Mock

docker = Mock()
docker.client = Mock()
sys.modules['docker'] = docker

from cni.kube_cni import kube_params

class DockerClientMock(object):
    def __init__(self):
        pass

    def inspect_container(self, id):
        return {
            'Config': {
                'Labels': {
                    'io.kubernetes.pod.uid': "id" + id
                }
            }
        }

class K8SParamsTest(unittest.TestCase):
    def setUp(self):
        pass

    def tearDown(self):