Exemple #1
0
 def test_delete(self):
     certfile = '/tmp/SomeCert.pem'
     open(certfile, 'w+')
     with patch('os.access'):
         client = Client(vospace_certfile=certfile)
     uri1 = 'vos://cadc.nrc.ca!vospace/nosuchfile1'
     url = 'https://www.canfar.phys.uvic.ca/vospace/nodes/nosuchfile1'
     mock_session = Mock()
     client.get_session = Mock(return_value=mock_session)
     client.delete(uri1)
     mock_session.delete.assert_called_once_with(url)
Exemple #2
0
    def test_init_client(self):
        # No parameters uses cert in ~/.ssl giving authenticated / https
        # create a fake pem file
        certfile = '/tmp/some-cert-file.pem'
        open(certfile, 'w+')
        Client.VOSPACE_CERTFILE = "some-cert-file.pem"
        with patch('os.access'):
            client = Client(vospace_certfile=certfile)
        client.get_session(uri='ivo://cadc.nrc.ca/vault')
        conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn
        self.assertTrue(conn.subject.certificate)
        self.assertFalse(conn.vo_token)

        # Supplying an empty string for certfile implies anonymous / http
        client = Client(vospace_certfile='')
        client.get_session(uri='ivo://cadc.nrc.ca/vault')
        conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn
        self.assertTrue(conn.subject.anon)
        self.assertFalse(conn.vo_token)

        # Specifying a token implies authenticated / http
        client = Client(vospace_token='a_token_string')
        client.get_session(uri='ivo://cadc.nrc.ca/vault')
        conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn
        self.assertTrue(conn.subject.anon)
        self.assertTrue(conn.vo_token)

        # Specifying both a certfile and token implies token (auth) / http
        with patch('os.access'):
            client = Client(vospace_certfile=certfile,
                            vospace_token='a_token_string')
        client.get_session(uri='ivo://cadc.nrc.ca/vault')
        conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn
        self.assertTrue(conn.subject.anon)
        self.assertTrue(conn.vo_token)

        # update auth for specific service
        with patch('os.access'):
            client.set_auth(uri='ivo://cadc.nrc.ca/vault',
                            vospace_certfile=certfile)
        conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn
        self.assertTrue(conn.subject.certificate)
        self.assertFalse(conn.vo_token)
Exemple #3
0
    def test_create(self):
        uri = 'vos://create.vospace.auth!vospace/bar'
        client = Client()
        node = Node(client.fix_uri(uri))
        node2 = Node(str(node))
        self.assertEqual(node, node2)
        data = str(node)
        headers = {'size': str(len(data))}

        client = Client()
        # client.get_node_url = Mock(return_value='http://foo.com/bar')
        session_mock = MagicMock()
        client.get_session = Mock(return_value=session_mock)
        session_mock.put.return_value = Mock(content=str(node))

        result = client.create(uri)
        self.assertEqual(node, result)
        session_mock.put.assert_called_with(
            'http://www.canfar.phys.uvic.ca/vospace/nodes/bar',
            headers=headers, data=data)
Exemple #4
0
    def test_add_props(self):
        old_node = Node(ElementTree.fromstring(NODE_XML))
        old_node.uri = 'vos:sometest'
        new_node = Node(ElementTree.fromstring(NODE_XML))
        new_node.props['quota'] = '1000'
        new_node.create = Mock(return_value=new_node.node)

        data = str(new_node)
        headers = {'size': str(len(data))}

        client = Client()
        client.get_node = Mock(return_value=old_node)
        client.get_node_url = Mock(return_value='http://foo.com/bar')
        mock_session = Mock()
        client.get_session = Mock(return_value=mock_session)

        client.add_props(new_node)
        client.get_session.assert_called_with('vos://foo.com!vospace/bar')
        mock_session.post.assert_called_with('http://foo.com/bar',
                                             headers=headers,
                                             data=data)
Exemple #5
0
    def test_update(self):
        node = Node(ElementTree.fromstring(NODE_XML))

        resp = Mock()
        resp.headers.get = Mock(
            return_value="https://www.canfar.phys.uvic.ca/vospace")

        session = Mock(spec=vos.Connection)
        session.post = Mock(return_value=resp)
        client = Client()
        client.get_session = Mock(return_value=session)
        client.get_node_url = Mock(
            return_value='https://www.canfar.phys.uvic.ca/vospace')
        client.get_transfer_error = Mock()
        client.protocol = 'https'

        data = str(node)
        property_url = 'https://www.canfar.phys.uvic.ca/vospace/nodeprops'
        endpoints_mock = Mock()
        endpoints_mock.properties = property_url
        client.get_endpoints = Mock(return_value=endpoints_mock)
        result = client.update(node, False)
        self.assertEqual(result, 0)
        session.post.assert_called_with(
            'https://www.canfar.phys.uvic.ca/vospace',
            data=data, allow_redirects=False)

        call1 = call(property_url, allow_redirects=False, data=data,
                     headers={'Content-type': 'text/xml'})
        call2 = call('https://www.canfar.phys.uvic.ca/vospace/phase',
                     allow_redirects=False, data="PHASE=RUN",
                     headers={
                         'Content-type': 'application/x-www-form-urlencoded'})
        calls = [call1, call2]

        session.post = Mock(return_value=resp)
        result = client.update(node, True)
        self.assertEqual(result, 0)
        session.post.assert_has_calls(calls)
Exemple #6
0
    def test_copy(self, computed_md5_mock):
        file_content = 'File content'.encode('utf-8')

        # the md5sum of the file being copied
        transfer_md5 = hashlib.md5()
        transfer_md5.update(file_content)
        md5sum = transfer_md5.hexdigest()
        # patch the compute_md5 function in vos to return the above value
        computed_md5_mock.return_value = md5sum

        # mock the props of the corresponding node
        props = MagicMock()
        props.get.return_value = md5sum
        # add props to the mocked node
        node = MagicMock(spec=Node)
        node.props = {'MD5': md5sum, 'length': 12}

        # mock one by one the chain of connection.session.response.headers
        session = MagicMock()
        response = MagicMock()
        headers = MagicMock()
        headers.get.return_value = md5sum
        response.headers = headers
        session.get.return_value = response
        response.iter_content.return_value = BytesIO(file_content)

        test_client = Client()
        test_client.get_session = Mock(return_value=session)
        # use the mocked connection instead of the real one
        get_node_url_mock = Mock(
            return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
        test_client.get_node_url = get_node_url_mock
        mock_update = Mock()
        test_client.update = mock_update

        # patch Client.get_node to return our mocked node
        get_node_mock = Mock(return_value=node)
        test_client.get_node = get_node_mock

        # time to test...
        vospaceLocation = 'vos://test/foo'
        osLocation = '/tmp/foo'
        if os.path.isfile(osLocation):
            os.remove(osLocation)
        # copy from vospace
        test_client.copy(vospaceLocation, osLocation)
        get_node_url_mock.assert_called_once_with(vospaceLocation,
                                                  method='GET',
                                                  cutout=None, view='data')
        assert not computed_md5_mock.called,\
            'MD5 should be computed on the fly'
        assert get_node_mock.called

        # repeat - local file and vospace file are now the same -> only
        # get_node is called to get the md5 of remote file
        get_node_url_mock.reset_mock()
        computed_md5_mock.reset_mock()
        get_node_mock.reset_mock()
        props.reset_mock()
        props.get.return_value = md5sum
        test_client.copy(vospaceLocation, osLocation)
        assert not get_node_url_mock.called
        computed_md5_mock.assert_called_once_with(osLocation)
        get_node_mock.assert_called_once_with(vospaceLocation, force=True)

        # change the content of local files to trigger a new copy
        get_node_url_mock.reset_mock()
        get_node_mock.reset_mock()

        computed_md5_mock.reset_mock()
        computed_md5_mock.return_value = 'd002233'
        response.iter_content.return_value = BytesIO(file_content)
        test_client.copy(vospaceLocation, osLocation)
        get_node_url_mock.assert_called_once_with(vospaceLocation,
                                                  method='GET',
                                                  cutout=None, view='data')
        computed_md5_mock.assert_called_with(osLocation)
        get_node_mock.assert_called_once_with(vospaceLocation, force=True)

        # change the content of local files to trigger a new copy
        get_node_url_mock.reset_mock()
        get_node_url_mock.return_value = \
            ['https://mysite.com/node/node123/cutout']
        computed_md5_mock.reset_mock()
        computed_md5_mock.return_value = 'd002233'
        # computed_md5_mock.side_effect = ['d002233', md5sum]
        get_node_mock.reset_mock()
        response.iter_content.return_value = BytesIO(file_content)
        session.get.return_value = response
        test_client.get_session = Mock(return_value=session)
        test_client.copy('{}{}'.format(vospaceLocation,
                                       '[1][10:60]'), osLocation)
        get_node_url_mock.assert_called_once_with(
            vospaceLocation, method='GET', cutout='[1][10:60]', view='cutout')

        # copy to vospace when md5 sums are the same -> only update occurs
        get_node_url_mock.reset_mock()
        computed_md5_mock.reset_mock()
        computed_md5_mock.side_effect = None
        computed_md5_mock.return_value = md5sum
        test_client.copy(osLocation, vospaceLocation)
        mock_update.assert_called_once()
        assert not get_node_url_mock.called

        # make md5 different on destination
        get_node_url_mock.reset_mock()
        get_node_url_mock.return_value =\
            ['http://cadc.ca/test', 'http://cadc.ca/test']
        computed_md5_mock.reset_mock()
        mock_update.reset_mock()
        computed_md5_mock.return_value = md5sum
        to_update_node = MagicMock()
        to_update_node.props = {'MD5': 'abcde', 'length': 12}
        test_client.get_node = Mock(side_effect=[to_update_node, node])
        test_client.copy(osLocation, vospaceLocation)
        assert not mock_update.called
        get_node_url_mock.assert_called_once_with(vospaceLocation, 'PUT',
                                                  content_length=12,
                                                  md5_checksum='abcde')
        computed_md5_mock.assert_called_once_with(osLocation)

        # copy 0 size file -> delete and create node but no bytes
        # transferred
        get_node_url_mock.reset_mock()
        computed_md5_mock.reset_mock()
        test_client.get_node = Mock(return_value=node)
        node.props['length'] = 0
        mock_delete = Mock()
        mock_create = Mock()
        test_client.delete = mock_delete
        test_client.create = mock_create
        with patch('vos.vos.os.stat', Mock()) as stat_mock:
            stat_mock.return_value = Mock(st_size=0)
            test_client.copy(osLocation, vospaceLocation)
        mock_create.assert_called_once_with(vospaceLocation)
        mock_delete.assert_called_once_with(vospaceLocation)
        assert not get_node_url_mock.called

        # copy new 0 size file -> create node but no bytes transferred
        get_node_url_mock.reset_mock()
        computed_md5_mock.reset_mock()
        mock_delete.reset_mock()
        mock_create.reset_mock()
        test_client.get_node = Mock(side_effect=[Exception(), node])
        mock_delete = Mock()
        mock_create = Mock()
        test_client.delete = mock_delete
        test_client.create = mock_create
        with patch('vos.vos.os.stat', Mock()) as stat_mock:
            stat_mock.return_value = Mock(st_size=0)
            test_client.copy(osLocation, vospaceLocation)
        mock_create.assert_called_once_with(vospaceLocation)
        assert not mock_delete.called
        assert not get_node_url_mock.called

        # error tests - md5sum mismatch
        node.props['length'] = 12
        computed_md5_mock.return_value = '000bad000'
        test_client.get_node = Mock(return_value=node)
        with self.assertRaises(OSError):
            test_client.copy(vospaceLocation, osLocation)

        # existing file
        mock_delete.reset_mock()
        with self.assertRaises(OSError):
            with patch('vos.vos.os.stat', Mock()) as stat_mock:
                stat_mock.return_value = Mock(st_size=12)
                test_client.copy(osLocation, vospaceLocation)
        assert not mock_delete.called  # server takes care of cleanup

        # new file
        mock_delete.reset_mock()
        with self.assertRaises(OSError):
            with patch('vos.vos.os.stat', Mock()) as stat_mock:
                stat_mock.return_value = Mock(st_size=12)
                node.props['MD5'] = None
                test_client.copy(osLocation, vospaceLocation)
        assert mock_delete.called  # cleanup required

        # requests just the headers when md5 not provided in the header
        props.get.side_effect = [None]
        get_node_url_mock = Mock(
            return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
        test_client.get_node_url = get_node_url_mock
        get_node_mock.reset_mock()
        headers.get.return_value = None
        test_client.copy(vospaceLocation, osLocation, head=True)
        get_node_url_mock.assert_called_once_with(vospaceLocation,
                                                  method='GET',
                                                  cutout=None, view='header')

        # repeat headers request when md5 provided in the header
        props.get.side_effect = md5sum
        get_node_url_mock = Mock(
            return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
        test_client.get_node_url = get_node_url_mock
        get_node_mock.reset_mock()
        response.iter_content.return_value = BytesIO(file_content)
        headers.get.return_value = None
        test_client.copy(vospaceLocation, osLocation, head=True)
        get_node_url_mock.assert_called_once_with(vospaceLocation,
                                                  method='GET',
                                                  cutout=None, view='header')