def test_update(self):
node = Node(ElementTree.fromstring(NODE_XML))
resp = Mock()
resp.headers.get = Mock(return_value="https://www.canfar.phys.uvic.ca/vospace")
client = Client()
client.get_node_url = Mock(return_value='https://www.canfar.phys.uvic.ca/vospace')
client.conn = Mock()
client.conn.session.post = Mock(return_value=resp)
client.get_transfer_error = Mock()
client.protocol = 'https'
data = str(node)
property_url = 'https://www.canfar.phys.uvic.ca/vospace/nodeprops'
result = client.update(node, False)
self.assertEqual(result, 0)
client.conn.session.post.assert_called_with('https://www.canfar.phys.uvic.ca/vospace',
data=data, allow_redirects=False)
call1 = call(property_url, allow_redirects=False, data=data,
headers={'Content-type': 'text/xml'})
call2 = call('https://www.canfar.phys.uvic.ca/vospace/phase', allow_redirects=False, data="PHASE=RUN",
headers={'Content-type': "text/text"})
calls = [call1, call2]
client.conn = Mock()
client.conn.session.post = Mock(return_value=resp)
with patch('vos.vos.EndPoints.properties', property_url):
result = client.update(node, True)
self.assertEqual(result, 0)
client.conn.session.post.assert_has_calls(calls)
def test_copy(self, computed_md5_mock):
# the md5sum of the file being copied
md5sum = 'd41d8cd98f00b204e9800998ecf84eee'
# patch the compute_md5 function in vos to return the above value
computed_md5_mock.return_value = md5sum
#mock the props of the corresponding node
props = MagicMock()
props.get.return_value = md5sum
#add props to the mocked node
node = MagicMock(spec=Node)
node.props = props
# mock one by one the chain of connection.session.response.headers
conn = MagicMock(spec=Connection)
session = MagicMock()
response = MagicMock()
headers = MagicMock()
headers.get.return_value = md5sum
response.headers = headers
session.get.return_value = response
conn.session = session
test_client = Client()
# use the mocked connection instead of the real one
test_client.conn = conn
get_node_url_mock = Mock(
return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
test_client.get_node_url = get_node_url_mock
#patch Client.get_node to return our mocked node
get_node_mock = Mock(return_value=node)
test_client.get_node = get_node_mock
# time to test...
vospaceLocation = 'vos://test/foo'
osLocation = '/tmp/foo'
# copy from vospace
test_client.copy(vospaceLocation, osLocation)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None,
view='data')
computed_md5_mock.assert_called_once_with(osLocation)
get_node_mock.assert_called_once_with(vospaceLocation)
# copy to vospace
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
test_client.copy(osLocation, vospaceLocation)
get_node_url_mock.assert_called_once_with(vospaceLocation, 'PUT')
computed_md5_mock.assert_called_once_with(osLocation)
# error tests - md5sum mismatch
computed_md5_mock.return_value = '000bad000'
with self.assertRaises(OSError):
test_client.copy(vospaceLocation, osLocation)
with self.assertRaises(OSError):
test_client.copy(osLocation, vospaceLocation)
def test_transfer_error(self, mock_vofile):
vofile = MagicMock()
mock_vofile.return_value = vofile
vospace_url = 'https://somevospace.server/vospace'
session = Mock()
session.get.side_effect = [Mock(content='COMPLETED')]
conn = Mock(spec=Connection)
conn.session = session
test_client = Client()
# use the mocked connection instead of the real one
test_client.conn = conn
# job successfully completed
vofile.read.side_effect = [b'QUEUED', b'COMPLETED']
self.assertFalse(test_client.get_transfer_error(
vospace_url + '/results/transferDetails', 'vos://vospace'))
session.get.assert_called_once_with(vospace_url + '/phase',
allow_redirects=False)
# job suspended
session.reset_mock()
session.get.side_effect = [Mock(content=b'COMPLETED')]
vofile.read.side_effect = [b'QUEUED', b'SUSPENDED']
with self.assertRaises(OSError):
test_client.get_transfer_error(
vospace_url + '/results/transferDetails', 'vos://vospace')
# check arguments for session.get calls
self.assertEquals(
[call(vospace_url + '/phase', allow_redirects=False)],
session.get.call_args_list)
# job encountered an internal error
session.reset_mock()
vofile.read.side_effect = Mock(side_effect=[b'QUEUED', b'ERROR'])
session.get.side_effect = [Mock(content=b'COMPLETED'),
Mock(text='InternalFault')]
with self.assertRaises(OSError):
test_client.get_transfer_error(
vospace_url + '/results/transferDetails', 'vos://vospace')
self.assertEquals([call(vospace_url + '/phase', allow_redirects=False),
call(vospace_url + '/error')],
session.get.call_args_list)
# job encountered an unsupported link error
session.reset_mock()
link_file = 'testlink.fits'
vofile.read.side_effect = Mock(side_effect=[b'QUEUED', b'ERROR'])
session.get.side_effect = [Mock(content=b'COMPLETED'),
Mock(
text="Unsupported link target: " +
link_file)]
self.assertEquals(link_file, test_client.get_transfer_error(
vospace_url + '/results/transferDetails', 'vos://vospace'))
self.assertEquals([call(vospace_url + '/phase', allow_redirects=False),
call(vospace_url + '/error')],
session.get.call_args_list)
def test_transfer_error(self, mock_vofile):
vofile = MagicMock()
mock_vofile.return_value = vofile
vospace_url = 'https://somevospace.server/vospace'
session = Mock()
session.get.side_effect = [Mock(content='COMPLETED')]
conn = Mock(spec=Connection)
conn.session = session
test_client = Client()
# use the mocked connection instead of the real one
test_client.conn = conn
# job successfully completed
vofile.read.side_effect = [b'QUEUED', b'COMPLETED']
self.assertFalse(test_client.get_transfer_error(
vospace_url +'/results/transferDetails', 'vos://vospace'))
session.get.assert_called_once_with(vospace_url + '/phase', allow_redirects=False)
# job suspended
session.reset_mock()
session.get.side_effect = [Mock(content=b'COMPLETED')]
vofile.read.side_effect = [b'QUEUED', b'SUSPENDED']
with self.assertRaises(OSError):
test_client.get_transfer_error(
vospace_url +'/results/transferDetails', 'vos://vospace')
#check arguments for session.get calls
self.assertEquals([call(vospace_url + '/phase', allow_redirects=False)],
session.get.call_args_list )
# job encountered an internal error
session.reset_mock()
vofile.read.side_effect = Mock(side_effect=[b'QUEUED', b'ERROR'])
session.get.side_effect = [Mock(content=b'COMPLETED'), Mock(text='InternalFault')]
with self.assertRaises(OSError):
test_client.get_transfer_error(
vospace_url +'/results/transferDetails', 'vos://vospace')
self.assertEquals([call(vospace_url + '/phase', allow_redirects=False),
call(vospace_url + '/error')],
session.get.call_args_list )
# job encountered an unsupported link error
session.reset_mock()
link_file = 'testlink.fits'
vofile.read.side_effect = Mock(side_effect=[b'QUEUED', b'ERROR'])
session.get.side_effect = [Mock(content=b'COMPLETED'),
Mock(text="Unsupported link target: " + link_file)]
self.assertEquals(link_file, test_client.get_transfer_error(
vospace_url +'/results/transferDetails', 'vos://vospace'))
self.assertEquals([call(vospace_url + '/phase', allow_redirects=False),
call(vospace_url + '/error')],
session.get.call_args_list )
def off_quota(self):
"""
Test that a 413 raised by the server gets a reasonable error to the user.
@return:
"""
with patch('vos.vos.VOFile') as mockVOFile:
mockVOFile.open = Mock()
mockVOFile.read = Mock()
mockVOFile.write = Mock()
client = Client()
client.conn = Mock()
client.transfer(uri='vos:test', direction="pushToVoSpace")
def off_quota(self):
"""
Test that a 413 raised by the server gets a reasonable error to the
user.
@return:
"""
with patch('vos.vos.VOFile') as mockVOFile:
mockVOFile.open = Mock()
mockVOFile.read = Mock()
mockVOFile.write = Mock()
client = Client()
client.conn = Mock()
client.transfer(uri='vos:test', direction="pushToVoSpace")
def test_add_props(self):
old_node = Node(ElementTree.fromstring(NODE_XML))
new_node = Node(ElementTree.fromstring(NODE_XML))
new_node.props['quota'] = '1000'
new_node.create = Mock(return_value=new_node.node)
data = str(new_node)
headers = {'size': str(len(data))}
client = Client()
client.get_node = Mock(return_value=old_node)
client.get_node_url = Mock(return_value='http://foo.com/bar')
client.conn = Mock()
with patch('vos.Client', client) as mock:
mock.add_props(new_node)
mock.conn.session.post.assert_called_with('http://foo.com/bar',
headers=headers, data=data)
def test_update(self):
node = Node(ElementTree.fromstring(NODE_XML))
resp = Mock()
resp.headers.get = Mock(
return_value="https://www.canfar.phys.uvic.ca/vospace")
conn = Mock(spec=vos.Connection)
conn.session.post = Mock(return_value=resp)
client = Client(conn=conn)
client.get_node_url = Mock(
return_value='https://www.canfar.phys.uvic.ca/vospace')
client.get_transfer_error = Mock()
client.protocol = 'https'
data = str(node)
property_url = 'https://www.canfar.phys.uvic.ca/vospace/nodeprops'
endpoints_mock = Mock()
endpoints_mock.properties = property_url
client.get_endpoints = Mock(return_value=endpoints_mock)
result = client.update(node, False)
self.assertEqual(result, 0)
client.conn.session.post.assert_called_with(
'https://www.canfar.phys.uvic.ca/vospace',
data=data,
allow_redirects=False)
call1 = call(property_url,
allow_redirects=False,
data=data,
headers={'Content-type': 'text/xml'})
call2 = call(
'https://www.canfar.phys.uvic.ca/vospace/phase',
allow_redirects=False,
data="PHASE=RUN",
headers={'Content-type': 'application/x-www-form-urlencoded'})
calls = [call1, call2]
client.conn = Mock(spec=vos.Connection)
client.conn.session.post = Mock(return_value=resp)
result = client.update(node, True)
self.assertEqual(result, 0)
client.conn.session.post.assert_has_calls(calls)
def test_add_props(self):
old_node = Node(ElementTree.fromstring(NODE_XML))
new_node = Node(ElementTree.fromstring(NODE_XML))
new_node.props['quota'] = '1000'
new_node.create = Mock(return_value=new_node.node)
data = str(new_node)
headers = {'size': str(len(data))}
client = Client()
client.get_node = Mock(return_value=old_node)
client.get_node_url = Mock(return_value='http://foo.com/bar')
client.conn = Mock()
with patch('vos.Client', client) as mock:
mock.add_props(new_node)
mock.conn.session.post.assert_called_with('http://foo.com/bar',
headers=headers,
data=data)
def test_create(self):
uri = 'vos://create.vospace.auth!vospace/bar'
client = Client()
node = Node(client.fix_uri(uri))
node2 = Node(str(node))
self.assertEquals(node, node2)
data = str(node)
headers = {'size': str(len(data))}
client = Client()
#client.get_node_url = Mock(return_value='http://foo.com/bar')
session_mock = MagicMock()
client.conn = Mock()
client.conn.session = session_mock
session_mock.put.return_value = Mock(content=str(node))
result = client.create(uri)
self.assertEquals(node, result)
session_mock.put.assert_called_with('http://www.canfar.phys.uvic.ca/vospace/nodes/bar',
headers=headers, data=data)
def test_create(self):
uri = 'vos://create.vospace.auth!vospace/bar'
client = Client()
node = Node(client.fix_uri(uri))
node2 = Node(str(node))
self.assertEquals(node, node2)
data = str(node)
headers = {'size': str(len(data))}
client = Client()
# client.get_node_url = Mock(return_value='http://foo.com/bar')
session_mock = MagicMock()
client.conn = Mock()
client.conn.session = session_mock
session_mock.put.return_value = Mock(content=str(node))
result = client.create(uri)
self.assertEquals(node, result)
session_mock.put.assert_called_with(
'http://www.canfar.phys.uvic.ca/vospace/nodes/bar',
headers=headers, data=data)
def test_update(self):
node = Node(ElementTree.fromstring(NODE_XML))
resp = Mock()
resp.headers.get = Mock(
return_value="https://www.canfar.phys.uvic.ca/vospace")
conn = Mock(spec=vos.Connection)
conn.session.post = Mock(return_value=resp)
client = Client(conn=conn)
client.get_node_url = Mock(
return_value='https://www.canfar.phys.uvic.ca/vospace')
client.get_transfer_error = Mock()
client.protocol = 'https'
data = str(node)
property_url = 'https://www.canfar.phys.uvic.ca/vospace/nodeprops'
endpoints_mock = Mock()
endpoints_mock.properties = property_url
client.get_endpoints = Mock(return_value=endpoints_mock)
result = client.update(node, False)
self.assertEqual(result, 0)
client.conn.session.post.assert_called_with(
'https://www.canfar.phys.uvic.ca/vospace',
data=data, allow_redirects=False)
call1 = call(property_url, allow_redirects=False, data=data,
headers={'Content-type': 'text/xml'})
call2 = call('https://www.canfar.phys.uvic.ca/vospace/phase',
allow_redirects=False, data="PHASE=RUN",
headers={'Content-type': "text/text"})
calls = [call1, call2]
client.conn = Mock(spec=vos.Connection)
client.conn.session.post = Mock(return_value=resp)
result = client.update(node, True)
self.assertEqual(result, 0)
client.conn.session.post.assert_has_calls(calls)
def test_copy(self, computed_md5_mock):
# the md5sum of the file being copied
md5sum = 'd41d8cd98f00b204e9800998ecf84eee'
# patch the compute_md5 function in vos to return the above value
computed_md5_mock.return_value = md5sum
# mock the props of the corresponding node
props = MagicMock()
props.get.return_value = md5sum
# add props to the mocked node
node = MagicMock(spec=Node)
node.props = props
# mock one by one the chain of connection.session.response.headers
conn = MagicMock(spec=Connection)
session = MagicMock()
response = MagicMock()
headers = MagicMock()
headers.get.return_value = md5sum
response.headers = headers
session.get.return_value = response
conn.session = session
test_client = Client()
# use the mocked connection instead of the real one
test_client.conn = conn
get_node_url_mock = Mock(
return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
test_client.get_node_url = get_node_url_mock
mock_update = Mock()
test_client.update = mock_update
# patch Client.get_node to return our mocked node
get_node_mock = Mock(return_value=node)
test_client.get_node = get_node_mock
# time to test...
vospaceLocation = 'vos://test/foo'
osLocation = '/tmp/foo'
if os.path.isfile(osLocation):
os.remove(osLocation)
# copy from vospace
test_client.copy(vospaceLocation, osLocation)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None,
view='data')
computed_md5_mock.assert_called_once_with(osLocation)
assert get_node_mock.called
# repeat - local file and vospace file are now the same -> only
# get_node is called to get the md5 of remote file
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
get_node_mock.reset_mock()
test_client.copy(vospaceLocation, osLocation)
assert not get_node_url_mock.called
computed_md5_mock.assert_called_once_with(osLocation)
get_node_mock.assert_called_once_with(vospaceLocation)
# change the content of local files to trigger a new copy
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
computed_md5_mock.side_effect = ['d002233', md5sum]
get_node_mock.reset_mock()
test_client.copy(vospaceLocation, osLocation)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None,
view='data')
computed_md5_mock.assert_called_with(osLocation)
get_node_mock.assert_called_once_with(vospaceLocation)
# copy to vospace when md5 sums are the same -> only update occurs
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
computed_md5_mock.side_effect = None
computed_md5_mock.return_value = md5sum
test_client.copy(osLocation, vospaceLocation)
mock_update.assert_called_once()
assert not get_node_url_mock.called
# make md5 different
get_node_url_mock.reset_mock()
get_node_url_mock.return_value =\
['http://cadc.ca/test', 'http://cadc.ca/test']
computed_md5_mock.reset_mock()
mock_update.reset_mock()
props.get.side_effect = ['d00223344', md5sum]
test_client.copy(osLocation, vospaceLocation)
assert not mock_update.called
get_node_url_mock.assert_called_once_with(vospaceLocation, 'PUT')
computed_md5_mock.assert_called_once_with(osLocation)
# copy 0 size file -> delete and create on client but no bytes
# transferred
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
computed_md5_mock.return_value = vos.ZERO_MD5
props.get.side_effect = [md5sum]
mock_delete = Mock()
mock_create = Mock()
test_client.delete = mock_delete
test_client.create = mock_create
test_client.copy(osLocation, vospaceLocation)
mock_create.assert_called_once_with(vospaceLocation)
mock_delete.assert_called_once_with(vospaceLocation)
assert not get_node_url_mock.called
# copy new 0 size file -> reate on client but no bytes transferred
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
mock_delete.reset_mock()
mock_create.reset_mock()
computed_md5_mock.return_value = vos.ZERO_MD5
props.get.side_effect = [None]
mock_delete = Mock()
mock_create = Mock()
test_client.delete = mock_delete
test_client.create = mock_create
test_client.copy(osLocation, vospaceLocation)
mock_create.assert_called_once_with(vospaceLocation)
assert not mock_delete.called
assert not get_node_url_mock.called
# error tests - md5sum mismatch
props.get.side_effect = [md5sum]
computed_md5_mock.return_value = '000bad000'
with self.assertRaises(OSError):
test_client.copy(vospaceLocation, osLocation)
with self.assertRaises(OSError):
test_client.copy(osLocation, vospaceLocation)
# requests just the headers
props.get.side_effect = [None]
get_node_url_mock = Mock(
return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
test_client.get_node_url = get_node_url_mock
computed_md5_mock.reset_mock()
computed_md5_mock.side_effect = ['d002233', md5sum]
get_node_mock.reset_mock()
test_client.copy(vospaceLocation, osLocation, head=True)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None,
view='header')
def test_copy(self, computed_md5_mock):
# the md5sum of the file being copied
md5sum = 'd41d8cd98f00b204e9800998ecf84eee'
# patch the compute_md5 function in vos to return the above value
computed_md5_mock.return_value = md5sum
# mock the props of the corresponding node
props = MagicMock()
props.get.return_value = md5sum
# add props to the mocked node
node = MagicMock(spec=Node)
node.props = props
# mock one by one the chain of connection.session.response.headers
conn = MagicMock(spec=Connection)
session = MagicMock()
response = MagicMock()
headers = MagicMock()
headers.get.return_value = md5sum
response.headers = headers
session.get.return_value = response
conn.session = session
test_client = Client()
# use the mocked connection instead of the real one
test_client.conn = conn
get_node_url_mock = Mock(
return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
test_client.get_node_url = get_node_url_mock
mock_update = Mock()
test_client.update = mock_update
# patch Client.get_node to return our mocked node
get_node_mock = Mock(return_value=node)
test_client.get_node = get_node_mock
# time to test...
vospaceLocation = 'vos://test/foo'
osLocation = '/tmp/foo'
if os.path.isfile(osLocation):
os.remove(osLocation)
# copy from vospace
test_client.copy(vospaceLocation, osLocation)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None, view='data')
computed_md5_mock.assert_called_once_with(osLocation)
assert get_node_mock.called
# repeat - local file and vospace file are now the same -> only
# get_node is called to get the md5 of remote file
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
get_node_mock.reset_mock()
test_client.copy(vospaceLocation, osLocation)
assert not get_node_url_mock.called
computed_md5_mock.assert_called_once_with(osLocation)
get_node_mock.assert_called_once_with(vospaceLocation)
# change the content of local files to trigger a new copy
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
computed_md5_mock.side_effect = ['d002233', md5sum]
get_node_mock.reset_mock()
test_client.copy(vospaceLocation, osLocation)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None, view='data')
computed_md5_mock.assert_called_with(osLocation)
get_node_mock.assert_called_once_with(vospaceLocation)
# copy to vospace when md5 sums are the same -> only update occurs
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
computed_md5_mock.side_effect = None
computed_md5_mock.return_value = md5sum
test_client.copy(osLocation, vospaceLocation)
mock_update.assert_called_once()
assert not get_node_url_mock.called
# make md5 different
get_node_url_mock.reset_mock()
get_node_url_mock.return_value =\
['http://cadc.ca/test', 'http://cadc.ca/test']
computed_md5_mock.reset_mock()
mock_update.reset_mock()
props.get.side_effect = ['d00223344', md5sum]
test_client.copy(osLocation, vospaceLocation)
assert not mock_update.called
get_node_url_mock.assert_called_once_with(vospaceLocation, 'PUT')
computed_md5_mock.assert_called_once_with(osLocation)
# copy 0 size file -> delete and create on client but no bytes
# transferred
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
computed_md5_mock.return_value = vos.ZERO_MD5
props.get.side_effect = [md5sum]
mock_delete = Mock()
mock_create = Mock()
test_client.delete = mock_delete
test_client.create = mock_create
test_client.copy(osLocation, vospaceLocation)
mock_create.assert_called_once_with(vospaceLocation)
mock_delete.assert_called_once_with(vospaceLocation)
assert not get_node_url_mock.called
# copy new 0 size file -> reate on client but no bytes transferred
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
mock_delete.reset_mock()
mock_create.reset_mock()
computed_md5_mock.return_value = vos.ZERO_MD5
props.get.side_effect = [None]
mock_delete = Mock()
mock_create = Mock()
test_client.delete = mock_delete
test_client.create = mock_create
test_client.copy(osLocation, vospaceLocation)
mock_create.assert_called_once_with(vospaceLocation)
assert not mock_delete.called
assert not get_node_url_mock.called
# error tests - md5sum mismatch
props.get.side_effect = [md5sum]
computed_md5_mock.return_value = '000bad000'
with self.assertRaises(OSError):
test_client.copy(vospaceLocation, osLocation)
with self.assertRaises(OSError):
test_client.copy(osLocation, vospaceLocation)
# requests just the headers
props.get.side_effect = [None]
get_node_url_mock = Mock(
return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
test_client.get_node_url = get_node_url_mock
computed_md5_mock.reset_mock()
computed_md5_mock.side_effect = ['d002233', md5sum]
get_node_mock.reset_mock()
test_client.copy(vospaceLocation, osLocation, head=True)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None, view='header')