def test_init_client(self): # No parameters uses cert in ~/.ssl giving authenticated / https # create a fake pem file certfile = '/tmp/some-cert-file.pem' open(certfile, 'w+') Client.VOSPACE_CERTFILE = "some-cert-file.pem" with patch('os.access'): client = Client(vospace_certfile=certfile) self.assertTrue(client.conn.subject.certificate) self.assertFalse(client.conn.vo_token) # Supplying an empty string for certfile implies anonymous / http client = Client(vospace_certfile='') self.assertTrue(client.conn.subject.anon) self.assertFalse(client.conn.vo_token) # Specifying a token implies authenticated / http client = Client(vospace_token='a_token_string') self.assertTrue(client.conn.subject.anon) self.assertTrue(client.conn.vo_token) # Specifying both a certfile and token implies token (auth) / http with patch('os.access'): client = Client(vospace_certfile=certfile, vospace_token='a_token_string') self.assertTrue(client.conn.subject.anon) self.assertTrue(client.conn.vo_token)
def test_delete(self): certfile = '/tmp/SomeCert.pem' open(certfile, 'w+') with patch('os.access'): client = Client(vospace_certfile=certfile) uri1 = 'vos://cadc.nrc.ca!vospace/nosuchfile1' url = 'https://www.canfar.phys.uvic.ca/vospace/nodes/nosuchfile1' client.conn.session.delete = Mock() client.delete(uri1) client.conn.session.delete.assert_called_once_with(url)
def test_update(self): node = Node(ElementTree.fromstring(NODE_XML)) resp = Mock() resp.headers.get = Mock(return_value="https://www.canfar.phys.uvic.ca/vospace") client = Client() client.get_node_url = Mock(return_value='https://www.canfar.phys.uvic.ca/vospace') client.conn = Mock() client.conn.session.post = Mock(return_value=resp) client.get_transfer_error = Mock() client.protocol = 'https' data = str(node) property_url = 'https://www.canfar.phys.uvic.ca/vospace/nodeprops' result = client.update(node, False) self.assertEqual(result, 0) client.conn.session.post.assert_called_with('https://www.canfar.phys.uvic.ca/vospace', data=data, allow_redirects=False) call1 = call(property_url, allow_redirects=False, data=data, headers={'Content-type': 'text/xml'}) call2 = call('https://www.canfar.phys.uvic.ca/vospace/phase', allow_redirects=False, data="PHASE=RUN", headers={'Content-type': "text/text"}) calls = [call1, call2] client.conn = Mock() client.conn.session.post = Mock(return_value=resp) with patch('vos.vos.EndPoints.properties', property_url): result = client.update(node, True) self.assertEqual(result, 0) client.conn.session.post.assert_has_calls(calls)
def off_quota(self): """ Test that a 413 raised by the server gets a reasonable error to the user. @return: """ with patch('vos.vos.VOFile') as mockVOFile: mockVOFile.open = Mock() mockVOFile.read = Mock() mockVOFile.write = Mock() client = Client() client.conn = Mock() client.transfer(uri='vos:test', direction="pushToVoSpace")
def test_get_info_list(self): # list tuples of a LinkNode mock_node = MagicMock(type='vos:DataNode') mock_node.return_value = mock_node mock_node.name = 'testnode' mock_node.get_info.return_value = {'name': 'aa'} mock_link_node = Mock(type='vos:LinkNode') mock_link_node.target = 'vos:/somefile' client = Client() client.get_node = MagicMock(side_effect=[mock_link_node, mock_node]) self.assertEquals([mock_node], client.get_children_info('vos:/somenode'))
def test_move(self): mock_resp_403 = Mock(name="mock_resp_303") mock_resp_403.status_code = 403 conn = Connection() conn.session.post = Mock(return_value=mock_resp_403) client = Client(conn=conn) uri1 = 'notvos://cadc.nrc.ca!vospace/nosuchfile1' uri2 = 'notvos://cadc.nrc.ca!vospace/nosuchfile2' with self.assertRaises(OSError): client.move(uri1, uri2)
def test_transfer_error(self, mock_vofile): vofile = MagicMock() mock_vofile.return_value = vofile vospace_url = 'https://somevospace.server/vospace' session = Mock() session.get.side_effect = [Mock(content='COMPLETED')] conn = Mock(spec=Connection) conn.session = session test_client = Client() # use the mocked connection instead of the real one test_client.conn = conn # job successfully completed vofile.read.side_effect = [b'QUEUED', b'COMPLETED'] self.assertFalse(test_client.get_transfer_error( vospace_url + '/results/transferDetails', 'vos://vospace')) session.get.assert_called_once_with(vospace_url + '/phase', allow_redirects=False) # job suspended session.reset_mock() session.get.side_effect = [Mock(content=b'COMPLETED')] vofile.read.side_effect = [b'QUEUED', b'SUSPENDED'] with self.assertRaises(OSError): test_client.get_transfer_error( vospace_url + '/results/transferDetails', 'vos://vospace') # check arguments for session.get calls self.assertEquals( [call(vospace_url + '/phase', allow_redirects=False)], session.get.call_args_list) # job encountered an internal error session.reset_mock() vofile.read.side_effect = Mock(side_effect=[b'QUEUED', b'ERROR']) session.get.side_effect = [Mock(content=b'COMPLETED'), Mock(text='InternalFault')] with self.assertRaises(OSError): test_client.get_transfer_error( vospace_url + '/results/transferDetails', 'vos://vospace') self.assertEquals([call(vospace_url + '/phase', allow_redirects=False), call(vospace_url + '/error')], session.get.call_args_list) # job encountered an unsupported link error session.reset_mock() link_file = 'testlink.fits' vofile.read.side_effect = Mock(side_effect=[b'QUEUED', b'ERROR']) session.get.side_effect = [Mock(content=b'COMPLETED'), Mock( text="Unsupported link target: " + link_file)] self.assertEquals(link_file, test_client.get_transfer_error( vospace_url + '/results/transferDetails', 'vos://vospace')) self.assertEquals([call(vospace_url + '/phase', allow_redirects=False), call(vospace_url + '/error')], session.get.call_args_list)
def test_transfer_error(self, mock_vofile): vofile = MagicMock() mock_vofile.return_value = vofile vospace_url = 'https://somevospace.server/vospace' session = Mock() session.get.side_effect = [Mock(content='COMPLETED')] conn = Mock(spec=Connection) conn.session = session test_client = Client() # use the mocked connection instead of the real one test_client.conn = conn # job successfully completed vofile.read.side_effect = [b'QUEUED', b'COMPLETED'] self.assertFalse(test_client.get_transfer_error( vospace_url +'/results/transferDetails', 'vos://vospace')) session.get.assert_called_once_with(vospace_url + '/phase', allow_redirects=False) # job suspended session.reset_mock() session.get.side_effect = [Mock(content=b'COMPLETED')] vofile.read.side_effect = [b'QUEUED', b'SUSPENDED'] with self.assertRaises(OSError): test_client.get_transfer_error( vospace_url +'/results/transferDetails', 'vos://vospace') #check arguments for session.get calls self.assertEquals([call(vospace_url + '/phase', allow_redirects=False)], session.get.call_args_list ) # job encountered an internal error session.reset_mock() vofile.read.side_effect = Mock(side_effect=[b'QUEUED', b'ERROR']) session.get.side_effect = [Mock(content=b'COMPLETED'), Mock(text='InternalFault')] with self.assertRaises(OSError): test_client.get_transfer_error( vospace_url +'/results/transferDetails', 'vos://vospace') self.assertEquals([call(vospace_url + '/phase', allow_redirects=False), call(vospace_url + '/error')], session.get.call_args_list ) # job encountered an unsupported link error session.reset_mock() link_file = 'testlink.fits' vofile.read.side_effect = Mock(side_effect=[b'QUEUED', b'ERROR']) session.get.side_effect = [Mock(content=b'COMPLETED'), Mock(text="Unsupported link target: " + link_file)] self.assertEquals(link_file, test_client.get_transfer_error( vospace_url +'/results/transferDetails', 'vos://vospace')) self.assertEquals([call(vospace_url + '/phase', allow_redirects=False), call(vospace_url + '/error')], session.get.call_args_list )
def test_open(self): # Invalid mode raises OSError with self.assertRaises(OSError): client = Client() client.open('vos://foo/bar', mode=-10000) with self.assertRaises(OSError): client = Client() client.get_node_url = Mock(return_value=None) client.open(None, url=None) conn = Connection() mock_vofile = VOFile(['http://foo.com/bar'], conn, 'GET') client = Client() client.get_node_url = Mock(return_value=mock_vofile) vofile = client.open(None, url=None) self.assertEquals(vofile.url.URLs[0], 'http://foo.com/bar')
def test_getNode(self): """ @return: """ uri = "vos://foo.com!vospace/bar" nodes = (' <vos:nodes>\n' '<vos:node uri="vos://cadc.nrc.ca!vospace/mydir/file123" ' 'xs:type="vos:DataNode">\n' ' <vos:properties>\n' ' <vos:property ' 'uri="ivo://ivoa.net/vospace/core#date">2016-05-10T09:52:13' '</vos:property>\n' ' </vos:properties>\n' '</vos:node>\n' '<vos:node uri="vos://cadc.nrc.ca!vospace/mydir/file456" ' 'xs:type="vos:DataNode">\n' ' <vos:properties>\n' ' <vos:property uri="ivo://ivoa.net/vospace/core#date">' '2016-05-19T09:52:14</vos:property>\n' ' </vos:properties>\n' '</vos:node>\n' '</vos:nodes>\n') mock_vofile = Mock() client = Client() client.open = Mock(return_value=mock_vofile) mock_vofile.read = Mock( return_value=NODE_XML.format(uri, '').encode('UTF-8')) my_node = client.get_node(uri, limit=0, force=False) self.assertEqual(uri, my_node.uri) self.assertEqual(len(my_node.node_list), 0) mock_vofile.read = Mock( return_value=NODE_XML.format(uri, nodes).encode('UTF-8')) my_node = client.get_node(uri, limit=2, force=True) self.assertEqual(uri, my_node.uri) self.assertEqual(len(my_node.node_list), 2) my_node = client.get_node(uri, limit=2, force=False) self.assertEqual(uri, my_node.uri) self.assertEqual(len(my_node.node_list), 2)
def test_add_props(self): old_node = Node(ElementTree.fromstring(NODE_XML)) new_node = Node(ElementTree.fromstring(NODE_XML)) new_node.props['quota'] = '1000' new_node.create = Mock(return_value=new_node.node) data = str(new_node) headers = {'size': str(len(data))} client = Client() client.get_node = Mock(return_value=old_node) client.get_node_url = Mock(return_value='http://foo.com/bar') client.conn = Mock() with patch('vos.Client', client) as mock: mock.add_props(new_node) mock.conn.session.post.assert_called_with('http://foo.com/bar', headers=headers, data=data)
def test_get_node_url(): client = Client() with pytest.raises(TypeError): client.get_node_url('vos://cadc.nrc.ca!vospace/auser', sort='Blah') with pytest.raises(ValueError): client.get_node_url('vos://cadc.nrc.ca!vospace/auser', order='Blah') response = Mock(spec=requests.Response) response.status_code = 303 client.conn.session.get = Mock(return_value=response) equery = urlparse( client.get_node_url('vos://cadc.nrc.ca!vospace/auser', sort=vos.SortNodeProperty.DATE)).query assert (urllib.parse.unquote(equery) == 'sort={}'.format( vos.SortNodeProperty.DATE.value)) equery = urlparse( client.get_node_url('vos://cadc.nrc.ca!vospace/auser', sort=vos.SortNodeProperty.LENGTH, order='asc')).query args = urllib.parse.unquote(equery).split('&') assert (2 == len(args)) assert ('order=asc' in args) assert ('sort={}'.format(vos.SortNodeProperty.LENGTH.value) in args) equery = urlparse( client.get_node_url('vos://cadc.nrc.ca!vospace/auser', order='desc')).query assert ('order=desc' == urllib.parse.unquote(equery)) # test header view transfer_url = 'https://some.location/some/headers' client.conn.session.get = Mock(return_value=response) response.headers = {'Location': transfer_url} assert transfer_url == \ client.get_node_url('vos://cadc.nrc.ca!vospace/auser', view='header')[0] # get the argument lists for client.conn.session.get call = client.conn.session.get.call_args_list[0] args, kwargs = call # check head is amongst the other parameters assert kwargs['params']['view'] == 'header'
def test_create(self): uri = 'vos://create.vospace.auth!vospace/bar' client = Client() node = Node(client.fix_uri(uri)) node2 = Node(str(node)) self.assertEquals(node, node2) data = str(node) headers = {'size': str(len(data))} client = Client() # client.get_node_url = Mock(return_value='http://foo.com/bar') session_mock = MagicMock() client.conn = Mock() client.conn.session = session_mock session_mock.put.return_value = Mock(content=str(node)) result = client.create(uri) self.assertEquals(node, result) session_mock.put.assert_called_with( 'http://www.canfar.phys.uvic.ca/vospace/nodes/bar', headers=headers, data=data)
def test_update(self): node = Node(ElementTree.fromstring(NODE_XML)) resp = Mock() resp.headers.get = Mock( return_value="https://www.canfar.phys.uvic.ca/vospace") conn = Mock(spec=vos.Connection) conn.session.post = Mock(return_value=resp) client = Client(conn=conn) client.get_node_url = Mock( return_value='https://www.canfar.phys.uvic.ca/vospace') client.get_transfer_error = Mock() client.protocol = 'https' data = str(node) property_url = 'https://www.canfar.phys.uvic.ca/vospace/nodeprops' endpoints_mock = Mock() endpoints_mock.properties = property_url client.get_endpoints = Mock(return_value=endpoints_mock) result = client.update(node, False) self.assertEqual(result, 0) client.conn.session.post.assert_called_with( 'https://www.canfar.phys.uvic.ca/vospace', data=data, allow_redirects=False) call1 = call(property_url, allow_redirects=False, data=data, headers={'Content-type': 'text/xml'}) call2 = call( 'https://www.canfar.phys.uvic.ca/vospace/phase', allow_redirects=False, data="PHASE=RUN", headers={'Content-type': 'application/x-www-form-urlencoded'}) calls = [call1, call2] client.conn = Mock(spec=vos.Connection) client.conn.session.post = Mock(return_value=resp) result = client.update(node, True) self.assertEqual(result, 0) client.conn.session.post.assert_has_calls(calls)
def rs(): config = mc.Config() config.get_executors() vos_client = Client(vospace_certfile=config.proxy_fqn) builder = GemProcBuilder(config) storage_name = builder.build(sys.argv[1]) store_transfer = tc.VoFitsTransfer(vos_client) rc.run_single( config=config, storage_name=storage_name, command_name='gemProc2caom2', meta_visitors=[], data_visitors=[provenance_augmentation, preview_augmentation], store_transfer=store_transfer, )
def test_get_node_url(): client = Client() with pytest.raises(TypeError): client.get_node_url('vos://cadc.nrc.ca!vospace/auser', sort='Blah') with pytest.raises(ValueError): client.get_node_url('vos://cadc.nrc.ca!vospace/auser', order='Blah') response = Mock(spec=requests.Response) response.status_code = 303 client.conn.session.get = Mock(return_value=response) equery = urlparse(client.get_node_url('vos://cadc.nrc.ca!vospace/auser', sort=vos.SortNodeProperty.DATE)).query assert(urllib.parse.unquote(equery) == 'sort={}'.format(vos.SortNodeProperty.DATE.value)) equery = urlparse(client.get_node_url('vos://cadc.nrc.ca!vospace/auser', sort=vos.SortNodeProperty.LENGTH, order='asc')).query args = urllib.parse.unquote(equery).split('&') assert(2 == len(args)) assert('order=asc' in args) assert('sort={}'.format(vos.SortNodeProperty.LENGTH.value) in args) equery = urlparse(client.get_node_url('vos://cadc.nrc.ca!vospace/auser', order='desc')).query assert('order=desc' == urllib.parse.unquote(equery)) # test header view transfer_url = 'https://some.location/some/headers' client.conn.session.get = Mock(return_value=response) response.headers = {'Location': transfer_url} assert transfer_url == \ client.get_node_url('vos://cadc.nrc.ca!vospace/auser', view='header')[0] # get the argument lists for client.conn.session.get call = client.conn.session.get.call_args_list[0] args, kwargs = call # check head is amongst the other parameters assert kwargs['params']['view'] == 'header'
def bad_exposure_list(): """ Retreieve the list of bad exposrures from S. Gwyn's VOSpace listing. :return: list of bad exposures :rtype list """ Client().copy(u'vos:sgwyn/tkBAD', u'tkBAD') bad_expnum_list = [] for lines in open('tkBAD').readlines(): # noinspection PyBroadException try: bad_expnum_list.append(lines.split()[0]) except: sys.stderr.write(lines) return bad_expnum_list
def _run_remote(): """ Uses a todo file to identify the work to be done. :return 0 if successful, -1 if there's any sort of failure. Return status is used by airflow for task instance management and reporting. """ config = mc.Config() config.get_executors() name_builder = nbc.FileNameBuilder(GemProcName) vos_client = Client(vospace_certfile=config.proxy_fqn) store_transfer = tc.VoFitsTransfer(vos_client) data_source = dsc.VaultListDirDataSource(vos_client, config) return rc.run_by_todo(config=config, name_builder=name_builder, command_name=APPLICATION, source=data_source, meta_visitors=META_VISITORS, data_visitors=DATA_VISITORS, store_transfer=store_transfer)
def test_update(self): node = Node(ElementTree.fromstring(NODE_XML)) resp = Mock() resp.headers.get = Mock( return_value="https://www.canfar.phys.uvic.ca/vospace") conn = Mock(spec=vos.Connection) conn.session.post = Mock(return_value=resp) client = Client(conn=conn) client.get_node_url = Mock( return_value='https://www.canfar.phys.uvic.ca/vospace') client.get_transfer_error = Mock() client.protocol = 'https' data = str(node) property_url = 'https://www.canfar.phys.uvic.ca/vospace/nodeprops' endpoints_mock = Mock() endpoints_mock.properties = property_url client.get_endpoints = Mock(return_value=endpoints_mock) result = client.update(node, False) self.assertEqual(result, 0) client.conn.session.post.assert_called_with( 'https://www.canfar.phys.uvic.ca/vospace', data=data, allow_redirects=False) call1 = call(property_url, allow_redirects=False, data=data, headers={'Content-type': 'text/xml'}) call2 = call('https://www.canfar.phys.uvic.ca/vospace/phase', allow_redirects=False, data="PHASE=RUN", headers={'Content-type': "text/text"}) calls = [call1, call2] client.conn = Mock(spec=vos.Connection) client.conn.session.post = Mock(return_value=resp) result = client.update(node, True) self.assertEqual(result, 0) client.conn.session.post.assert_has_calls(calls)
def test_create(self): uri = 'vos://create.vospace.auth!vospace/bar' client = Client() node = Node(client.fix_uri(uri)) node2 = Node(str(node)) self.assertEqual(node, node2) data = str(node) headers = {'size': str(len(data))} client = Client() # client.get_node_url = Mock(return_value='http://foo.com/bar') session_mock = MagicMock() client.get_session = Mock(return_value=session_mock) session_mock.put.return_value = Mock(content=str(node)) result = client.create(uri) self.assertEqual(node, result) session_mock.put.assert_called_with( 'http://www.canfar.phys.uvic.ca/vospace/nodes/bar', headers=headers, data=data)
def test_add_props(self): old_node = Node(ElementTree.fromstring(NODE_XML)) old_node.uri = 'vos:sometest' new_node = Node(ElementTree.fromstring(NODE_XML)) new_node.props['quota'] = '1000' new_node.create = Mock(return_value=new_node.node) data = str(new_node) headers = {'size': str(len(data))} client = Client() client.get_node = Mock(return_value=old_node) client.get_node_url = Mock(return_value='http://foo.com/bar') mock_session = Mock() client.get_session = Mock(return_value=mock_session) client.add_props(new_node) client.get_session.assert_called_with('vos://foo.com!vospace/bar') mock_session.post.assert_called_with('http://foo.com/bar', headers=headers, data=data)
def _run_vo(): """ Uses a VOS listdir to identify the work to be done. :return 0 if successful, -1 if there's any sort of failure. Return status is used by airflow for task instance management and reporting. """ config, clients, name_builder, metadata_reader = _common() vos_client = Client(vospace_certfile=config.proxy_file_name) clients.vo_client = vos_client source = data_source.DAOVaultDataSource( config, clients.vo_client, clients.data_client ) store_transferrer = transfer.VoFitsCleanupTransfer(vos_client, config) return rc.run_by_todo( name_builder=name_builder, meta_visitors=META_VISITORS, data_visitors=DATA_VISITORS, source=source, clients=clients, store_transfer=store_transferrer, metadata_reader=metadata_reader, )
def test_glob(self): # test the pattern matches in directories and file names # simple test for listing of directory, no wild cards # NOTE: Mock class also has a 'name' attribute so we cannot # instantiate a mock node with Mock(name='blah'). There are # two other wasy to do it as seen below mock_node = MagicMock(type='vos:ContainerNode') mock_node.configure_mock(name='anode') client = Client() client.get_node = Mock(return_value=mock_node) self.assertEquals(['vos:/anode/'], client.glob('vos:/anode/')) # simple test for file listing of file mock_node = MagicMock(type='vos:DataNode') mock_node.configure_mock(name='afile') client = Client() client.get_node = Mock(return_value=mock_node) self.assertEquals(['vos:/afile'], client.glob('vos:/afile')) # create a mock directory structure on the form # /anode/abc /anode/def - > anode/a* should return # /anode/adc mock_node = MagicMock(type='vos:ContainerNode') mock_node.configure_mock(name='anode') mock_child_node1 = Mock(type='vos:DataNode') mock_child_node1.name = 'abc' mock_child_node2 = Mock(type='vos:DataNode') mock_child_node2.name = 'def' # because we use wild characters in the root node, # we need to create a corresponding node for the base node mock_base_node = Mock(type='vos:ContainerNode') mock_base_node.name = 'vos:' mock_base_node.node_list = [mock_node] mock_node.node_list = [mock_base_node, mock_child_node1, mock_child_node2] client = Client() client.get_node = Mock( side_effect=[mock_node, mock_base_node, mock_node]) self.assertEquals(['vos:/anode/abc'], client.glob('vos:/anode/a*')) self.assertEquals(['vos:/anode/abc'], client.glob('vos:/*node/abc')) # test nodes: # /anode/.test1 /bnode/sometests /bnode/blah # /[a,c]node/*test* should return /bnode/somtests (.test1 is filtered # out as a special file) mock_node1 = MagicMock(type='vos:ContainerNode') mock_node1.configure_mock(name='anode') mock_node1.node_list = [mock_child_node1] mock_child_node2 = Mock(type='vos:DataNode') mock_child_node2.name = 'sometests' mock_child_node3 = Mock(type='vos:DataNode') mock_child_node3.name = 'blah' mock_node2 = MagicMock(type='vos:ContainerNode') mock_node2.configure_mock(name='bnode') mock_node2.node_list = [mock_child_node2, mock_child_node3] # because we use wild characters in the root node, # we need to create a corresponding node for the base node mock_base_node = Mock(type='vos:DataNode') mock_base_node.name = 'vos:' mock_base_node.node_list = [mock_node1, mock_node2] client = Client() client.get_node = Mock( side_effect=[mock_base_node, mock_node1, mock_node2]) self.assertEquals(['vos:/bnode/sometests'], client.glob('vos:/[a,b]node/*test*'))
def test_copy(self, computed_md5_mock): # the md5sum of the file being copied md5sum = 'd41d8cd98f00b204e9800998ecf84eee' # patch the compute_md5 function in vos to return the above value computed_md5_mock.return_value = md5sum # mock the props of the corresponding node props = MagicMock() props.get.return_value = md5sum # add props to the mocked node node = MagicMock(spec=Node) node.props = props # mock one by one the chain of connection.session.response.headers conn = MagicMock(spec=Connection) session = MagicMock() response = MagicMock() headers = MagicMock() headers.get.return_value = md5sum response.headers = headers session.get.return_value = response conn.session = session test_client = Client() # use the mocked connection instead of the real one test_client.conn = conn get_node_url_mock = Mock( return_value=['http://cadc.ca/test', 'http://cadc.ca/test']) test_client.get_node_url = get_node_url_mock mock_update = Mock() test_client.update = mock_update # patch Client.get_node to return our mocked node get_node_mock = Mock(return_value=node) test_client.get_node = get_node_mock # time to test... vospaceLocation = 'vos://test/foo' osLocation = '/tmp/foo' if os.path.isfile(osLocation): os.remove(osLocation) # copy from vospace test_client.copy(vospaceLocation, osLocation) get_node_url_mock.assert_called_once_with(vospaceLocation, method='GET', cutout=None, view='data') computed_md5_mock.assert_called_once_with(osLocation) assert get_node_mock.called # repeat - local file and vospace file are now the same -> only # get_node is called to get the md5 of remote file get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() get_node_mock.reset_mock() test_client.copy(vospaceLocation, osLocation) assert not get_node_url_mock.called computed_md5_mock.assert_called_once_with(osLocation) get_node_mock.assert_called_once_with(vospaceLocation) # change the content of local files to trigger a new copy get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() computed_md5_mock.side_effect = ['d002233', md5sum] get_node_mock.reset_mock() test_client.copy(vospaceLocation, osLocation) get_node_url_mock.assert_called_once_with(vospaceLocation, method='GET', cutout=None, view='data') computed_md5_mock.assert_called_with(osLocation) get_node_mock.assert_called_once_with(vospaceLocation) # copy to vospace when md5 sums are the same -> only update occurs get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() computed_md5_mock.side_effect = None computed_md5_mock.return_value = md5sum test_client.copy(osLocation, vospaceLocation) mock_update.assert_called_once() assert not get_node_url_mock.called # make md5 different get_node_url_mock.reset_mock() get_node_url_mock.return_value =\ ['http://cadc.ca/test', 'http://cadc.ca/test'] computed_md5_mock.reset_mock() mock_update.reset_mock() props.get.side_effect = ['d00223344', md5sum] test_client.copy(osLocation, vospaceLocation) assert not mock_update.called get_node_url_mock.assert_called_once_with(vospaceLocation, 'PUT') computed_md5_mock.assert_called_once_with(osLocation) # copy 0 size file -> delete and create on client but no bytes # transferred get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() computed_md5_mock.return_value = vos.ZERO_MD5 props.get.side_effect = [md5sum] mock_delete = Mock() mock_create = Mock() test_client.delete = mock_delete test_client.create = mock_create test_client.copy(osLocation, vospaceLocation) mock_create.assert_called_once_with(vospaceLocation) mock_delete.assert_called_once_with(vospaceLocation) assert not get_node_url_mock.called # copy new 0 size file -> reate on client but no bytes transferred get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() mock_delete.reset_mock() mock_create.reset_mock() computed_md5_mock.return_value = vos.ZERO_MD5 props.get.side_effect = [None] mock_delete = Mock() mock_create = Mock() test_client.delete = mock_delete test_client.create = mock_create test_client.copy(osLocation, vospaceLocation) mock_create.assert_called_once_with(vospaceLocation) assert not mock_delete.called assert not get_node_url_mock.called # error tests - md5sum mismatch props.get.side_effect = [md5sum] computed_md5_mock.return_value = '000bad000' with self.assertRaises(OSError): test_client.copy(vospaceLocation, osLocation) with self.assertRaises(OSError): test_client.copy(osLocation, vospaceLocation) # requests just the headers props.get.side_effect = [None] get_node_url_mock = Mock( return_value=['http://cadc.ca/test', 'http://cadc.ca/test']) test_client.get_node_url = get_node_url_mock computed_md5_mock.reset_mock() computed_md5_mock.side_effect = ['d002233', md5sum] get_node_mock.reset_mock() test_client.copy(vospaceLocation, osLocation, head=True) get_node_url_mock.assert_called_once_with(vospaceLocation, method='GET', cutout=None, view='header')
def test_copy(self, computed_md5_mock): # the md5sum of the file being copied md5sum = 'd41d8cd98f00b204e9800998ecf84eee' # patch the compute_md5 function in vos to return the above value computed_md5_mock.return_value = md5sum #mock the props of the corresponding node props = MagicMock() props.get.return_value = md5sum #add props to the mocked node node = MagicMock(spec=Node) node.props = props # mock one by one the chain of connection.session.response.headers conn = MagicMock(spec=Connection) session = MagicMock() response = MagicMock() headers = MagicMock() headers.get.return_value = md5sum response.headers = headers session.get.return_value = response conn.session = session test_client = Client() # use the mocked connection instead of the real one test_client.conn = conn get_node_url_mock = Mock( return_value=['http://cadc.ca/test', 'http://cadc.ca/test']) test_client.get_node_url = get_node_url_mock #patch Client.get_node to return our mocked node get_node_mock = Mock(return_value=node) test_client.get_node = get_node_mock # time to test... vospaceLocation = 'vos://test/foo' osLocation = '/tmp/foo' # copy from vospace test_client.copy(vospaceLocation, osLocation) get_node_url_mock.assert_called_once_with(vospaceLocation, method='GET', cutout=None, view='data') computed_md5_mock.assert_called_once_with(osLocation) get_node_mock.assert_called_once_with(vospaceLocation) # copy to vospace get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() test_client.copy(osLocation, vospaceLocation) get_node_url_mock.assert_called_once_with(vospaceLocation, 'PUT') computed_md5_mock.assert_called_once_with(osLocation) # error tests - md5sum mismatch computed_md5_mock.return_value = '000bad000' with self.assertRaises(OSError): test_client.copy(vospaceLocation, osLocation) with self.assertRaises(OSError): test_client.copy(osLocation, vospaceLocation)
def test_glob(self): # test the pattern matches in directories and file names # simple test for listing of directory, no wild cards # NOTE: Mock class also has a 'name' attribute so we cannot # instantiate a mock node with Mock(name='blah'). There are # two other wasy to do it as seen below mock_node = MagicMock(type='vos:ContainerNode') mock_node.configure_mock(name='anode') client = Client() client.get_node = Mock(return_value=mock_node) self.assertEquals(['vos:/anode/'], client.glob('vos:/anode/')) # simple test for file listing of file mock_node = MagicMock(type='vos:DataNode') mock_node.configure_mock(name='afile') client = Client() client.get_node = Mock(return_value=mock_node) self.assertEquals(['vos:/afile'], client.glob('vos:/afile')) # create a mock directory structure on the form # /anode/abc /anode/def - > anode/a* should return # /anode/adc mock_node = MagicMock(type='vos:ContainerNode') mock_node.configure_mock(name='anode') mock_child_node1 = Mock(type='vos:DataNode') mock_child_node1.name = 'abc' mock_child_node2 = Mock(type='vos:DataNode') mock_child_node2.name = 'def' # because we use wild characters in the root node, # we need to create a corresponding node for the base node mock_base_node = Mock(type='vos:ContainerNode') mock_base_node.name = 'vos:' mock_base_node.node_list = [mock_node] mock_node.node_list = [ mock_base_node, mock_child_node1, mock_child_node2 ] client = Client() client.get_node = Mock( side_effect=[mock_node, mock_base_node, mock_node]) self.assertEquals(['vos:/anode/abc'], client.glob('vos:/anode/a*')) self.assertEquals(['vos:/anode/abc'], client.glob('vos:/*node/abc')) # test nodes: # /anode/.test1 /bnode/sometests /bnode/blah # /[a,c]node/*test* should return /bnode/somtests (.test1 is filtered # out as a special file) mock_child_node1 = Mock(type='vos:DataNode') mock_child_node1.name = '.test1' mock_node1 = MagicMock(type='vos:ContainerNode') mock_node1.configure_mock(name='anode') mock_node1.node_list = [mock_child_node1] mock_child_node2 = Mock(type='vos:DataNode') mock_child_node2.name = 'sometests' mock_child_node3 = Mock(type='vos:DataNode') mock_child_node3.name = 'blah' mock_node2 = MagicMock(type='vos:ContainerNode') mock_node2.configure_mock(name='bnode') mock_node2.node_list = [mock_child_node2, mock_child_node3] # because we use wild characters in the root node, # we need to create a corresponding node for the base node mock_base_node = Mock(type='vos:DataNode') mock_base_node.name = 'vos:' mock_base_node.node_list = [mock_node1, mock_node2] client = Client() client.get_node = Mock( side_effect=[mock_base_node, mock_node1, mock_node2]) self.assertEquals(['vos:/bnode/sometests'], client.glob('vos:/[a,b]node/*test*'))
def test_nodetype(self): mock_node = MagicMock(id=333) mock_node.type = 'vos:ContainerNode' client = Client() client.get_node = Mock(return_value=mock_node) self.assertEquals('vos:ContainerNode', client._node_type('vos:/somenode')) self.assertTrue(client.isdir('vos:/somenode')) mock_node.type = 'vos:DataNode' self.assertEquals('vos:DataNode', client._node_type('vos:/somenode')) self.assertTrue(client.isfile('vos:/somenode')) # through a link mock_node.type = 'vos:ContainerNode' mock_link_node = Mock(type='vos:LinkNode') mock_link_node.target = 'vos:/somefile' client.get_node = Mock( side_effect=[mock_link_node, mock_node, mock_link_node, mock_node]) self.assertEquals('vos:ContainerNode', client._node_type('vos:/somenode')) self.assertTrue(client.isdir('vos:/somenode')) # through an external link - not sure why the type is DataNode in this case??? mock_link_node.target = '/somefile' client.get_node = Mock(side_effect=[mock_link_node, mock_link_node]) self.assertEquals('vos:DataNode', client._node_type('vos:/somenode')) self.assertTrue(client.isfile('vos:/somenode'))
def test_copy(self, computed_md5_mock): file_content = 'File content'.encode('utf-8') # the md5sum of the file being copied transfer_md5 = hashlib.md5() transfer_md5.update(file_content) md5sum = transfer_md5.hexdigest() # patch the compute_md5 function in vos to return the above value computed_md5_mock.return_value = md5sum # mock the props of the corresponding node props = MagicMock() props.get.return_value = md5sum # add props to the mocked node node = MagicMock(spec=Node) node.props = {'MD5': md5sum, 'length': 12} # mock one by one the chain of connection.session.response.headers session = MagicMock() response = MagicMock() headers = MagicMock() headers.get.return_value = md5sum response.headers = headers session.get.return_value = response response.iter_content.return_value = BytesIO(file_content) test_client = Client() test_client.get_session = Mock(return_value=session) # use the mocked connection instead of the real one get_node_url_mock = Mock( return_value=['http://cadc.ca/test', 'http://cadc.ca/test']) test_client.get_node_url = get_node_url_mock mock_update = Mock() test_client.update = mock_update # patch Client.get_node to return our mocked node get_node_mock = Mock(return_value=node) test_client.get_node = get_node_mock # time to test... vospaceLocation = 'vos://test/foo' osLocation = '/tmp/foo' if os.path.isfile(osLocation): os.remove(osLocation) # copy from vospace test_client.copy(vospaceLocation, osLocation) get_node_url_mock.assert_called_once_with(vospaceLocation, method='GET', cutout=None, view='data') assert not computed_md5_mock.called,\ 'MD5 should be computed on the fly' assert get_node_mock.called # repeat - local file and vospace file are now the same -> only # get_node is called to get the md5 of remote file get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() get_node_mock.reset_mock() props.reset_mock() props.get.return_value = md5sum test_client.copy(vospaceLocation, osLocation) assert not get_node_url_mock.called computed_md5_mock.assert_called_once_with(osLocation) get_node_mock.assert_called_once_with(vospaceLocation, force=True) # change the content of local files to trigger a new copy get_node_url_mock.reset_mock() get_node_mock.reset_mock() computed_md5_mock.reset_mock() computed_md5_mock.return_value = 'd002233' response.iter_content.return_value = BytesIO(file_content) test_client.copy(vospaceLocation, osLocation) get_node_url_mock.assert_called_once_with(vospaceLocation, method='GET', cutout=None, view='data') computed_md5_mock.assert_called_with(osLocation) get_node_mock.assert_called_once_with(vospaceLocation, force=True) # change the content of local files to trigger a new copy get_node_url_mock.reset_mock() get_node_url_mock.return_value = \ ['https://mysite.com/node/node123/cutout'] computed_md5_mock.reset_mock() computed_md5_mock.return_value = 'd002233' # computed_md5_mock.side_effect = ['d002233', md5sum] get_node_mock.reset_mock() response.iter_content.return_value = BytesIO(file_content) session.get.return_value = response test_client.get_session = Mock(return_value=session) test_client.copy('{}{}'.format(vospaceLocation, '[1][10:60]'), osLocation) get_node_url_mock.assert_called_once_with( vospaceLocation, method='GET', cutout='[1][10:60]', view='cutout') # copy to vospace when md5 sums are the same -> only update occurs get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() computed_md5_mock.side_effect = None computed_md5_mock.return_value = md5sum test_client.copy(osLocation, vospaceLocation) mock_update.assert_called_once() assert not get_node_url_mock.called # make md5 different on destination get_node_url_mock.reset_mock() get_node_url_mock.return_value =\ ['http://cadc.ca/test', 'http://cadc.ca/test'] computed_md5_mock.reset_mock() mock_update.reset_mock() computed_md5_mock.return_value = md5sum to_update_node = MagicMock() to_update_node.props = {'MD5': 'abcde', 'length': 12} test_client.get_node = Mock(side_effect=[to_update_node, node]) test_client.copy(osLocation, vospaceLocation) assert not mock_update.called get_node_url_mock.assert_called_once_with(vospaceLocation, 'PUT', content_length=12, md5_checksum='abcde') computed_md5_mock.assert_called_once_with(osLocation) # copy 0 size file -> delete and create node but no bytes # transferred get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() test_client.get_node = Mock(return_value=node) node.props['length'] = 0 mock_delete = Mock() mock_create = Mock() test_client.delete = mock_delete test_client.create = mock_create with patch('vos.vos.os.stat', Mock()) as stat_mock: stat_mock.return_value = Mock(st_size=0) test_client.copy(osLocation, vospaceLocation) mock_create.assert_called_once_with(vospaceLocation) mock_delete.assert_called_once_with(vospaceLocation) assert not get_node_url_mock.called # copy new 0 size file -> create node but no bytes transferred get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() mock_delete.reset_mock() mock_create.reset_mock() test_client.get_node = Mock(side_effect=[Exception(), node]) mock_delete = Mock() mock_create = Mock() test_client.delete = mock_delete test_client.create = mock_create with patch('vos.vos.os.stat', Mock()) as stat_mock: stat_mock.return_value = Mock(st_size=0) test_client.copy(osLocation, vospaceLocation) mock_create.assert_called_once_with(vospaceLocation) assert not mock_delete.called assert not get_node_url_mock.called # error tests - md5sum mismatch node.props['length'] = 12 computed_md5_mock.return_value = '000bad000' test_client.get_node = Mock(return_value=node) with self.assertRaises(OSError): test_client.copy(vospaceLocation, osLocation) # existing file mock_delete.reset_mock() with self.assertRaises(OSError): with patch('vos.vos.os.stat', Mock()) as stat_mock: stat_mock.return_value = Mock(st_size=12) test_client.copy(osLocation, vospaceLocation) assert not mock_delete.called # server takes care of cleanup # new file mock_delete.reset_mock() with self.assertRaises(OSError): with patch('vos.vos.os.stat', Mock()) as stat_mock: stat_mock.return_value = Mock(st_size=12) node.props['MD5'] = None test_client.copy(osLocation, vospaceLocation) assert mock_delete.called # cleanup required # requests just the headers when md5 not provided in the header props.get.side_effect = [None] get_node_url_mock = Mock( return_value=['http://cadc.ca/test', 'http://cadc.ca/test']) test_client.get_node_url = get_node_url_mock get_node_mock.reset_mock() headers.get.return_value = None test_client.copy(vospaceLocation, osLocation, head=True) get_node_url_mock.assert_called_once_with(vospaceLocation, method='GET', cutout=None, view='header') # repeat headers request when md5 provided in the header props.get.side_effect = md5sum get_node_url_mock = Mock( return_value=['http://cadc.ca/test', 'http://cadc.ca/test']) test_client.get_node_url = get_node_url_mock get_node_mock.reset_mock() response.iter_content.return_value = BytesIO(file_content) headers.get.return_value = None test_client.copy(vospaceLocation, osLocation, head=True) get_node_url_mock.assert_called_once_with(vospaceLocation, method='GET', cutout=None, view='header')
def test_open(self): # Invalid mode raises OSError with self.assertRaises(OSError): client = Client() client.open('vos://foo/bar', mode=-1) with self.assertRaises(OSError): client = Client() client.get_node_url = Mock(return_value=None) client.open(None, url=None) conn = Connection() mock_vofile = VOFile(['http://foo.com/bar'], conn, 'GET') client = Client() client.get_node_url = Mock(return_value=mock_vofile) vofile = client.open(None, url=None) self.assertEquals(vofile.url.URLs[0], 'http://foo.com/bar')
def test_nodetype(self): mock_node = MagicMock(id=333) mock_node.type = 'vos:ContainerNode' client = Client() client.get_node = Mock(return_value=mock_node) self.assertEquals('vos:ContainerNode', client._node_type('vos:/somenode')) self.assertTrue(client.isdir('vos:/somenode')) mock_node.type = 'vos:DataNode' self.assertEquals('vos:DataNode', client._node_type('vos:/somenode')) self.assertTrue(client.isfile('vos:/somenode')) # through a link mock_node.type = 'vos:ContainerNode' mock_link_node = Mock(type='vos:LinkNode') mock_link_node.target = 'vos:/somefile' client.get_node = Mock(side_effect=[mock_link_node, mock_node, mock_link_node, mock_node]) self.assertEquals('vos:ContainerNode', client._node_type('vos:/somenode')) self.assertTrue(client.isdir('vos:/somenode')) # through an external link - not sure why the type is DataNode in # this case??? mock_link_node.target = '/somefile' client.get_node = Mock(side_effect=[mock_link_node, mock_link_node]) self.assertEquals('vos:DataNode', client._node_type('vos:/somenode')) self.assertTrue(client.isfile('vos:/somenode'))
from vos import Client c = Client() #for expnum in c.listdir('vos:cfis/solar_system/dbimages', force=True): for expnum in open('expnum_list.txt').readlines(): expnum = expnum.strip() n = c.get_node('vos:cfis/solar_system/dbimages/{}'.format(expnum), force=True) print n.uri for prop in n.props: if 'stationary' in prop: n.props[prop] = None c.add_props(n) print("Done")
def test_init_client(self): # No parameters uses cert in ~/.ssl giving authenticated / https # create a fake pem file certfile = '/tmp/some-cert-file.pem' open(certfile, 'w+') Client.VOSPACE_CERTFILE = "some-cert-file.pem" with patch('os.access'): client = Client(vospace_certfile=certfile) client.get_session(uri='ivo://cadc.nrc.ca/vault') conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn self.assertTrue(conn.subject.certificate) self.assertFalse(conn.vo_token) # Supplying an empty string for certfile implies anonymous / http client = Client(vospace_certfile='') client.get_session(uri='ivo://cadc.nrc.ca/vault') conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn self.assertTrue(conn.subject.anon) self.assertFalse(conn.vo_token) # Specifying a token implies authenticated / http client = Client(vospace_token='a_token_string') client.get_session(uri='ivo://cadc.nrc.ca/vault') conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn self.assertTrue(conn.subject.anon) self.assertTrue(conn.vo_token) # Specifying both a certfile and token implies token (auth) / http with patch('os.access'): client = Client(vospace_certfile=certfile, vospace_token='a_token_string') client.get_session(uri='ivo://cadc.nrc.ca/vault') conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn self.assertTrue(conn.subject.anon) self.assertTrue(conn.vo_token) # update auth for specific service with patch('os.access'): client.set_auth(uri='ivo://cadc.nrc.ca/vault', vospace_certfile=certfile) conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn self.assertTrue(conn.subject.certificate) self.assertFalse(conn.vo_token)
def test_open(self): # Invalid mode raises OSError with self.assertRaises(OSError): client = Client() client.open('vos://foo/bar', mode=-10000) with self.assertRaises(OSError): client = Client() client.get_node_url = Mock(return_value=None) client.open(None, url=None) conn = Connection(resource_id='ivo://cadc.nrc.ca/vault') mock_vofile = VOFile(['http://foo.com/bar'], conn, 'GET') client = Client() client.get_node_url = Mock(return_value=mock_vofile) endpoints_mock = Mock(conn=conn) client.get_endpoints = Mock(return_value=endpoints_mock) vofile = client.open(None, url=None) self.assertEqual(vofile.url.URLs[0], 'http://foo.com/bar')
#!python from multiprocessing import Pool from ossos import mop_file, storage import os from astropy.io import fits import numpy from vos import Client import sys from ossos import junk_keywords from ossos import util import argparse import logging client = Client() RA = 'RA_J2000' DEC = 'DE_J2000' del_keyword_list = junk_keywords.bad_keywords_list task = 'slow' dependency = 'step1' def run(expnum, ccd, version='p', prefix='', dry_run=False, force=False): message = 'success' if storage.get_status(task, prefix, expnum, version=version, ccd=ccd) and not force: logging.info("{} completed successfully for {} {} {} {}".format( task, prefix, expnum, version, ccd)) return
def main(orbit=None, **kwargs): """ This is the driver program. Gets the images from VOSpace or the local filesystem. expected kwargs: pointing, index, chip, rate, angle, ra, dec, p_name, discovery, nstk, zpt, dbimages :param kwargs: these are the 'args' sent on the commandline or in an input file. :param orbit: the orbit of the object that should be in the frames reference by kwargs provided. :type orbit: BKOrbit :return: list of observations :rtype list(ObsRecod) """ pointing = kwargs['pointing'] chip = kwargs['chip'] index = kwargs['index'] rate = kwargs['rate'] angle = kwargs['angle'] ra = kwargs['ra'] dec = kwargs['dec'] p_name = kwargs['p_name'] discovery = kwargs['discovery'] nstk = kwargs['nstk'] rejected = kwargs.get('rejected', False) zpt = kwargs.get('zpt', 26.9) dbimages = kwargs.get('dbimages', 'vos:NewHorizons/dbimages/') client = Client() int_rate = int(rate * 10) int_angle = int((angle % 360) * 10) images = [] # Load the 3 images associated with this point/chip/rate/angle set. epoch = None for idx in range(nstk): expnum = f'{int(pointing)}{int_rate:02d}{int_angle:04d}{idx}' image = f'{expnum}p{chip:02d}.fits' url = f'{dbimages}/{pointing:05d}/{chip:03d}/{index:04d}/{image}' logging.info(f"Looking for image at {url}") try: if os.access(url, os.R_OK): image = url elif not os.access(image, os.R_OK): # get from VOSpace is not already on disk client.copy(url, image) except Exception as ex: logging.error(str(ex)) # Return empty set on VOSpace copy error. return {} images.append(image) epoch = Time(fits.open(images[len(images) // 2])[0].header['DATE-AVG'], scale='tai').utc regions = f'{dbimages}/{pointing:05d}.reg' try: if not os.access(regions, os.R_OK): regions = client.copy(regions, '.', disposition=True) except Exception as ex: logging.debug(f"{ex}") regions = None wcs_dict = {} if orbit is not None: epoch = orbit.epoch.mjd if epoch is None: epoch = Time("2020-05-22T00:00:00") epoch = Time(kwargs.get('epoch', epoch), format='mjd') load_images(images, ra, dec, wcs_dict, orbit, dra=rate * math.cos(math.radians(angle)), ddec=rate * math.sin(math.radians(angle)), regions=regions, rejected=rejected, basedate=epoch) obs = measure_image(p_name, images, wcs_dict, discovery=discovery, zpt=zpt) return obs