Ejemplo n.º 1
0
    def test_init_client(self):
        # No parameters uses cert in ~/.ssl giving authenticated / https
        # create a fake pem file
        certfile = '/tmp/some-cert-file.pem'
        open(certfile, 'w+')
        Client.VOSPACE_CERTFILE = "some-cert-file.pem"
        with patch('os.access'):
            client = Client(vospace_certfile=certfile)
        self.assertTrue(client.conn.subject.certificate)
        self.assertFalse(client.conn.vo_token)

        # Supplying an empty string for certfile implies anonymous / http
        client = Client(vospace_certfile='')
        self.assertTrue(client.conn.subject.anon)
        self.assertFalse(client.conn.vo_token)

        # Specifying a token implies authenticated / http
        client = Client(vospace_token='a_token_string')
        self.assertTrue(client.conn.subject.anon)
        self.assertTrue(client.conn.vo_token)

        # Specifying both a certfile and token implies token (auth) / http
        with patch('os.access'):
            client = Client(vospace_certfile=certfile,
                            vospace_token='a_token_string')
        self.assertTrue(client.conn.subject.anon)
        self.assertTrue(client.conn.vo_token)
Ejemplo n.º 2
0
    def test_nodetype(self):
        mock_node = MagicMock(id=333)
        mock_node.type = 'vos:ContainerNode'
        client = Client()
        client.get_node = Mock(return_value=mock_node)
        self.assertEquals('vos:ContainerNode',
                          client._node_type('vos:/somenode'))
        self.assertTrue(client.isdir('vos:/somenode'))

        mock_node.type = 'vos:DataNode'
        self.assertEquals('vos:DataNode', client._node_type('vos:/somenode'))
        self.assertTrue(client.isfile('vos:/somenode'))

        # through a link
        mock_node.type = 'vos:ContainerNode'
        mock_link_node = Mock(type='vos:LinkNode')
        mock_link_node.target = 'vos:/somefile'
        client.get_node = Mock(
            side_effect=[mock_link_node, mock_node, mock_link_node, mock_node])
        self.assertEquals('vos:ContainerNode',
                          client._node_type('vos:/somenode'))
        self.assertTrue(client.isdir('vos:/somenode'))

        # through an external link - not sure why the type is DataNode in this case???
        mock_link_node.target = '/somefile'
        client.get_node = Mock(side_effect=[mock_link_node, mock_link_node])
        self.assertEquals('vos:DataNode', client._node_type('vos:/somenode'))
        self.assertTrue(client.isfile('vos:/somenode'))
Ejemplo n.º 3
0
    def test_update(self):
        node = Node(ElementTree.fromstring(NODE_XML))

        resp = Mock()
        resp.headers.get = Mock(return_value="https://www.canfar.phys.uvic.ca/vospace")

        client = Client()
        client.get_node_url = Mock(return_value='https://www.canfar.phys.uvic.ca/vospace')
        client.conn = Mock()
        client.conn.session.post = Mock(return_value=resp)
        client.get_transfer_error = Mock()
        client.protocol = 'https'

        data = str(node)
        property_url = 'https://www.canfar.phys.uvic.ca/vospace/nodeprops'
        result = client.update(node, False)
        self.assertEqual(result, 0)
        client.conn.session.post.assert_called_with('https://www.canfar.phys.uvic.ca/vospace',
                                                      data=data, allow_redirects=False)

        call1 = call(property_url, allow_redirects=False, data=data,
                     headers={'Content-type': 'text/xml'})
        call2 = call('https://www.canfar.phys.uvic.ca/vospace/phase', allow_redirects=False, data="PHASE=RUN",
                     headers={'Content-type': "text/text"})
        calls = [call1, call2]

        client.conn = Mock()
        client.conn.session.post = Mock(return_value=resp)
        with patch('vos.vos.EndPoints.properties', property_url):
            result = client.update(node, True)
        self.assertEqual(result, 0)
        client.conn.session.post.assert_has_calls(calls)
Ejemplo n.º 4
0
    def test_copy(self, computed_md5_mock):
        # the md5sum of the file being copied
        md5sum = 'd41d8cd98f00b204e9800998ecf84eee'
        # patch the compute_md5 function in vos to return the above value
        computed_md5_mock.return_value = md5sum

        #mock the props of the corresponding node
        props = MagicMock()
        props.get.return_value = md5sum
        #add props to the mocked node
        node = MagicMock(spec=Node)
        node.props = props

        # mock one by one the chain of connection.session.response.headers
        conn = MagicMock(spec=Connection)
        session = MagicMock()
        response = MagicMock()
        headers = MagicMock()
        headers.get.return_value = md5sum
        response.headers = headers
        session.get.return_value = response
        conn.session = session

        test_client = Client()
        # use the mocked connection instead of the real one
        test_client.conn = conn
        get_node_url_mock = Mock(
            return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
        test_client.get_node_url = get_node_url_mock

        #patch Client.get_node to return our mocked node
        get_node_mock = Mock(return_value=node)
        test_client.get_node = get_node_mock

        # time to test...
        vospaceLocation = 'vos://test/foo'
        osLocation = '/tmp/foo'
        # copy from vospace
        test_client.copy(vospaceLocation, osLocation)
        get_node_url_mock.assert_called_once_with(vospaceLocation,
                                                  method='GET',
                                                  cutout=None,
                                                  view='data')
        computed_md5_mock.assert_called_once_with(osLocation)
        get_node_mock.assert_called_once_with(vospaceLocation)

        # copy to vospace
        get_node_url_mock.reset_mock()
        computed_md5_mock.reset_mock()
        test_client.copy(osLocation, vospaceLocation)
        get_node_url_mock.assert_called_once_with(vospaceLocation, 'PUT')
        computed_md5_mock.assert_called_once_with(osLocation)

        # error tests - md5sum mismatch
        computed_md5_mock.return_value = '000bad000'
        with self.assertRaises(OSError):
            test_client.copy(vospaceLocation, osLocation)

        with self.assertRaises(OSError):
            test_client.copy(osLocation, vospaceLocation)
Ejemplo n.º 5
0
    def test_open(self):
        # Invalid mode raises OSError
        with self.assertRaises(OSError):
            client = Client()
            client.open('vos://foo/bar', mode=-1)

        with self.assertRaises(OSError):
            client = Client()
            client.get_node_url = Mock(return_value=None)
            client.open(None, url=None)

        conn = Connection()
        mock_vofile = VOFile(['http://foo.com/bar'], conn, 'GET')
        client = Client()
        client.get_node_url = Mock(return_value=mock_vofile)
        vofile = client.open(None, url=None)
        self.assertEquals(vofile.url.URLs[0], 'http://foo.com/bar')
Ejemplo n.º 6
0
    def test_init_client(self):
        # No parameters uses cert in ~/.ssl giving authenticated / https
        # create a fake pem file
        certfile = '/tmp/some-cert-file.pem'
        open(certfile, 'w+')
        Client.VOSPACE_CERTFILE = "some-cert-file.pem"
        with patch('os.access'):
            client = Client(vospace_certfile=certfile)
        client.get_session(uri='ivo://cadc.nrc.ca/vault')
        conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn
        self.assertTrue(conn.subject.certificate)
        self.assertFalse(conn.vo_token)

        # Supplying an empty string for certfile implies anonymous / http
        client = Client(vospace_certfile='')
        client.get_session(uri='ivo://cadc.nrc.ca/vault')
        conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn
        self.assertTrue(conn.subject.anon)
        self.assertFalse(conn.vo_token)

        # Specifying a token implies authenticated / http
        client = Client(vospace_token='a_token_string')
        client.get_session(uri='ivo://cadc.nrc.ca/vault')
        conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn
        self.assertTrue(conn.subject.anon)
        self.assertTrue(conn.vo_token)

        # Specifying both a certfile and token implies token (auth) / http
        with patch('os.access'):
            client = Client(vospace_certfile=certfile,
                            vospace_token='a_token_string')
        client.get_session(uri='ivo://cadc.nrc.ca/vault')
        conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn
        self.assertTrue(conn.subject.anon)
        self.assertTrue(conn.vo_token)

        # update auth for specific service
        with patch('os.access'):
            client.set_auth(uri='ivo://cadc.nrc.ca/vault',
                            vospace_certfile=certfile)
        conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn
        self.assertTrue(conn.subject.certificate)
        self.assertFalse(conn.vo_token)
Ejemplo n.º 7
0
 def test_delete(self):
     certfile = '/tmp/SomeCert.pem'
     open(certfile, 'w+')
     with patch('os.access'):
         client = Client(vospace_certfile=certfile)
     uri1 = 'vos://cadc.nrc.ca!vospace/nosuchfile1'
     url = 'https://www.canfar.phys.uvic.ca/vospace/nodes/nosuchfile1'
     client.conn.session.delete = Mock()
     client.delete(uri1)
     client.conn.session.delete.assert_called_once_with(url)
Ejemplo n.º 8
0
    def test_open(self):
        # Invalid mode raises OSError
        with self.assertRaises(OSError):
            client = Client()
            client.open('vos://foo/bar', mode=-10000)

        with self.assertRaises(OSError):
            client = Client()
            client.get_node_url = Mock(return_value=None)
            client.open(None, url=None)

        conn = Connection(resource_id='ivo://cadc.nrc.ca/vault')
        mock_vofile = VOFile(['http://foo.com/bar'], conn, 'GET')
        client = Client()
        client.get_node_url = Mock(return_value=mock_vofile)
        endpoints_mock = Mock(conn=conn)
        client.get_endpoints = Mock(return_value=endpoints_mock)
        vofile = client.open(None, url=None)
        self.assertEqual(vofile.url.URLs[0], 'http://foo.com/bar')
Ejemplo n.º 9
0
    def test_create(self):
        uri = 'vos://create.vospace.auth!vospace/bar'
        client = Client()
        node = Node(client.fix_uri(uri))
        node2 = Node(str(node))
        self.assertEqual(node, node2)
        data = str(node)
        headers = {'size': str(len(data))}

        client = Client()
        # client.get_node_url = Mock(return_value='http://foo.com/bar')
        session_mock = MagicMock()
        client.get_session = Mock(return_value=session_mock)
        session_mock.put.return_value = Mock(content=str(node))

        result = client.create(uri)
        self.assertEqual(node, result)
        session_mock.put.assert_called_with(
            'http://www.canfar.phys.uvic.ca/vospace/nodes/bar',
            headers=headers, data=data)
Ejemplo n.º 10
0
    def test_transfer_error(self, mock_vofile):
        vofile = MagicMock()
        mock_vofile.return_value = vofile

        vospace_url = 'https://somevospace.server/vospace'
        session = Mock()
        session.get.side_effect = [Mock(content='COMPLETED')] 
        conn = Mock(spec=Connection)
        conn.session = session

        test_client = Client()

        # use the mocked connection instead of the real one
        test_client.conn = conn
        
        # job successfully completed
        vofile.read.side_effect = [b'QUEUED', b'COMPLETED']
        self.assertFalse(test_client.get_transfer_error(
                vospace_url +'/results/transferDetails', 'vos://vospace'))
        session.get.assert_called_once_with(vospace_url + '/phase', allow_redirects=False)
        
        # job suspended
        session.reset_mock()
        session.get.side_effect = [Mock(content=b'COMPLETED')]
        vofile.read.side_effect = [b'QUEUED', b'SUSPENDED']
        with self.assertRaises(OSError):
            test_client.get_transfer_error(
                vospace_url +'/results/transferDetails', 'vos://vospace')
        #check arguments for session.get calls
        self.assertEquals([call(vospace_url + '/phase', allow_redirects=False)], 
                          session.get.call_args_list )

        # job encountered an internal error
        session.reset_mock()
        vofile.read.side_effect = Mock(side_effect=[b'QUEUED', b'ERROR'])
        session.get.side_effect = [Mock(content=b'COMPLETED'), Mock(text='InternalFault')]
        with self.assertRaises(OSError):
            test_client.get_transfer_error(
                vospace_url +'/results/transferDetails', 'vos://vospace')
        self.assertEquals([call(vospace_url + '/phase', allow_redirects=False),
                           call(vospace_url + '/error')], 
                          session.get.call_args_list )

        # job encountered an unsupported link error
        session.reset_mock()
        link_file = 'testlink.fits'
        vofile.read.side_effect = Mock(side_effect=[b'QUEUED', b'ERROR'])
        session.get.side_effect = [Mock(content=b'COMPLETED'),
                                   Mock(text="Unsupported link target: " + link_file)]
        self.assertEquals(link_file, test_client.get_transfer_error(
            vospace_url +'/results/transferDetails', 'vos://vospace'))
        self.assertEquals([call(vospace_url + '/phase', allow_redirects=False),
                   call(vospace_url + '/error')], 
                  session.get.call_args_list )
Ejemplo n.º 11
0
 def test_get_info_list(self):
     # list tuples of a LinkNode
     mock_node = MagicMock(type='vos:DataNode')
     mock_node.return_value = mock_node
     mock_node.name = 'testnode'
     mock_node.get_info.return_value = {'name': 'aa'}
     mock_link_node = Mock(type='vos:LinkNode')
     mock_link_node.target = 'vos:/somefile'
     client = Client()
     client.get_node = MagicMock(side_effect=[mock_link_node, mock_node])
     self.assertEquals([mock_node],
                       client.get_children_info('vos:/somenode'))
Ejemplo n.º 12
0
 def off_quota(self):
     """
     Test that a 413 raised by the server gets a reasonable error to the user.
     @return:
     """
     with patch('vos.vos.VOFile') as mockVOFile:
         mockVOFile.open = Mock()
         mockVOFile.read = Mock()
         mockVOFile.write = Mock()
     client = Client()
     client.conn = Mock()
     client.transfer(uri='vos:test', direction="pushToVoSpace")
Ejemplo n.º 13
0
    def test_move(self):
        mock_resp_403 = Mock(name="mock_resp_303")
        mock_resp_403.status_code = 403

        conn = Connection()
        conn.session.post = Mock(return_value=mock_resp_403)
        client = Client(conn=conn)

        uri1 = 'notvos://cadc.nrc.ca!vospace/nosuchfile1'
        uri2 = 'notvos://cadc.nrc.ca!vospace/nosuchfile2'

        with self.assertRaises(OSError):
            client.move(uri1, uri2)
Ejemplo n.º 14
0
def rs():
    config = mc.Config()
    config.get_executors()
    vos_client = Client(vospace_certfile=config.proxy_fqn)
    builder = GemProcBuilder(config)
    storage_name = builder.build(sys.argv[1])
    store_transfer = tc.VoFitsTransfer(vos_client)
    rc.run_single(
        config=config,
        storage_name=storage_name,
        command_name='gemProc2caom2',
        meta_visitors=[],
        data_visitors=[provenance_augmentation, preview_augmentation],
        store_transfer=store_transfer,
    )
Ejemplo n.º 15
0
    def test_getNode(self):
        """

        @return:
        """
        uri = "vos://foo.com!vospace/bar"

        nodes = (' <vos:nodes>\n'
                 '<vos:node uri="vos://cadc.nrc.ca!vospace/mydir/file123" '
                 'xs:type="vos:DataNode">\n'
                 '   <vos:properties>\n'
                 '       <vos:property '
                 'uri="ivo://ivoa.net/vospace/core#date">2016-05-10T09:52:13'
                 '</vos:property>\n'
                 '   </vos:properties>\n'
                 '</vos:node>\n'
                 '<vos:node uri="vos://cadc.nrc.ca!vospace/mydir/file456" '
                 'xs:type="vos:DataNode">\n'
                 '   <vos:properties>\n'
                 '       <vos:property uri="ivo://ivoa.net/vospace/core#date">'
                 '2016-05-19T09:52:14</vos:property>\n'
                 '   </vos:properties>\n'
                 '</vos:node>\n'
                 '</vos:nodes>\n')

        mock_vofile = Mock()
        client = Client()
        client.open = Mock(return_value=mock_vofile)

        mock_vofile.read = Mock(
            return_value=NODE_XML.format(uri, '').encode('UTF-8'))
        my_node = client.get_node(uri, limit=0, force=False)
        self.assertEqual(uri, my_node.uri)
        self.assertEqual(len(my_node.node_list), 0)

        mock_vofile.read = Mock(
            return_value=NODE_XML.format(uri, nodes).encode('UTF-8'))

        my_node = client.get_node(uri, limit=2, force=True)
        self.assertEqual(uri, my_node.uri)
        self.assertEqual(len(my_node.node_list), 2)

        my_node = client.get_node(uri, limit=2, force=False)
        self.assertEqual(uri, my_node.uri)
        self.assertEqual(len(my_node.node_list), 2)
Ejemplo n.º 16
0
def bad_exposure_list():
    """
    Retreieve the list of bad exposrures from S. Gwyn's VOSpace listing.
    
    :return: list of bad exposures
     :rtype list
    """

    Client().copy(u'vos:sgwyn/tkBAD', u'tkBAD')

    bad_expnum_list = []
    for lines in open('tkBAD').readlines():
        # noinspection PyBroadException
        try:
            bad_expnum_list.append(lines.split()[0])
        except:
            sys.stderr.write(lines)
    return bad_expnum_list
Ejemplo n.º 17
0
    def test_add_props(self):
        old_node = Node(ElementTree.fromstring(NODE_XML))
        new_node = Node(ElementTree.fromstring(NODE_XML))
        new_node.props['quota'] = '1000'
        new_node.create = Mock(return_value=new_node.node)

        data = str(new_node)
        headers = {'size': str(len(data))}

        client = Client()
        client.get_node = Mock(return_value=old_node)
        client.get_node_url = Mock(return_value='http://foo.com/bar')
        client.conn = Mock()

        with patch('vos.Client', client) as mock:
            mock.add_props(new_node)
            mock.conn.session.post.assert_called_with('http://foo.com/bar',
                                                      headers=headers, data=data)
Ejemplo n.º 18
0
    def test_update(self):
        node = Node(ElementTree.fromstring(NODE_XML))

        resp = Mock()
        resp.headers.get = Mock(
            return_value="https://www.canfar.phys.uvic.ca/vospace")

        conn = Mock(spec=vos.Connection)
        conn.session.post = Mock(return_value=resp)
        client = Client(conn=conn)
        client.get_node_url = Mock(
            return_value='https://www.canfar.phys.uvic.ca/vospace')
        client.get_transfer_error = Mock()
        client.protocol = 'https'

        data = str(node)
        property_url = 'https://www.canfar.phys.uvic.ca/vospace/nodeprops'
        endpoints_mock = Mock()
        endpoints_mock.properties = property_url
        client.get_endpoints = Mock(return_value=endpoints_mock)
        result = client.update(node, False)
        self.assertEqual(result, 0)
        client.conn.session.post.assert_called_with(
            'https://www.canfar.phys.uvic.ca/vospace',
            data=data,
            allow_redirects=False)

        call1 = call(property_url,
                     allow_redirects=False,
                     data=data,
                     headers={'Content-type': 'text/xml'})
        call2 = call(
            'https://www.canfar.phys.uvic.ca/vospace/phase',
            allow_redirects=False,
            data="PHASE=RUN",
            headers={'Content-type': 'application/x-www-form-urlencoded'})
        calls = [call1, call2]

        client.conn = Mock(spec=vos.Connection)
        client.conn.session.post = Mock(return_value=resp)
        result = client.update(node, True)
        self.assertEqual(result, 0)
        client.conn.session.post.assert_has_calls(calls)
Ejemplo n.º 19
0
def test_get_node_url():
    client = Client()
    with pytest.raises(TypeError):
        client.get_node_url('vos://cadc.nrc.ca!vospace/auser', sort='Blah')
    with pytest.raises(ValueError):
        client.get_node_url('vos://cadc.nrc.ca!vospace/auser', order='Blah')

    response = Mock(spec=requests.Response)
    response.status_code = 303
    resource_id = 'ivo://cadc.nrc.ca/vospace'
    session_mock = Mock(spec=requests.Session, get=Mock(return_value=response))
    session_mock.headers = Mock()
    client._endpoints[resource_id] = vos.EndPoints(resource_id_uri=resource_id)
    client._endpoints[resource_id].conn.ws_client._session = session_mock
    equery = urlparse(client.get_node_url('vos://cadc.nrc.ca!vospace/auser',
                      sort=vos.SortNodeProperty.DATE)).query
    assert(urllib.parse.unquote(equery) ==
           'sort={}'.format(vos.SortNodeProperty.DATE.value))

    equery = urlparse(client.get_node_url('vos://cadc.nrc.ca!vospace/auser',
                      sort=vos.SortNodeProperty.LENGTH, order='asc')).query
    args = urllib.parse.unquote(equery).split('&')
    assert(2 == len(args))
    assert('order=asc' in args)
    assert('sort={}'.format(vos.SortNodeProperty.LENGTH.value) in args)

    equery = urlparse(client.get_node_url('vos://cadc.nrc.ca!vospace/auser',
                                          order='desc')).query
    assert('order=desc' == urllib.parse.unquote(equery))

    # test header view
    transfer_url = 'https://some.location/some/headers'
    response.headers = {'Location': transfer_url}
    client._endpoints[resource_id].conn.ws_client._session.get = \
        Mock(return_value=response)
    assert transfer_url == \
        client.get_node_url('vos://cadc.nrc.ca!vospace/auser',
                            view='header')[0]
    # get the argument lists for client.conn.session.get
    args, kwargs = client._endpoints[
        resource_id].conn.ws_client._session.get.call_args_list[0]
    # check head is amongst the other parameters
    assert kwargs['params']['view'] == 'header'
Ejemplo n.º 20
0
def _run_remote():
    """
    Uses a todo file to identify the work to be done.

    :return 0 if successful, -1 if there's any sort of failure. Return status
        is used by airflow for task instance management and reporting.
    """
    config = mc.Config()
    config.get_executors()
    name_builder = nbc.FileNameBuilder(GemProcName)
    vos_client = Client(vospace_certfile=config.proxy_fqn)
    store_transfer = tc.VoFitsTransfer(vos_client)
    data_source = dsc.VaultListDirDataSource(vos_client, config)
    return rc.run_by_todo(config=config,
                          name_builder=name_builder,
                          command_name=APPLICATION,
                          source=data_source,
                          meta_visitors=META_VISITORS,
                          data_visitors=DATA_VISITORS,
                          store_transfer=store_transfer)
Ejemplo n.º 21
0
    def test_add_props(self):
        old_node = Node(ElementTree.fromstring(NODE_XML))
        old_node.uri = 'vos:sometest'
        new_node = Node(ElementTree.fromstring(NODE_XML))
        new_node.props['quota'] = '1000'
        new_node.create = Mock(return_value=new_node.node)

        data = str(new_node)
        headers = {'size': str(len(data))}

        client = Client()
        client.get_node = Mock(return_value=old_node)
        client.get_node_url = Mock(return_value='http://foo.com/bar')
        mock_session = Mock()
        client.get_session = Mock(return_value=mock_session)

        client.add_props(new_node)
        client.get_session.assert_called_with('vos://foo.com!vospace/bar')
        mock_session.post.assert_called_with('http://foo.com/bar',
                                             headers=headers,
                                             data=data)
Ejemplo n.º 22
0
def _run_vo():
    """
    Uses a VOS listdir to identify the work to be done.

    :return 0 if successful, -1 if there's any sort of failure. Return status
        is used by airflow for task instance management and reporting.
    """
    config, clients, name_builder, metadata_reader = _common()
    vos_client = Client(vospace_certfile=config.proxy_file_name)
    clients.vo_client = vos_client
    source = data_source.DAOVaultDataSource(
        config, clients.vo_client, clients.data_client
    )
    store_transferrer = transfer.VoFitsCleanupTransfer(vos_client, config)
    return rc.run_by_todo(
        name_builder=name_builder,
        meta_visitors=META_VISITORS,
        data_visitors=DATA_VISITORS,
        source=source,
        clients=clients,
        store_transfer=store_transferrer,
        metadata_reader=metadata_reader,
    )
Ejemplo n.º 23
0
def main(orbit=None, **kwargs):
    """
    This is the driver program.  Gets the images from VOSpace or the local filesystem.

    expected kwargs:
    pointing, index, chip, rate, angle, ra, dec, p_name, discovery, nstk, zpt, dbimages

    :param kwargs: these are the 'args' sent on the commandline or in an input file.
    :param orbit: the orbit of the object that should be in the frames reference by kwargs provided.
    :type orbit: BKOrbit
    :return: list of observations
    :rtype list(ObsRecod)
    """

    pointing = kwargs['pointing']
    chip = kwargs['chip']
    index = kwargs['index']
    rate = kwargs['rate']
    angle = kwargs['angle']
    ra = kwargs['ra']
    dec = kwargs['dec']
    p_name = kwargs['p_name']
    discovery = kwargs['discovery']
    nstk = kwargs['nstk']
    rejected = kwargs.get('rejected', False)
    zpt = kwargs.get('zpt', 26.9)
    dbimages = kwargs.get('dbimages', 'vos:NewHorizons/dbimages/')

    client = Client()

    int_rate = int(rate * 10)
    int_angle = int((angle % 360) * 10)
    images = []
    # Load the 3 images associated with this point/chip/rate/angle set.
    epoch = None
    for idx in range(nstk):
        expnum = f'{int(pointing)}{int_rate:02d}{int_angle:04d}{idx}'
        image = f'{expnum}p{chip:02d}.fits'
        url = f'{dbimages}/{pointing:05d}/{chip:03d}/{index:04d}/{image}'
        logging.info(f"Looking for image at {url}")
        try:
            if os.access(url, os.R_OK):
                image = url

            elif not os.access(image, os.R_OK):
                # get from VOSpace is not already on disk
                client.copy(url, image)
        except Exception as ex:
            logging.error(str(ex))
            # Return empty set on VOSpace copy error.
            return {}
        images.append(image)
    epoch = Time(fits.open(images[len(images) // 2])[0].header['DATE-AVG'],
                 scale='tai').utc

    regions = f'{dbimages}/{pointing:05d}.reg'
    try:
        if not os.access(regions, os.R_OK):
            regions = client.copy(regions, '.', disposition=True)
    except Exception as ex:
        logging.debug(f"{ex}")
        regions = None

    wcs_dict = {}
    if orbit is not None:
        epoch = orbit.epoch.mjd
    if epoch is None:
        epoch = Time("2020-05-22T00:00:00")

    epoch = Time(kwargs.get('epoch', epoch), format='mjd')

    load_images(images,
                ra,
                dec,
                wcs_dict,
                orbit,
                dra=rate * math.cos(math.radians(angle)),
                ddec=rate * math.sin(math.radians(angle)),
                regions=regions,
                rejected=rejected,
                basedate=epoch)

    obs = measure_image(p_name, images, wcs_dict, discovery=discovery, zpt=zpt)
    return obs
Ejemplo n.º 24
0
from vos import Client

c = Client()

#for expnum in c.listdir('vos:cfis/solar_system/dbimages', force=True):
for expnum in open('expnum_list.txt').readlines():
    expnum = expnum.strip()
    n = c.get_node('vos:cfis/solar_system/dbimages/{}'.format(expnum), force=True)
    print n.uri
    for prop in n.props:
        if 'stationary' in prop:
           n.props[prop] = None
    c.add_props(n)
    print("Done")
Ejemplo n.º 25
0
Archivo: slow.py Proyecto: ijiraq/MOP
#!python
from multiprocessing import Pool
from ossos import mop_file, storage
import os
from astropy.io import fits
import numpy
from vos import Client
import sys
from ossos import junk_keywords
from ossos import util
import argparse
import logging

client = Client()

RA = 'RA_J2000'
DEC = 'DE_J2000'
del_keyword_list = junk_keywords.bad_keywords_list
task = 'slow'
dependency = 'step1'


def run(expnum, ccd, version='p', prefix='', dry_run=False, force=False):

    message = 'success'

    if storage.get_status(task, prefix, expnum, version=version,
                          ccd=ccd) and not force:
        logging.info("{} completed successfully for {} {} {} {}".format(
            task, prefix, expnum, version, ccd))
        return
Ejemplo n.º 26
0
    def test_copy(self, computed_md5_mock):
        file_content = 'File content'.encode('utf-8')

        # the md5sum of the file being copied
        transfer_md5 = hashlib.md5()
        transfer_md5.update(file_content)
        md5sum = transfer_md5.hexdigest()
        # patch the compute_md5 function in vos to return the above value
        computed_md5_mock.return_value = md5sum

        # mock the props of the corresponding node
        props = MagicMock()
        props.get.return_value = md5sum
        # add props to the mocked node
        node = MagicMock(spec=Node)
        node.props = {'MD5': md5sum, 'length': 12}

        # mock one by one the chain of connection.session.response.headers
        session = MagicMock()
        response = MagicMock()
        headers = MagicMock()
        headers.get.return_value = md5sum
        response.headers = headers
        session.get.return_value = response
        response.iter_content.return_value = BytesIO(file_content)

        test_client = Client()
        test_client.get_session = Mock(return_value=session)
        # use the mocked connection instead of the real one
        get_node_url_mock = Mock(
            return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
        test_client.get_node_url = get_node_url_mock
        mock_update = Mock()
        test_client.update = mock_update

        # patch Client.get_node to return our mocked node
        get_node_mock = Mock(return_value=node)
        test_client.get_node = get_node_mock

        # time to test...
        vospaceLocation = 'vos://test/foo'
        osLocation = '/tmp/foo'
        if os.path.isfile(osLocation):
            os.remove(osLocation)
        # copy from vospace
        test_client.copy(vospaceLocation, osLocation)
        get_node_url_mock.assert_called_once_with(vospaceLocation,
                                                  method='GET',
                                                  cutout=None, view='data')
        assert not computed_md5_mock.called,\
            'MD5 should be computed on the fly'
        assert get_node_mock.called

        # repeat - local file and vospace file are now the same -> only
        # get_node is called to get the md5 of remote file
        get_node_url_mock.reset_mock()
        computed_md5_mock.reset_mock()
        get_node_mock.reset_mock()
        props.reset_mock()
        props.get.return_value = md5sum
        test_client.copy(vospaceLocation, osLocation)
        assert not get_node_url_mock.called
        computed_md5_mock.assert_called_once_with(osLocation)
        get_node_mock.assert_called_once_with(vospaceLocation, force=True)

        # change the content of local files to trigger a new copy
        get_node_url_mock.reset_mock()
        get_node_mock.reset_mock()

        computed_md5_mock.reset_mock()
        computed_md5_mock.return_value = 'd002233'
        response.iter_content.return_value = BytesIO(file_content)
        test_client.copy(vospaceLocation, osLocation)
        get_node_url_mock.assert_called_once_with(vospaceLocation,
                                                  method='GET',
                                                  cutout=None, view='data')
        computed_md5_mock.assert_called_with(osLocation)
        get_node_mock.assert_called_once_with(vospaceLocation, force=True)

        # change the content of local files to trigger a new copy
        get_node_url_mock.reset_mock()
        get_node_url_mock.return_value = \
            ['https://mysite.com/node/node123/cutout']
        computed_md5_mock.reset_mock()
        computed_md5_mock.return_value = 'd002233'
        # computed_md5_mock.side_effect = ['d002233', md5sum]
        get_node_mock.reset_mock()
        response.iter_content.return_value = BytesIO(file_content)
        session.get.return_value = response
        test_client.get_session = Mock(return_value=session)
        test_client.copy('{}{}'.format(vospaceLocation,
                                       '[1][10:60]'), osLocation)
        get_node_url_mock.assert_called_once_with(
            vospaceLocation, method='GET', cutout='[1][10:60]', view='cutout')

        # copy to vospace when md5 sums are the same -> only update occurs
        get_node_url_mock.reset_mock()
        computed_md5_mock.reset_mock()
        computed_md5_mock.side_effect = None
        computed_md5_mock.return_value = md5sum
        test_client.copy(osLocation, vospaceLocation)
        mock_update.assert_called_once()
        assert not get_node_url_mock.called

        # make md5 different on destination
        get_node_url_mock.reset_mock()
        get_node_url_mock.return_value =\
            ['http://cadc.ca/test', 'http://cadc.ca/test']
        computed_md5_mock.reset_mock()
        mock_update.reset_mock()
        computed_md5_mock.return_value = md5sum
        to_update_node = MagicMock()
        to_update_node.props = {'MD5': 'abcde', 'length': 12}
        test_client.get_node = Mock(side_effect=[to_update_node, node])
        test_client.copy(osLocation, vospaceLocation)
        assert not mock_update.called
        get_node_url_mock.assert_called_once_with(vospaceLocation, 'PUT',
                                                  content_length=12,
                                                  md5_checksum='abcde')
        computed_md5_mock.assert_called_once_with(osLocation)

        # copy 0 size file -> delete and create node but no bytes
        # transferred
        get_node_url_mock.reset_mock()
        computed_md5_mock.reset_mock()
        test_client.get_node = Mock(return_value=node)
        node.props['length'] = 0
        mock_delete = Mock()
        mock_create = Mock()
        test_client.delete = mock_delete
        test_client.create = mock_create
        with patch('vos.vos.os.stat', Mock()) as stat_mock:
            stat_mock.return_value = Mock(st_size=0)
            test_client.copy(osLocation, vospaceLocation)
        mock_create.assert_called_once_with(vospaceLocation)
        mock_delete.assert_called_once_with(vospaceLocation)
        assert not get_node_url_mock.called

        # copy new 0 size file -> create node but no bytes transferred
        get_node_url_mock.reset_mock()
        computed_md5_mock.reset_mock()
        mock_delete.reset_mock()
        mock_create.reset_mock()
        test_client.get_node = Mock(side_effect=[Exception(), node])
        mock_delete = Mock()
        mock_create = Mock()
        test_client.delete = mock_delete
        test_client.create = mock_create
        with patch('vos.vos.os.stat', Mock()) as stat_mock:
            stat_mock.return_value = Mock(st_size=0)
            test_client.copy(osLocation, vospaceLocation)
        mock_create.assert_called_once_with(vospaceLocation)
        assert not mock_delete.called
        assert not get_node_url_mock.called

        # error tests - md5sum mismatch
        node.props['length'] = 12
        computed_md5_mock.return_value = '000bad000'
        test_client.get_node = Mock(return_value=node)
        with self.assertRaises(OSError):
            test_client.copy(vospaceLocation, osLocation)

        # existing file
        mock_delete.reset_mock()
        with self.assertRaises(OSError):
            with patch('vos.vos.os.stat', Mock()) as stat_mock:
                stat_mock.return_value = Mock(st_size=12)
                test_client.copy(osLocation, vospaceLocation)
        assert not mock_delete.called  # server takes care of cleanup

        # new file
        mock_delete.reset_mock()
        with self.assertRaises(OSError):
            with patch('vos.vos.os.stat', Mock()) as stat_mock:
                stat_mock.return_value = Mock(st_size=12)
                node.props['MD5'] = None
                test_client.copy(osLocation, vospaceLocation)
        assert mock_delete.called  # cleanup required

        # requests just the headers when md5 not provided in the header
        props.get.side_effect = [None]
        get_node_url_mock = Mock(
            return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
        test_client.get_node_url = get_node_url_mock
        get_node_mock.reset_mock()
        headers.get.return_value = None
        test_client.copy(vospaceLocation, osLocation, head=True)
        get_node_url_mock.assert_called_once_with(vospaceLocation,
                                                  method='GET',
                                                  cutout=None, view='header')

        # repeat headers request when md5 provided in the header
        props.get.side_effect = md5sum
        get_node_url_mock = Mock(
            return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
        test_client.get_node_url = get_node_url_mock
        get_node_mock.reset_mock()
        response.iter_content.return_value = BytesIO(file_content)
        headers.get.return_value = None
        test_client.copy(vospaceLocation, osLocation, head=True)
        get_node_url_mock.assert_called_once_with(vospaceLocation,
                                                  method='GET',
                                                  cutout=None, view='header')
Ejemplo n.º 27
0
    def test_copy(self, computed_md5_mock):
        # the md5sum of the file being copied
        md5sum = 'd41d8cd98f00b204e9800998ecf84eee'
        # patch the compute_md5 function in vos to return the above value
        computed_md5_mock.return_value = md5sum

        # mock the props of the corresponding node
        props = MagicMock()
        props.get.return_value = md5sum
        # add props to the mocked node
        node = MagicMock(spec=Node)
        node.props = props

        # mock one by one the chain of connection.session.response.headers
        conn = MagicMock(spec=Connection)
        session = MagicMock()
        response = MagicMock()
        headers = MagicMock()
        headers.get.return_value = md5sum
        response.headers = headers
        session.get.return_value = response
        conn.session = session

        test_client = Client()
        # use the mocked connection instead of the real one
        test_client.conn = conn
        get_node_url_mock = Mock(
            return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
        test_client.get_node_url = get_node_url_mock
        mock_update = Mock()
        test_client.update = mock_update

        # patch Client.get_node to return our mocked node
        get_node_mock = Mock(return_value=node)
        test_client.get_node = get_node_mock

        # time to test...
        vospaceLocation = 'vos://test/foo'
        osLocation = '/tmp/foo'
        if os.path.isfile(osLocation):
            os.remove(osLocation)
        # copy from vospace
        test_client.copy(vospaceLocation, osLocation)
        get_node_url_mock.assert_called_once_with(vospaceLocation,
                                                  method='GET',
                                                  cutout=None,
                                                  view='data')
        computed_md5_mock.assert_called_once_with(osLocation)
        assert get_node_mock.called

        # repeat - local file and vospace file are now the same -> only
        # get_node is called to get the md5 of remote file
        get_node_url_mock.reset_mock()
        computed_md5_mock.reset_mock()
        get_node_mock.reset_mock()
        test_client.copy(vospaceLocation, osLocation)
        assert not get_node_url_mock.called
        computed_md5_mock.assert_called_once_with(osLocation)
        get_node_mock.assert_called_once_with(vospaceLocation)

        # change the content of local files to trigger a new copy
        get_node_url_mock.reset_mock()
        computed_md5_mock.reset_mock()
        computed_md5_mock.side_effect = ['d002233', md5sum]
        get_node_mock.reset_mock()
        test_client.copy(vospaceLocation, osLocation)
        get_node_url_mock.assert_called_once_with(vospaceLocation,
                                                  method='GET',
                                                  cutout=None,
                                                  view='data')
        computed_md5_mock.assert_called_with(osLocation)
        get_node_mock.assert_called_once_with(vospaceLocation)

        # copy to vospace when md5 sums are the same -> only update occurs
        get_node_url_mock.reset_mock()
        computed_md5_mock.reset_mock()
        computed_md5_mock.side_effect = None
        computed_md5_mock.return_value = md5sum
        test_client.copy(osLocation, vospaceLocation)
        mock_update.assert_called_once()
        assert not get_node_url_mock.called

        # make md5 different
        get_node_url_mock.reset_mock()
        get_node_url_mock.return_value =\
            ['http://cadc.ca/test', 'http://cadc.ca/test']
        computed_md5_mock.reset_mock()
        mock_update.reset_mock()
        props.get.side_effect = ['d00223344', md5sum]
        test_client.copy(osLocation, vospaceLocation)
        assert not mock_update.called
        get_node_url_mock.assert_called_once_with(vospaceLocation, 'PUT')
        computed_md5_mock.assert_called_once_with(osLocation)

        # copy 0 size file -> delete and create on client but no bytes
        # transferred
        get_node_url_mock.reset_mock()
        computed_md5_mock.reset_mock()
        computed_md5_mock.return_value = vos.ZERO_MD5
        props.get.side_effect = [md5sum]
        mock_delete = Mock()
        mock_create = Mock()
        test_client.delete = mock_delete
        test_client.create = mock_create
        test_client.copy(osLocation, vospaceLocation)
        mock_create.assert_called_once_with(vospaceLocation)
        mock_delete.assert_called_once_with(vospaceLocation)
        assert not get_node_url_mock.called

        # copy new 0 size file -> reate on client but no bytes transferred
        get_node_url_mock.reset_mock()
        computed_md5_mock.reset_mock()
        mock_delete.reset_mock()
        mock_create.reset_mock()
        computed_md5_mock.return_value = vos.ZERO_MD5
        props.get.side_effect = [None]
        mock_delete = Mock()
        mock_create = Mock()
        test_client.delete = mock_delete
        test_client.create = mock_create
        test_client.copy(osLocation, vospaceLocation)
        mock_create.assert_called_once_with(vospaceLocation)
        assert not mock_delete.called
        assert not get_node_url_mock.called

        # error tests - md5sum mismatch
        props.get.side_effect = [md5sum]
        computed_md5_mock.return_value = '000bad000'
        with self.assertRaises(OSError):
            test_client.copy(vospaceLocation, osLocation)

        with self.assertRaises(OSError):
            test_client.copy(osLocation, vospaceLocation)

        # requests just the headers
        props.get.side_effect = [None]
        get_node_url_mock = Mock(
            return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
        test_client.get_node_url = get_node_url_mock
        computed_md5_mock.reset_mock()
        computed_md5_mock.side_effect = ['d002233', md5sum]
        get_node_mock.reset_mock()
        test_client.copy(vospaceLocation, osLocation, head=True)
        get_node_url_mock.assert_called_once_with(vospaceLocation,
                                                  method='GET',
                                                  cutout=None,
                                                  view='header')
Ejemplo n.º 28
0
    def test_glob(self):
        # test the pattern matches in directories and file names

        # simple test for listing of directory, no wild cards

        # NOTE: Mock class also has a 'name' attribute so we cannot
        # instantiate a mock node with Mock(name='blah'). There are
        # two other wasy to do it as seen below
        mock_node = MagicMock(type='vos:ContainerNode')
        mock_node.configure_mock(name='anode')
        client = Client()
        client.get_node = Mock(return_value=mock_node)
        self.assertEquals(['vos:/anode/'], client.glob('vos:/anode/'))

        # simple test for file listing of file
        mock_node = MagicMock(type='vos:DataNode')
        mock_node.configure_mock(name='afile')
        client = Client()
        client.get_node = Mock(return_value=mock_node)
        self.assertEquals(['vos:/afile'], client.glob('vos:/afile'))

        # create a mock directory structure on the form
        # /anode/abc /anode/def - > anode/a* should return
        # /anode/adc

        mock_node = MagicMock(type='vos:ContainerNode')
        mock_node.configure_mock(name='anode')
        mock_child_node1 = Mock(type='vos:DataNode')
        mock_child_node1.name = 'abc'
        mock_child_node2 = Mock(type='vos:DataNode')
        mock_child_node2.name = 'def'
        # because we use wild characters in the root node,
        # we need to create a corresponding node for the base node
        mock_base_node = Mock(type='vos:ContainerNode')
        mock_base_node.name = 'vos:'
        mock_base_node.node_list = [mock_node]
        mock_node.node_list = [
            mock_base_node, mock_child_node1, mock_child_node2
        ]
        client = Client()
        client.get_node = Mock(
            side_effect=[mock_node, mock_base_node, mock_node])
        self.assertEquals(['vos:/anode/abc'], client.glob('vos:/anode/a*'))
        self.assertEquals(['vos:/anode/abc'], client.glob('vos:/*node/abc'))

        # test nodes:
        # /anode/.test1 /bnode/sometests /bnode/blah
        # /[a,c]node/*test* should return /bnode/somtests (.test1 is filtered
        # out as a special file)

        mock_child_node1 = Mock(type='vos:DataNode')
        mock_child_node1.name = '.test1'
        mock_node1 = MagicMock(type='vos:ContainerNode')
        mock_node1.configure_mock(name='anode')
        mock_node1.node_list = [mock_child_node1]

        mock_child_node2 = Mock(type='vos:DataNode')
        mock_child_node2.name = 'sometests'
        mock_child_node3 = Mock(type='vos:DataNode')
        mock_child_node3.name = 'blah'
        mock_node2 = MagicMock(type='vos:ContainerNode')
        mock_node2.configure_mock(name='bnode')
        mock_node2.node_list = [mock_child_node2, mock_child_node3]

        # because we use wild characters in the root node,
        # we need to create a corresponding node for the base node
        mock_base_node = Mock(type='vos:DataNode')
        mock_base_node.name = 'vos:'
        mock_base_node.node_list = [mock_node1, mock_node2]
        client = Client()
        client.get_node = Mock(
            side_effect=[mock_base_node, mock_node1, mock_node2])
        self.assertEquals(['vos:/bnode/sometests'],
                          client.glob('vos:/[a,b]node/*test*'))