Esempio n. 1
0
    def test_no_ce(self):
        data = np.rec.fromrecords(
                D1.Drifters.data.tolist(), names=list(D1.Drifters.keys()))

        projection, selection = parse_ce('')
        dataset = BaseHandler(D1).parse(projection, selection)
        np.testing.assert_array_equal(data, dataset.Drifters.data)
Esempio n. 2
0
    def __init__(self, url):
        # download DDS/DAS
        scheme, netloc, path, query, fragment = urlsplit(url)
        ddsurl = urlunsplit((scheme, netloc, path + '.dds', query, fragment))
        dds = requests.get(ddsurl).text.encode('utf-8')
        dasurl = urlunsplit((scheme, netloc, path + '.das', query, fragment))
        das = requests.get(dasurl).text.encode('utf-8')

        # build the dataset from the DDS and add attributes from the DAS
        self.dataset = build_dataset(dds)
        add_attributes(self.dataset, parse_das(das))

        # remove any projection from the url, leaving selections
        projection, selection = parse_ce(query)
        url = urlunsplit((scheme, netloc, path, '&'.join(selection), fragment))

        # now add data proxies
        for var in walk(self.dataset, BaseType):
            var.data = BaseProxy(url, var.id, var.descr)
        for var in walk(self.dataset, SequenceType):
            var.data = SequenceProxy(url, var.id, var.descr)

        # apply projections
        for var in projection:
            target = self.dataset
            while var:
                token, index = var.pop(0)
                target = target[token]
                if index and isinstance(target.data, BaseProxy):
                    target.data.slice = fix_slice(index, target.shape)
Esempio n. 3
0
    def test_no_ce(self):
        data = np.rec.fromrecords(D1.Drifters.data.tolist(),
                                  names=D1.Drifters.keys())

        projection, selection = parse_ce('')
        dataset = BaseHandler(D1).parse(projection, selection)
        np.testing.assert_array_equal(data, dataset.Drifters.data)
Esempio n. 4
0
 def test_complex(self):
     """Test a more complex constraint expression."""
     projection, selection = parse_ce("a,b[0:2:9],c&a>1&b<2")
     self.assertEqual(
         projection,
         [[("a", ())], [("b", (slice(0, 10, 2),))], [("c", ())]])
     self.assertEqual(selection, ["a>1", "b<2"])
Esempio n. 5
0
    def __call__(self, environ, start_response):
        req = Request(environ)
        path, response = req.path.rsplit('.', 1)
        if response == 'das':
            req.query_string = ''
        projection, selection = parse_ce(req.query_string)
        buffer_size = environ.get('pydap.buffer_size', BUFFER_SIZE)

        try:
            # build the dataset and pass it to the proper response, returning a
            # WSGI app
            dataset = self.parse(projection, selection, buffer_size)
            app = self.responses[response](dataset)
            app.close = self.close

            # now build a Response and set additional headers
            res = req.get_response(app)
            for key, value in self.additional_headers:
                res.headers.add(key, value)

            # CORS for Javascript requests
            if response in CORS_RESPONSES:
                res.headers.add('Access-Control-Allow-Origin', '*')
                res.headers.add(
                    'Access-Control-Allow-Headers',
                    'Origin, X-Requested-With, Content-Type')

            return res(environ, start_response)
        except:
            # should the exception be catched?
            if environ.get('x-wsgiorg.throw_errors'):
                raise
            else:
                res = ErrorResponse(info=sys.exc_info())
                return res(environ, start_response)
Esempio n. 6
0
    def __init__(self, url):
        # download DDS/DAS
        scheme, netloc, path, query, fragment = urlsplit(url)
        ddsurl = urlunsplit((scheme, netloc, path + '.dds', query, fragment))
        r = requests.get(ddsurl)
        r.raise_for_status()
        dds = r.text.encode('utf-8')
        dasurl = urlunsplit((scheme, netloc, path + '.das', query, fragment))
        r = requests.get(dasurl)
        r.raise_for_status()
        das = r.text.encode('utf-8')

        # build the dataset from the DDS and add attributes from the DAS
        self.dataset = build_dataset(dds)
        add_attributes(self.dataset, parse_das(das))

        # remove any projection from the url, leaving selections
        projection, selection = parse_ce(query)
        url = urlunsplit((scheme, netloc, path, '&'.join(selection), fragment))

        # now add data proxies
        for var in walk(self.dataset, BaseType):
            var.data = BaseProxy(url, var.id, var.descr)
        for var in walk(self.dataset, SequenceType):
            var.data = SequenceProxy(url, var.id, var.descr)

        # apply projections
        for var in projection:
            target = self.dataset
            while var:
                token, index = var.pop(0)
                target = target[token]
                if index and isinstance(target.data, BaseProxy):
                    target.data.slice = fix_slice(index, target.shape)
Esempio n. 7
0
 def test_complex(self):
     """Test a more complex constraint expression."""
     projection, selection = parse_ce("a,b[0:2:9],c&a>1&b<2")
     self.assertEqual(
         projection,
         [[("a", ())], [("b", (slice(0, 10, 2),))], [("c", ())]])
     self.assertEqual(selection, ["a>1", "b<2"])
Esempio n. 8
0
    def __call__(self, environ, start_response):
        req = Request(environ)
        path, response = req.path.rsplit('.', 1)
        if response == 'das':
            req.query_string = ''
        projection, selection = parse_ce(req.query_string)
        buffer_size = environ.get('pydap.buffer_size', BUFFER_SIZE)

        try:
            # build the dataset and pass it to the proper response, returning a
            # WSGI app
            dataset = self.parse(projection, selection, buffer_size)
            app = self.responses[response](dataset)
            app.close = self.close

            # now build a Response and set additional headers
            res = req.get_response(app)
            for key, value in self.additional_headers:
                res.headers.add(key, value)

            # CORS for Javascript requests
            if response in CORS_RESPONSES:
                res.headers.add('Access-Control-Allow-Origin', '*')
                res.headers.add('Access-Control-Allow-Headers',
                                'Origin, X-Requested-With, Content-Type')

            return res(environ, start_response)
        except:
            # should the exception be catched?
            if environ.get('x-wsgiorg.throw_errors'):
                raise
            else:
                res = ErrorResponse(info=sys.exc_info())
                return res(environ, start_response)
Esempio n. 9
0
    def test_filtering(self):
        data = np.rec.fromrecords(DATA, names=self.dataset.Drifters.keys())
        filtered = data[ data['longitude'] < 999 ]

        projection, selection = parse_ce('Drifters.longitude<999')
        dataset = BaseHandler(self.dataset).parse(projection, selection)
        np.testing.assert_array_equal(filtered, dataset.Drifters.data)
Esempio n. 10
0
    def test_filtering(self):
        data = np.rec.fromrecords(D1.Drifters.data.tolist(),
                                  names=D1.Drifters.keys())
        filtered = data[data['longitude'] < 999]

        projection, selection = parse_ce('Drifters.longitude<999')
        dataset = BaseHandler(D1).parse(projection, selection)
        np.testing.assert_array_equal(filtered, dataset.Drifters.data)
Esempio n. 11
0
 def test_function(self):
     """Test a constraint expression with a function call."""
     projection, selection = parse_ce(
         "time&bounds(0,360,-90,90,0,500,00Z01JAN1970,00Z04JAN1970)")
     self.assertEqual(projection, [[("time", ())]])
     self.assertEqual(
         selection,
         ["bounds(0,360,-90,90,0,500,00Z01JAN1970,00Z04JAN1970)"])
Esempio n. 12
0
 def test_function(self):
     """Test a constraint expression with a function call."""
     projection, selection = parse_ce(
         "time&bounds(0,360,-90,90,0,500,00Z01JAN1970,00Z04JAN1970)")
     self.assertEqual(projection, [[("time", ())]])
     self.assertEqual(
         selection,
         ["bounds(0,360,-90,90,0,500,00Z01JAN1970,00Z04JAN1970)"])
Esempio n. 13
0
    def __init__(self, url, application=None, session=None, output_grid=True):
        # download DDS/DAS
        scheme, netloc, path, query, fragment = urlsplit(url)

        ddsurl = urlunsplit((scheme, netloc, path + '.dds', query, fragment))
        r = GET(ddsurl, application, session)
        raise_for_status(r)
        dds = r.text

        dasurl = urlunsplit((scheme, netloc, path + '.das', query, fragment))
        r = GET(dasurl, application, session)
        raise_for_status(r)
        das = r.text

        # build the dataset from the DDS and add attributes from the DAS
        self.dataset = build_dataset(dds)
        add_attributes(self.dataset, parse_das(das))

        # remove any projection from the url, leaving selections
        projection, selection = parse_ce(query)
        url = urlunsplit((scheme, netloc, path, '&'.join(selection), fragment))

        # now add data proxies
        for var in walk(self.dataset, BaseType):
            var.data = BaseProxy(url,
                                 var.id,
                                 var.dtype,
                                 var.shape,
                                 application=application,
                                 session=session)
        for var in walk(self.dataset, SequenceType):
            template = copy.copy(var)
            var.data = SequenceProxy(url,
                                     template,
                                     application=application,
                                     session=session)

        # apply projections
        for var in projection:
            target = self.dataset
            while var:
                token, index = var.pop(0)
                target = target[token]
                if isinstance(target, BaseType):
                    target.data.slice = fix_slice(index, target.shape)
                elif isinstance(target, GridType):
                    index = fix_slice(index, target.array.shape)
                    target.array.data.slice = index
                    for s, child in zip(index, target.maps):
                        target[child].data.slice = (s, )
                elif isinstance(target, SequenceType):
                    target.data.slice = index

        # retrieve only main variable for grid types:
        for var in walk(self.dataset, GridType):
            var.set_output_grid(output_grid)
Esempio n. 14
0
File: dap.py Progetto: pydap/pydap
    def __init__(self, url, application=None, session=None, output_grid=True):
        # download DDS/DAS
        scheme, netloc, path, query, fragment = urlsplit(url)

        ddsurl = urlunsplit((scheme, netloc, path + ".dds", query, fragment))
        r = GET(ddsurl, application, session)
        raise_for_status(r)
        dds = r.text

        dasurl = urlunsplit((scheme, netloc, path + ".das", query, fragment))
        r = GET(dasurl, application, session)
        raise_for_status(r)
        das = r.text

        # build the dataset from the DDS and add attributes from the DAS
        self.dataset = build_dataset(dds)
        add_attributes(self.dataset, parse_das(das))

        # remove any projection from the url, leaving selections
        projection, selection = parse_ce(query)
        url = urlunsplit((scheme, netloc, path, "&".join(selection), fragment))

        # now add data proxies
        for var in walk(self.dataset, BaseType):
            var.data = BaseProxy(url, var.id, var.dtype, var.shape, application=application, session=session)
        for var in walk(self.dataset, SequenceType):
            template = copy.copy(var)
            var.data = SequenceProxy(url, template, application=application, session=session)

        # apply projections
        for var in projection:
            target = self.dataset
            while var:
                token, index = var.pop(0)
                target = target[token]
                if isinstance(target, BaseType):
                    target.data.slice = fix_slice(index, target.shape)
                elif isinstance(target, GridType):
                    index = fix_slice(index, target.array.shape)
                    target.array.data.slice = index
                    for s, child in zip(index, target.maps):
                        target[child].data.slice = (s,)
                elif isinstance(target, SequenceType):
                    target.data.slice = index

        # retrieve only main variable for grid types:
        for var in walk(self.dataset, GridType):
            var.set_output_grid(output_grid)
Esempio n. 15
0
    def __call__(self, environ, start_response):
        req = Request(environ)
        path, response = req.path.rsplit('.', 1)
        if response == 'das':
            req.query_string = ''
        projection, selection = parse_ce(req.query_string)

        try:
            # build the dataset and pass it to the proper response, returning a
            # WSGI app
            dataset = self.parse(projection, selection)
            app = self.responses[response](dataset)
            app.close = self.close

            # now build a Response and set additional headers
            res = req.get_response(app)
            for key, value in self.additional_headers:
                res.headers.add(key, value)

            return res(environ, start_response)
        except HTTPException, exc:
            # HTTP exceptions are used to redirect the user
            return exc(environ, start_response)
Esempio n. 16
0
    def __call__(self, environ, start_response):
        req = Request(environ)
        path, response = req.path.rsplit('.', 1)
        if response == 'das':
            req.query_string = ''
        projection, selection = parse_ce(req.query_string)

        try:
            # build the dataset and pass it to the proper response, returning a 
            # WSGI app
            dataset = self.parse(projection, selection)
            app = self.responses[response](dataset)
            app.close = self.close

            # now build a Response and set additional headers
            res = req.get_response(app)
            for key, value in self.additional_headers:
                res.headers.add(key, value)

            return res(environ, start_response)
        except HTTPException, exc:
            # HTTP exceptions are used to redirect the user
            return exc(environ, start_response)
Esempio n. 17
0
    def __call__(self, environ, start_response):
        # specify that we want the parsed dataset
        environ['x-wsgiorg.want_parsed_response'] = True
        req = Request(environ)
        projection, selection = parse_ce(req.query_string)

        # check if there are any functions calls in the request
        called = (any(s for s in selection if FUNCTION.match(s))
                  or any(p for p in projection if isinstance(p, string_types)))

        # ignore DAS requests and requests without functions
        path, response = req.path.rsplit('.', 1)
        if response == 'das' or not called:
            return self.app(environ, start_response)

        # apply selection without any function calls
        req.query_string = '&'.join(s for s in selection
                                    if not FUNCTION.match(s))
        res = req.get_response(self.app)

        # get the dataset
        method = getattr(res.app_iter, 'x_wsgiorg_parsed_response', False)
        if not method:
            raise ServerError("Unable to call server-side function!")
        dataset = method(DatasetType)

        # apply selection containing server-side functions
        selection = (s for s in selection if FUNCTION.match(s))
        for expr in selection:
            if RELOP.search(expr):
                call, op, other = RELOP.split(expr)
                op = {
                    '<': operator.lt,
                    '>': operator.gt,
                    '!=': operator.ne,
                    '=': operator.eq,
                    '>=': operator.ge,
                    '<=': operator.le,
                    '=~': lambda a, b: re.match(b, a),
                }[op]
                other = ast.literal_eval(other)
            else:
                call, op, other = expr, operator.eq, 1

            # evaluate the function call
            sequence = eval_function(dataset, call, self.functions)

            # is this an inplace call?
            for var in walk(dataset, SequenceType):
                if sequence is var:
                    break
            else:
                # get the data from the resulting variable, and use it to
                # constrain the original dataset
                child = list(sequence.children())[0]
                data = np.fromiter(child.data, child.dtype)
                if data.dtype.char == "S":
                    valid = np.array(
                        list(map(lambda v: op(str(v), str(other)), data)),
                        bool)
                else:
                    valid = op(data, other)

                for sequence in walk(dataset, SequenceType):
                    sequence.data = np.rec.fromrecords(
                        [tuple(row) for row in sequence],
                        names=sequence.keys())[valid]

        # now apply projection
        if projection:
            projection = fix_shorthand(projection, dataset)
            base = [p for p in projection if not isinstance(p, string_types)]
            func = [p for p in projection if isinstance(p, string_types)]

            # apply non-function projection
            out = apply_projection(base, dataset)

            # apply function projection
            for call in func:
                var = eval_function(dataset, call, self.functions)
                for child in walk(var):
                    parent = reduce(operator.getitem,
                                    [out] + child.id.split('.')[:-1])
                    if child.name not in parent.keys():
                        parent[child.name] = child
                        break
            dataset = out

        # Return the original response (DDS, DAS, etc.)
        path, response = req.path.rsplit('.', 1)
        res = BaseHandler.responses[response](dataset)

        return res(environ, start_response)
Esempio n. 18
0
    def __call__(self, environ, start_response):
        # specify that we want the parsed dataset
        environ['x-wsgiorg.want_parsed_response'] = True
        req = Request(environ)
        original_query = req.query_string
        projection, selection = parse_ce(req.query_string)

        # apply selection without any function calls
        req.query_string = '&'.join(expr for expr in selection if not FUNCTION.match(expr))
        res = req.get_response(self.app)

        # ignore DAS requests
        path, response = req.path.rsplit('.', 1)
        if response == 'das':
            return self.app(environ, start_response)

        # get the dataset
        method = getattr(res.app_iter, 'x_wsgiorg_parsed_response', False)
        if not method:
            environ['QUERY_STRING'] = original_query
            return self.app(environ, start_response)
        dataset = method(DatasetType)

        # apply selection containing server-side functions
        selection = (expr for expr in selection if FUNCTION.match(expr))
        for expr in selection:
            if RELOP.search(expr):
                call, op, other = RELOP.split(expr)
                op = {
                        '<' : operator.lt,
                        '>' : operator.gt,
                        '!=': operator.ne,
                        '=' : operator.eq,
                        '>=': operator.ge,
                        '<=': operator.le,
                        '=~': lambda a, b: re.match(b, a),
                }[op]
                other = ast.literal_eval(other)
            else:
                call, op, other = expr, operator.eq, 1

            # evaluate the function call
            sequence = eval_function(dataset, call, self.functions)

            # is this an inplace call?
            for var in walk(dataset, SequenceType):
                if sequence is var:
                    break
            else:
                # get the data from the resulting variable, and use it to 
                # constrain the original dataset
                data = np.fromiter(sequence)
                print data.shape
                valid = op(data, other)

                for sequence in walk(dataset, SequenceType):
                    data = np.asarray(list(sequence), 'O')[valid]
                    sequence.data = np.asarray(list(sequence), 'O')[valid]
                dataset = out

        # now apply projection
        if projection:
            projection = fix_shorthand(projection, dataset)
            base = [p for p in projection if not isinstance(p, basestring)] 
            func = [p for p in projection if isinstance(p, basestring)] 

            # apply non-function projection
            out = apply_projection(base, dataset)

            # apply function projection
            for call in func:
                var = eval_function(dataset, call, self.functions)
                for child in walk(var):
                    parent = reduce(operator.getitem, [out] + child.id.split('.')[:-1])
                    if child.name not in parent.keys():
                        parent[child.name] = child
                        break
            dataset = out

        # Return the original response (DDS, DAS, etc.)
        path, response = req.path.rsplit('.', 1)
        res = BaseHandler.responses[response](dataset)

        return res(environ, start_response)
Esempio n. 19
0
 def test_only_projection(self):
     """Test a constraint expression with only the projection."""
     projection, selection = parse_ce("a,b")
     self.assertEqual(projection, [[("a", ())], [("b", ())]])
     self.assertEqual(selection, [])
Esempio n. 20
0
 def test_only_selection(self):
     """Test a constraint expression with only the selection."""
     projection, selection = parse_ce("c>1")
     self.assertEqual(projection, [])
     self.assertEqual(selection, ["c>1"])
Esempio n. 21
0
 def test_empty(self):
     """Test no constraint expression."""
     projection, selection = parse_ce('')
     self.assertEqual(projection, [])
     self.assertEqual(selection, [])
Esempio n. 22
0
 def test_basic(self):
     """Test a basic constraint expression."""
     projection, selection = parse_ce("a,b&c>1")
     self.assertEqual(projection, [[("a", ())], [("b", ())]])
     self.assertEqual(selection, ["c>1"])
Esempio n. 23
0
    def __call__(self, environ, start_response):
        # specify that we want the parsed dataset
        environ['x-wsgiorg.want_parsed_response'] = True
        req = Request(environ)
        projection, selection = parse_ce(req.query_string)

        # check if there are any functions calls in the request
        called = (
            any(s for s in selection if FUNCTION.match(s)) or
            any(p for p in projection if isinstance(p, string_types)))

        # ignore DAS requests and requests without functions
        path, response = req.path.rsplit('.', 1)
        if response == 'das' or not called:
            return self.app(environ, start_response)

        # apply selection without any function calls
        req.query_string = '&'.join(
            s for s in selection if not FUNCTION.match(s))
        res = req.get_response(self.app)

        # get the dataset
        method = getattr(res.app_iter, 'x_wsgiorg_parsed_response', False)
        if not method:
            raise ServerError("Unable to call server-side function!")
        dataset = method(DatasetType)

        # apply selection containing server-side functions
        selection = (s for s in selection if FUNCTION.match(s))
        for expr in selection:
            if RELOP.search(expr):
                call, op, other = RELOP.split(expr)
                op = {
                    '<':  operator.lt,
                    '>':  operator.gt,
                    '!=': operator.ne,
                    '=':  operator.eq,
                    '>=': operator.ge,
                    '<=': operator.le,
                    '=~': lambda a, b: re.match(b, a),
                }[op]
                other = ast.literal_eval(other)
            else:
                call, op, other = expr, operator.eq, 1

            # evaluate the function call
            sequence = eval_function(dataset, call, self.functions)

            # is this an inplace call?
            for var in walk(dataset, SequenceType):
                if sequence is var:
                    break
            else:
                # get the data from the resulting variable, and use it to
                # constrain the original dataset
                child = list(sequence.children())[0]
                data = np.fromiter(child.data, child.dtype)
                if data.dtype.char == "S":
                    valid = np.array(
                        list(map(lambda v: op(str(v), str(other)), data)),
                        bool)
                else:
                    valid = op(data, other)

                for sequence in walk(dataset, SequenceType):
                    sequence.data = np.rec.fromrecords(
                        [tuple(row) for row in sequence],
                        names=sequence.keys())[valid]

        # now apply projection
        if projection:
            projection = fix_shorthand(projection, dataset)
            base = [p for p in projection if not isinstance(p, string_types)]
            func = [p for p in projection if isinstance(p, string_types)]

            # apply non-function projection
            out = apply_projection(base, dataset)

            # apply function projection
            for call in func:
                var = eval_function(dataset, call, self.functions)
                for child in walk(var):
                    parent = reduce(
                        operator.getitem, [out] + child.id.split('.')[:-1])
                    if child.name not in parent.keys():
                        parent[child.name] = child
                        break
            dataset = out

        # Return the original response (DDS, DAS, etc.)
        path, response = req.path.rsplit('.', 1)
        res = BaseHandler.responses[response](dataset)

        return res(environ, start_response)
Esempio n. 24
0
 def test_only_selection(self):
     """Test a constraint expression with only the selection."""
     projection, selection = parse_ce("c>1")
     self.assertEqual(projection, [])
     self.assertEqual(selection, ["c>1"])
Esempio n. 25
0
 def test_only_projection(self):
     """Test a constraint expression with only the projection."""
     projection, selection = parse_ce("a,b")
     self.assertEqual(projection, [[("a", ())], [("b", ())]])
     self.assertEqual(selection, [])
Esempio n. 26
0
 def test_basic(self):
     """Test a basic constraint expression."""
     projection, selection = parse_ce("a,b&c>1")
     self.assertEqual(projection, [[("a", ())], [("b", ())]])
     self.assertEqual(selection, ["c>1"])
Esempio n. 27
0
 def test_empty(self):
     """Test no constraint expression."""
     projection, selection = parse_ce("")
     self.assertEqual(projection, [])
     self.assertEqual(selection, [])