def __getitem__(self, index): slice_ = combine_slices(self._slice, fix_slice(index, self.shape)) scheme, netloc, path, query, fragment = urlsplit(self.url) url = urlunsplit((scheme, netloc, path + '.dods', self.id + hyperslab(slice_) + '&' + query, fragment)) resp, data = request(url) dds, xdrdata = data.split('\nData:\n', 1) dataset = DDSParser(dds).parse() data = data2 = DapUnpacker(xdrdata, dataset).getvalue() # Retrieve the data from any parent structure(s). for var in walk(dataset): if type(var) in (StructureType, DatasetType): data = data[0] elif var.id == self.id: return data # Some old servers return the wrong response. :-/ # I found a server that would return an array to a request # for an array inside a grid (instead of a structure with # the array); this will take care of it. for var in walk(dataset): if type(var) in (StructureType, DatasetType): data2 = data2[0] elif self.id.endswith(var.id): return data2
def __iter__(self): scheme, netloc, path, query, fragment = urlsplit(self.baseurl) if isinstance(self.descr[1], list): id = ','.join('%s.%s' % (self.id, d[0]) for d in self.descr[1]) else: id = self.id url = urlunsplit( (scheme, netloc, path + '.dods', id + hyperslab(self.slice) + '&' + '&'.join(self.selection), fragment)).rstrip('&') # download and unpack data r = requests.get(url, prefetch=False) # strip dds response marker = '\nData:\n' buf = [] while ''.join(buf) != marker: chunk = r.raw.read(1) if not chunk: break buf.append(chunk) buf = buf[-len(marker):] return unpack_sequence(BufferedReader(r.raw), self.descr)
def __getitem__(self, index): # build download url index = combine_slices(self.slice, fix_slice(index, self.shape)) scheme, netloc, path, query, fragment = urlsplit(self.baseurl) url = urlunsplit( (scheme, netloc, path + '.dods', self.id + hyperslab(index) + '&' + query, fragment)).rstrip('&') # download and unpack data r = requests.get(url) dds, data = r.content.split('\nData:\n', 1) if self.shape: # skip size packing if self.dtype.char == 'S': data = data[4:] else: data = data[8:] # calculate array size shape = tuple((s.stop - s.start) / s.step for s in index) size = np.prod(shape) if self.dtype == np.byte: return np.fromstring(data[:size], 'B') elif self.dtype.char == 'S': out = [] for word in range(size): n = np.fromstring(data[:4], '>I') # read length data = data[4:] out.append(data[:n]) data = data[n + (-n % 4):] return np.array(out, 'S') else: return np.fromstring(data, self.dtype).reshape(shape)
def __getitem__(self, index): slice_ = combine_slices(self._slice, fix_slice(index, self.shape)) scheme, netloc, path, query, fragment = urlsplit(self.url) url = urlunsplit(( scheme, netloc, path + '.dods', quote(self.id) + hyperslab(slice_) + '&' + query, fragment)) resp, data = request(url) dds, xdrdata = data.split('\nData:\n', 1) dataset = DDSParser(dds).parse() data = data2 = DapUnpacker(xdrdata, dataset).getvalue() # Retrieve the data from any parent structure(s). for var in walk(dataset): if type(var) in (StructureType, DatasetType): data = data[0] elif var.id == self.id: return data # Some old servers return the wrong response. :-/ # I found a server that would return an array to a request # for an array inside a grid (instead of a structure with # the array); this will take care of it. for var in walk(dataset): if type(var) in (StructureType, DatasetType): data2 = data2[0] elif self.id.endswith(var.id): return data2
def __iter__(self): scheme, netloc, path, query, fragment = urlsplit(self.baseurl) if isinstance(self.descr[1], list): id = ','.join('%s.%s' % (self.id, d[0]) for d in self.descr[1]) else: id = self.id url = urlunsplit(( scheme, netloc, path + '.dods', id + hyperslab(self.slice) + '&' + '&'.join(self.selection), fragment)).rstrip('&') # download and unpack data r = requests.get(url, stream=True) r.raise_for_status() stream = StreamReader(r.iter_content(BLOCKSIZE)) # strip dds response marker = '\nData:\n' buf = [] while ''.join(buf) != marker: chunk = stream.read(1) if not chunk: break buf.append(chunk) buf = buf[-len(marker):] return unpack_sequence(stream, self.descr)
def url(self): """Return url from where data is fetched.""" scheme, netloc, path, query, fragment = urlsplit(self.baseurl) url = urlunsplit( (scheme, netloc, path + ".dods", self.id + hyperslab(self.slice) + "&" + "&".join(self.selection), fragment) ).rstrip("&") return url
def url(self): """Return url from where data is fetched.""" scheme, netloc, path, query, fragment = urlsplit(self.baseurl) url = urlunsplit( (scheme, netloc, path + '.dods', self.id + hyperslab(self.slice) + '&' + '&'.join(self.selection), fragment)).rstrip('&') return url
def __getitem__(self, index): # build download url index = combine_slices(self.slice, fix_slice(index, self.shape)) scheme, netloc, path, query, fragment = urlsplit(self.baseurl) url = urlunsplit( (scheme, netloc, path + ".dods", quote(self.id) + hyperslab(index) + "&" + query, fragment) ).rstrip("&") # download and unpack data logger.info("Fetching URL: %s" % url) r = GET(url, self.application, self.session) raise_for_status(r) dds, data = r.body.split(b"\nData:\n", 1) dds = dds.decode(r.content_encoding or "ascii") if self.shape: # skip size packing if self.dtype.char in "SU": data = data[4:] else: data = data[8:] # calculate array size shape = tuple(int(np.ceil((s.stop - s.start) / float(s.step))) for s in index) size = int(np.prod(shape)) if self.dtype == np.byte: return np.fromstring(data[:size], "B").reshape(shape) elif self.dtype.char in "SU": out = [] for word in range(size): n = np.asscalar(np.fromstring(data[:4], ">I")) # read length data = data[4:] out.append(data[:n]) data = data[n + (-n % 4) :] return np.array([text_type(x.decode("ascii")) for x in out], "S").reshape(shape) else: try: return np.fromstring(data, self.dtype).reshape(shape) except ValueError as e: if str(e) == "total size of new array must be unchanged": # server-side failure. # it is expected that the user should be mindful of this: raise RuntimeError( ( "variable {0} could not be properly " "retrieved. To avoid this " "error consider using open_url(..., " "output_grid=False)." ).format(quote(self.id)) ) else: raise
def __getitem__(self, index): # build download url index = combine_slices(self.slice, fix_slice(index, self.shape)) scheme, netloc, path, query, fragment = urlsplit(self.baseurl) url = urlunsplit(( scheme, netloc, path + '.dods', quote(self.id) + hyperslab(index) + '&' + query, fragment)).rstrip('&') # download and unpack data logger.info("Fetching URL: %s" % url) r = GET(url, self.application, self.session) raise_for_status(r) dds, data = r.body.split(b'\nData:\n', 1) dds = dds.decode(r.content_encoding or 'ascii') if self.shape: # skip size packing if self.dtype.char in 'SU': data = data[4:] else: data = data[8:] # calculate array size shape = tuple( int(np.ceil((s.stop-s.start)/float(s.step))) for s in index) size = int(np.prod(shape)) if self.dtype == np.byte: return np.fromstring(data[:size], 'B').reshape(shape) elif self.dtype.char in 'SU': out = [] for word in range(size): n = np.asscalar(np.fromstring(data[:4], '>I')) # read length data = data[4:] out.append(data[:n]) data = data[n + (-n % 4):] return np.array([text_type(x.decode('ascii')) for x in out], 'S').reshape(shape) else: try: return np.fromstring(data, self.dtype).reshape(shape) except ValueError as e: if str(e) == 'total size of new array must be unchanged': # server-side failure. # it is expected that the user should be mindful of this: raise RuntimeError( ('variable {0} could not be properly ' 'retrieved. To avoid this ' 'error consider using open_url(..., ' 'output_grid=False).').format(quote(self.id))) else: raise
def __getitem__(self, key): out = copy.deepcopy(self) if isinstance(key, ConstraintExpression): scheme, netloc, path, query, fragment = urlsplit(self.url) out.url = urlunsplit(( scheme, netloc, path, str(key & query), fragment)) if out._slice != (slice(None),): warnings.warn('Selection %s will be applied before projection "%s".' % ( key, hyperslab(out._slice))) elif isinstance(key, basestring): out._slice = (slice(None),) out.children = () parent = self.id if ',' in parent: parent = parent.split(',', 1)[0].rsplit('.', 1)[0] out.id = '%s%s.%s' % (parent, hyperslab(self._slice), key) elif isinstance(key, tuple): out.children = key[:] else: out._slice = combine_slices(self._slice, fix_slice(key, (sys.maxint,))) return out
def __getitem__(self, key): out = copy.deepcopy(self) if isinstance(key, ConstraintExpression): scheme, netloc, path, query, fragment = urlsplit(self.url) out.url = urlunsplit( (scheme, netloc, path, str(key & query), fragment)) if out._slice != (slice(None), ): warnings.warn( 'Selection %s will be applied before projection "%s".' % (key, hyperslab(out._slice))) elif isinstance(key, basestring): out._slice = (slice(None), ) out.children = () parent = self.id if ',' in parent: parent = parent.split(',', 1)[0].rsplit('.', 1)[0] out.id = '%s%s.%s' % (parent, hyperslab(self._slice), key) elif isinstance(key, tuple): out.children = key[:] else: out._slice = combine_slices(self._slice, fix_slice(key, (sys.maxint, ))) return out
def __getitem__(self, index): # build download url index = combine_slices(self.slice, fix_slice(index, self.shape)) scheme, netloc, path, query, fragment = urlsplit(self.baseurl) url = urlunsplit(( scheme, netloc, path + '.dods', quote(self.id) + hyperslab(index) + '&' + query, fragment)).rstrip('&') # download and unpack data logger.info("Fetching URL: %s" % url) r = GET(url, self.application, self.session) raise_for_status(r) dds, data = r.body.split(b'\nData:\n', 1) dds = dds.decode(r.content_encoding or 'ascii') if self.shape: # skip size packing if self.dtype.char in 'SU': data = data[4:] else: data = data[8:] # calculate array size shape = tuple( int(np.ceil((s.stop-s.start)/float(s.step))) for s in index) size = int(np.prod(shape)) if self.dtype == np.byte: return np.fromstring(data[:size], 'B') elif self.dtype.char in 'SU': out = [] for word in range(size): n = np.fromstring(data[:4], '>I') # read length data = data[4:] out.append(data[:n]) data = data[n + (-n % 4):] return np.array([ text_type(x.decode('ascii')) for x in out ], 'S') else: return np.fromstring(data, self.dtype).reshape(shape)
def __getitem__(self, index): # build download url index = combine_slices(self.slice, fix_slice(index, self.shape)) scheme, netloc, path, query, fragment = urlsplit(self.baseurl) url = urlunsplit(( scheme, netloc, path + '.dods', self.id + hyperslab(index) + '&' + query, fragment)).rstrip('&') # download and unpack data r = requests.get(url) r.raise_for_status() dds, data = r.content.split('\nData:\n', 1) if self.shape: # skip size packing if self.dtype.char == 'S': data = data[4:] else: data = data[8:] # calculate array size shape = tuple((s.stop-s.start)/s.step for s in index) size = np.prod(shape) if self.dtype == np.byte: return np.fromstring(data[:size], 'B') elif self.dtype.char == 'S': out = [] for word in range(size): n = np.fromstring(data[:4], '>I') # read length data = data[4:] out.append(data[:n]) data = data[n + (-n % 4):] return np.array(out, 'S') else: return np.fromstring(data, self.dtype).reshape(shape)
def __iter__(self): scheme, netloc, path, query, fragment = urlsplit(self.url) id_ = ','.join('%s.%s' % (self.id, child) for child in self.children) or self.id url = urlunsplit(( scheme, netloc, path + '.dods', quote(id_) + hyperslab(self._slice) + '&' + query, fragment)) resp, data = request(url) dds, xdrdata = data.split('\nData:\n', 1) dataset = DDSParser(dds).parse() dataset.data = DapUnpacker(xdrdata, dataset).getvalue() dataset._set_id() # Strip any projections from the request id. id_ = re.sub('\[.*?\]', '', self.id) # And return the proper data. for var in walk(dataset): if var.id == id_: data = var.data if isinstance(var, SequenceType): order = [var.keys().index(k) for k in self.children] data = reorder(order, data, var._nesting_level) return iter(data)
def __iter__(self): scheme, netloc, path, query, fragment = urlsplit(self.url) id_ = ','.join('%s.%s' % (self.id, child) for child in self.children) or self.id url = urlunsplit( (scheme, netloc, path + '.dods', id_ + hyperslab(self._slice) + '&' + query, fragment)) resp, data = request(url) dds, xdrdata = data.split('\nData:\n', 1) dataset = DDSParser(dds).parse() dataset.data = DapUnpacker(xdrdata, dataset).getvalue() dataset._set_id() # Strip any projections from the request id. id_ = re.sub('\[.*?\]', '', self.id) # And return the proper data. for var in walk(dataset): if var.id == id_: data = var.data if isinstance(var, SequenceType): order = [var.keys().index(k) for k in self.children] data = reorder(order, data, var._nesting_level) return iter(data)
def __repr__(self): return '<%s pointing to variable "%s%s" at "%s">' % ( self.__class__.__name__, self.id, hyperslab(self._slice), self.url)
def test_no_tuple(self): """Test that slices that are not tuples work.""" slice_ = slice(0) self.assertEqual(hyperslab(slice_), "[0:1:%d]" % (MAXSIZE-1))
def __repr__(self): id_ = ','.join('%s.%s' % (self.id, child) for child in self.children) or self.id return '<%s pointing to variable "%s%s" at "%s">' % ( self.__class__.__name__, id_, hyperslab(self._slice), self.url)
def test_remove(self): """Test that excess slices are removed.""" slice_ = (slice(0), slice(None)) self.assertEqual(hyperslab(slice_), "[0:1:%d]" % (MAXSIZE-1))
def test_ndimensional(self): """Test n-dimensions slices.""" slice_ = (slice(1, 10, 1), slice(2, 10, 2)) self.assertEqual(hyperslab(slice_), "[1:1:9][2:2:9]")
def test_no_tuple(self): """Test that slices that are not tuples work.""" slice_ = slice(0) self.assertEqual(hyperslab(slice_), "[0:1:%d]" % (MAXSIZE - 1))
def test_remove(self): """Test that excess slices are removed.""" slice_ = (slice(0), slice(None)) self.assertEqual(hyperslab(slice_), "[0:1:%d]" % (MAXSIZE - 1))