def __iter__(self): # download and unpack data r = GET(self.url, self.application, self.session) raise_for_status(r) i = r.app_iter if not hasattr(i, "__next__"): i = iter(i) # Fast forward past the DDS header # the pattern could span chunk boundaries though so make sure to check previous_chunk = b"" this_chunk = b"" pattern = b"Data:\n" for this_chunk in i: m = re.search(pattern, previous_chunk + this_chunk) if m: break if not m: raise ValueError("Could not find data segment in response from {}".format(self.url)) last_chunk = (previous_chunk + this_chunk)[m.end() :] # Then construct a stream consisting of everything from # 'Data:\n' to the end of the chunk + the rest of the stream def stream_start(): yield last_chunk stream = StreamReader(chain(stream_start(), i)) return unpack_sequence(stream, self.template)
def __iter__(self): # download and unpack data r = GET(self.url, self.application, self.session) raise_for_status(r) i = r.app_iter if not hasattr(i, '__next__'): i = iter(i) # Fast forward past the DDS header # the pattern could span chunk boundaries though so make sure to check previous_chunk = b'' this_chunk = b'' pattern = b'Data:\n' for this_chunk in i: m = re.search(pattern, previous_chunk + this_chunk) if m: break if not m: raise ValueError( "Could not find data segment in response from {}".format( self.url)) last_chunk = (previous_chunk + this_chunk)[m.end():] # Then construct a stream consisting of everything from # 'Data:\n' to the end of the chunk + the rest of the stream def stream_start(): yield last_chunk stream = StreamReader(chain(stream_start(), i)) return unpack_sequence(stream, self.template)
def __init__(self, url, application=None, session=None, output_grid=True): # download DDS/DAS scheme, netloc, path, query, fragment = urlsplit(url) ddsurl = urlunsplit((scheme, netloc, path + '.dds', query, fragment)) r = GET(ddsurl, application, session) raise_for_status(r) dds = r.text dasurl = urlunsplit((scheme, netloc, path + '.das', query, fragment)) r = GET(dasurl, application, session) raise_for_status(r) das = r.text # build the dataset from the DDS and add attributes from the DAS self.dataset = build_dataset(dds) add_attributes(self.dataset, parse_das(das)) # remove any projection from the url, leaving selections projection, selection = parse_ce(query) url = urlunsplit((scheme, netloc, path, '&'.join(selection), fragment)) # now add data proxies for var in walk(self.dataset, BaseType): var.data = BaseProxy(url, var.id, var.dtype, var.shape, application=application, session=session) for var in walk(self.dataset, SequenceType): template = copy.copy(var) var.data = SequenceProxy(url, template, application=application, session=session) # apply projections for var in projection: target = self.dataset while var: token, index = var.pop(0) target = target[token] if isinstance(target, BaseType): target.data.slice = fix_slice(index, target.shape) elif isinstance(target, GridType): index = fix_slice(index, target.array.shape) target.array.data.slice = index for s, child in zip(index, target.maps): target[child].data.slice = (s, ) elif isinstance(target, SequenceType): target.data.slice = index # retrieve only main variable for grid types: for var in walk(self.dataset, GridType): var.set_output_grid(output_grid)
def __getitem__(self, index): # build download url index = combine_slices(self.slice, fix_slice(index, self.shape)) scheme, netloc, path, query, fragment = urlsplit(self.baseurl) url = urlunsplit( (scheme, netloc, path + ".dods", quote(self.id) + hyperslab(index) + "&" + query, fragment) ).rstrip("&") # download and unpack data logger.info("Fetching URL: %s" % url) r = GET(url, self.application, self.session) raise_for_status(r) dds, data = r.body.split(b"\nData:\n", 1) dds = dds.decode(r.content_encoding or "ascii") if self.shape: # skip size packing if self.dtype.char in "SU": data = data[4:] else: data = data[8:] # calculate array size shape = tuple(int(np.ceil((s.stop - s.start) / float(s.step))) for s in index) size = int(np.prod(shape)) if self.dtype == np.byte: return np.fromstring(data[:size], "B").reshape(shape) elif self.dtype.char in "SU": out = [] for word in range(size): n = np.asscalar(np.fromstring(data[:4], ">I")) # read length data = data[4:] out.append(data[:n]) data = data[n + (-n % 4) :] return np.array([text_type(x.decode("ascii")) for x in out], "S").reshape(shape) else: try: return np.fromstring(data, self.dtype).reshape(shape) except ValueError as e: if str(e) == "total size of new array must be unchanged": # server-side failure. # it is expected that the user should be mindful of this: raise RuntimeError( ( "variable {0} could not be properly " "retrieved. To avoid this " "error consider using open_url(..., " "output_grid=False)." ).format(quote(self.id)) ) else: raise
def __getitem__(self, index): # build download url index = combine_slices(self.slice, fix_slice(index, self.shape)) scheme, netloc, path, query, fragment = urlsplit(self.baseurl) url = urlunsplit(( scheme, netloc, path + '.dods', quote(self.id) + hyperslab(index) + '&' + query, fragment)).rstrip('&') # download and unpack data logger.info("Fetching URL: %s" % url) r = GET(url, self.application, self.session) raise_for_status(r) dds, data = r.body.split(b'\nData:\n', 1) dds = dds.decode(r.content_encoding or 'ascii') if self.shape: # skip size packing if self.dtype.char in 'SU': data = data[4:] else: data = data[8:] # calculate array size shape = tuple( int(np.ceil((s.stop-s.start)/float(s.step))) for s in index) size = int(np.prod(shape)) if self.dtype == np.byte: return np.fromstring(data[:size], 'B').reshape(shape) elif self.dtype.char in 'SU': out = [] for word in range(size): n = np.asscalar(np.fromstring(data[:4], '>I')) # read length data = data[4:] out.append(data[:n]) data = data[n + (-n % 4):] return np.array([text_type(x.decode('ascii')) for x in out], 'S').reshape(shape) else: try: return np.fromstring(data, self.dtype).reshape(shape) except ValueError as e: if str(e) == 'total size of new array must be unchanged': # server-side failure. # it is expected that the user should be mindful of this: raise RuntimeError( ('variable {0} could not be properly ' 'retrieved. To avoid this ' 'error consider using open_url(..., ' 'output_grid=False).').format(quote(self.id))) else: raise
def __init__(self, url, application=None, session=None, output_grid=True): # download DDS/DAS scheme, netloc, path, query, fragment = urlsplit(url) ddsurl = urlunsplit((scheme, netloc, path + ".dds", query, fragment)) r = GET(ddsurl, application, session) raise_for_status(r) dds = r.text dasurl = urlunsplit((scheme, netloc, path + ".das", query, fragment)) r = GET(dasurl, application, session) raise_for_status(r) das = r.text # build the dataset from the DDS and add attributes from the DAS self.dataset = build_dataset(dds) add_attributes(self.dataset, parse_das(das)) # remove any projection from the url, leaving selections projection, selection = parse_ce(query) url = urlunsplit((scheme, netloc, path, "&".join(selection), fragment)) # now add data proxies for var in walk(self.dataset, BaseType): var.data = BaseProxy(url, var.id, var.dtype, var.shape, application=application, session=session) for var in walk(self.dataset, SequenceType): template = copy.copy(var) var.data = SequenceProxy(url, template, application=application, session=session) # apply projections for var in projection: target = self.dataset while var: token, index = var.pop(0) target = target[token] if isinstance(target, BaseType): target.data.slice = fix_slice(index, target.shape) elif isinstance(target, GridType): index = fix_slice(index, target.array.shape) target.array.data.slice = index for s, child in zip(index, target.maps): target[child].data.slice = (s,) elif isinstance(target, SequenceType): target.data.slice = index # retrieve only main variable for grid types: for var in walk(self.dataset, GridType): var.set_output_grid(output_grid)
def __getitem__(self, index): # build download url index = combine_slices(self.slice, fix_slice(index, self.shape)) scheme, netloc, path, query, fragment = urlsplit(self.baseurl) url = urlunsplit(( scheme, netloc, path + '.dods', quote(self.id) + hyperslab(index) + '&' + query, fragment)).rstrip('&') # download and unpack data logger.info("Fetching URL: %s" % url) r = GET(url, self.application, self.session) raise_for_status(r) dds, data = r.body.split(b'\nData:\n', 1) dds = dds.decode(r.content_encoding or 'ascii') if self.shape: # skip size packing if self.dtype.char in 'SU': data = data[4:] else: data = data[8:] # calculate array size shape = tuple( int(np.ceil((s.stop-s.start)/float(s.step))) for s in index) size = int(np.prod(shape)) if self.dtype == np.byte: return np.fromstring(data[:size], 'B') elif self.dtype.char in 'SU': out = [] for word in range(size): n = np.fromstring(data[:4], '>I') # read length data = data[4:] out.append(data[:n]) data = data[n + (-n % 4):] return np.array([ text_type(x.decode('ascii')) for x in out ], 'S') else: return np.fromstring(data, self.dtype).reshape(shape)