def __init__(self, ds): if get_requests() is None: raise ImportError("This functionality depends on the requests package") self._url = ds.base_url # This should eventually manage the IO and cache it self.total_bytes = 0 super(IOHandlerHTTPStream, self).__init__(ds)
def _parse_parameter_file(self): self.dimensionality = 3 self.refine_by = 2 self.parameters["HydroMethod"] = "sph" # Here's where we're going to grab the JSON index file requests = get_requests() hreq = requests.get(self.base_url + "/yt_index.json") if hreq.status_code != 200: raise RuntimeError header = json.loads(hreq.content) header['particle_count'] = dict((int(k), header['particle_count'][k]) for k in header['particle_count']) self.parameters = header # Now we get what we need self.domain_left_edge = np.array(header['domain_left_edge'], "float64") self.domain_right_edge = np.array(header['domain_right_edge'], "float64") nz = 1 << self.over_refine_factor self.domain_dimensions = np.ones(3, "int32") * nz self.periodicity = (True, True, True) self.current_time = header['current_time'] self.unique_identifier = header.get("unique_identifier", time.time()) self.cosmological_simulation = int(header['cosmological_simulation']) for attr in ('current_redshift', 'omega_lambda', 'omega_matter', 'hubble_constant'): setattr(self, attr, float(header[attr])) self.file_count = header['num_files']
def _is_valid(self, *args, **kwargs): if not args[0].startswith("http://"): return False requests = get_requests() if requests is None: return False hreq = requests.get(args[0] + "/yt_index.json") if hreq.status_code == 200: return True return False
def _is_valid(cls, filename, *args, **kwargs): if not filename.startswith("http://"): return False requests = get_requests() if requests is None: return False hreq = requests.get(filename + "/yt_index.json") if hreq.status_code == 200: return True return False
def __init__(self, base_url, dataset_type="http_particle_stream", unit_system="cgs", index_order=None, index_filename=None): if get_requests() is None: raise ImportError( "This functionality depends on the requests package") self.base_url = base_url super(HTTPStreamDataset, self).__init__( "", dataset_type=dataset_type, unit_system=unit_system, index_order=index_order, index_filename=index_filename)
def _open_stream(self, data_file, field): # This does not actually stream yet! ftype, fname = field s = "%s/%s/%s/%s" % (self._url, data_file.file_id, ftype, fname) mylog.info("Loading URL %s", s) requests = get_requests() resp = requests.get(s) if resp.status_code != 200: raise RuntimeError self.total_bytes += len(resp.content) return resp.content
def __init__(self, base_url, dataset_type="http_particle_stream", n_ref=64, over_refine_factor=1, unit_system="cgs"): if get_requests() is None: raise ImportError( "This functionality depends on the requests package") self.base_url = base_url self.n_ref = n_ref self.over_refine_factor = over_refine_factor super(HTTPStreamDataset, self).__init__("", dataset_type, unit_system=unit_system)
def _is_valid(cls, *args, **kwargs): sdf_header = kwargs.get('sdf_header', args[0]) if sdf_header.startswith("http"): requests = get_requests() if requests is None: return False hreq = requests.get(sdf_header, stream=True) if hreq.status_code != 200: return False # Grab a whole 4k page. line = next(hreq.iter_content(4096)) elif os.path.isfile(sdf_header): with safeopen(sdf_header, "r", encoding = 'ISO-8859-1') as f: line = f.read(10).strip() else: return False return line.startswith("# SDF")
def _parse_parameter_file(self): self.dimensionality = 3 self.refine_by = 2 self.parameters["HydroMethod"] = "sph" # Here's where we're going to grab the JSON index file requests = get_requests() hreq = requests.get(self.base_url + "/yt_index.json") if hreq.status_code != 200: raise RuntimeError header = json.loads(hreq.content) header["particle_count"] = { int(k): header["particle_count"][k] for k in header["particle_count"] } self.parameters = header # Now we get what we need self.domain_left_edge = np.array(header["domain_left_edge"], "float64") self.domain_right_edge = np.array(header["domain_right_edge"], "float64") self.domain_dimensions = np.ones(3, "int32") self._periodicity = (True, True, True) self.current_time = header["current_time"] self.unique_identifier = header.get("unique_identifier", time.time()) self.cosmological_simulation = int(header["cosmological_simulation"]) for attr in ( "current_redshift", "omega_lambda", "omega_matter", "hubble_constant", ): setattr(self, attr, float(header[attr])) self.file_count = header["num_files"]