def test_save_load_pickle(): """Main test for loading pickled objects""" return # Until boolean regions are implemented we can't test this test_ds = fake_random_ds(64) # create extracted region from boolean (fairly complex object) center = (test_ds.domain_left_edge + test_ds.domain_right_edge) / 2 sp_outer = test_ds.sphere(center, test_ds.domain_width[0]) sp_inner = test_ds.sphere(center, test_ds.domain_width[0] / 10.0) sp_boolean = test_ds.boolean([sp_outer, "NOT", sp_inner]) minv, maxv = sp_boolean.quantities["Extrema"]("density")[0] contour_threshold = min(minv * 10.0, 0.9 * maxv) contours = sp_boolean.extract_connected_sets( "density", 1, contour_threshold, maxv + 1, log_space=True, cache=True) # save object cpklfile = tempfile.NamedTemporaryFile(delete=False) cPickle.dump(contours[1][0], cpklfile) cpklfile.close() # load object test_load = cPickle.load(open(cpklfile.name, "rb")) assert_equal.description = \ "%s: File was pickle-loaded succesfully" % __name__ yield assert_equal, test_load is not None, True assert_equal.description = \ "%s: Length of pickle-loaded connected set object" % __name__ yield assert_equal, len(contours[1][0]), len(test_load) os.remove(cpklfile.name)
def __setitem__(self, test_name, result): # We have to close our shelf manually, # as the destructor does not necessarily do this. # Context managers would be more appropriate. f = open(self._fn(test_name), "wb") cPickle.dump(result, f, protocol=-1) f.close()
def _func(*args, **kwargs): name = kwargs.pop("result_basename", func.__name__) rv = func(*args, **kwargs) if hasattr(rv, "convert_to_base"): rv.convert_to_base() _rv = rv.ndarray_view() else: _rv = rv mi = _rv.min() ma = _rv.max() st = _rv.std(dtype="float64") su = _rv.sum(dtype="float64") si = _rv.size ha = hashlib.md5(_rv.tostring()).hexdigest() fn = "func_results_ref_%s.cpkl" % (name) with open(fn, "wb") as f: cPickle.dump((mi, ma, st, su, si, ha), f) return rv
def test_ytarray_pickle(): ds = fake_random_ds(64, nprocs=1) test_data = [ds.quan(12.0, 'code_length'), ds.arr([1, 2, 3], 'code_length')] for data in test_data: tempf = tempfile.NamedTemporaryFile(delete=False) pickle.dump(data, tempf) tempf.close() with open(tempf.name, "rb") as fname: loaded_data = pickle.load(fname) os.unlink(tempf.name) assert_array_equal(data, loaded_data) assert_equal(data.units, loaded_data.units) assert_array_equal(array(data.in_cgs()), array(loaded_data.in_cgs())) assert_equal(float(data.units.base_value), float(loaded_data.units.base_value))
def _func(*args, **kwargs): name = kwargs.pop("result_basename", func.__name__) rv = func(*args, **kwargs) if hasattr(rv, "convert_to_cgs"): rv.convert_to_cgs() _rv = rv.ndarray_view() else: _rv = rv mi = _rv.min() ma = _rv.max() st = _rv.std(dtype="float64") su = _rv.sum(dtype="float64") si = _rv.size ha = hashlib.md5(_rv.tostring()).hexdigest() fn = "func_results_ref_%s.cpkl" % (name) with open(fn, "wb") as f: cPickle.dump( (mi, ma, st, su, si, ha), f) return rv
def test_ytarray_pickle(): ds = fake_random_ds(64, nprocs=1) test_data = [ds.quan(12.0, 'code_length'), ds.arr([1, 2, 3], 'code_length')] for data in test_data: tempf = tempfile.NamedTemporaryFile(delete=False) pickle.dump(data, tempf) tempf.close() with open(tempf.name, "rb") as fname: loaded_data = pickle.load(fname) os.unlink(tempf.name) yield assert_array_equal, data, loaded_data yield assert_equal, data.units, loaded_data.units yield assert_array_equal, array(data.in_cgs()), \ array(loaded_data.in_cgs()) yield assert_equal, float(data.units.base_value), \ float(loaded_data.units.base_value)
def find_halo_relationships(output1_id, output2_id, output_basename=None, radius=0.10, external_FOF=True, FOF_directory='FOF'): r"""Calculate the parentage and child relationships between two EnzoFOF halo catalogs. This function performs a very simple merger tree calculation between two sets of halos. For every halo in the second halo catalog, it looks to the first halo catalog to find the parents by looking at particle IDs. The particle IDs from the child halos are identified in potential parents, and then both percent-of-parent and percent-to-child values are recorded. Note that this works with catalogs constructed by Enzo's FOF halo when used in external_FOF=True mode, whereas it will work with catalogs constructed by yt using external_FOF=False mode. Parameters ---------- output1_id : int This is the integer output id of the (first) halo catalog to parse and load. output2_id : int This is the integer output id of the (second) halo catalog to parse and load. output_basename : string If provided, both .cpkl and .txt files containing the parentage relationships will be output. radius : float, default to 0.10 In absolute units, the radius to examine when guessing possible parent/child relationships. If this value is too small, you will miss possible relationships. FOF_directory : str, optional Directory where FOF files are located Returns ------- pfrac : dict This is a dict of dicts. The first key is the parent halo id, the second is the child halo id. The values are the percent contributed from parent to child and the percent of a child that came from the parent. """ mylog.info("Parsing Halo Catalog %04i", output1_id) HC1 = HaloCatalog(output1_id, False, external_FOF=external_FOF, \ FOF_directory=FOF_directory) mylog.info("Parsing Halo Catalog %04i", output2_id) HC2 = HaloCatalog(output2_id, True, external_FOF=external_FOF, \ FOF_directory=FOF_directory) mylog.info("Calculating fractions") pfrac = HC1.calculate_parentage_fractions(HC2) if output_basename is not None and pfrac != {}: f = open("%s.txt" % (output_basename), "w") for hid1 in sorted(pfrac): for hid2 in sorted(pfrac[hid1]): if not str(hid2).isdigit(): continue p1, p2, npart = pfrac[hid1][hid2] if p1 == 0.0: continue f.write( "Halo %s (%s) contributed %0.3e of its particles to %s (%s), which makes up %0.3e of that halo\n" % (hid1, output1_id, p2, hid2, output2_id, p1)) f.close() cPickle.dump(pfrac, open("%s.cpkl" % (output_basename), "wb")) return HC1.redshift, HC2.redshift, pfrac
def save_tree(self, filename): cPickle.dump((self.redshifts, self.relationships), open(filename, "wb"))
def dump(self, storage): with open(storage, 'w') as fh: pickle.dump(self, fh)
def find_halo_relationships(output1_id, output2_id, output_basename = None, radius = 0.10, external_FOF=True, FOF_directory='FOF'): r"""Calculate the parentage and child relationships between two EnzoFOF halo catalogs. This function performs a very simple merger tree calculation between two sets of halos. For every halo in the second halo catalog, it looks to the first halo catalog to find the parents by looking at particle IDs. The particle IDs from the child halos are identified in potential parents, and then both percent-of-parent and percent-to-child values are recorded. Note that this works with catalogs constructed by Enzo's FOF halo when used in external_FOF=True mode, whereas it will work with catalogs constructed by yt using external_FOF=False mode. Parameters ---------- output1_id : int This is the integer output id of the (first) halo catalog to parse and load. output2_id : int This is the integer output id of the (second) halo catalog to parse and load. output_basename : string If provided, both .cpkl and .txt files containing the parentage relationships will be output. radius : float, default to 0.10 In absolute units, the radius to examine when guessing possible parent/child relationships. If this value is too small, you will miss possible relationships. FOF_directory : str, optional Directory where FOF files are located Returns ------- pfrac : dict This is a dict of dicts. The first key is the parent halo id, the second is the child halo id. The values are the percent contributed from parent to child and the percent of a child that came from the parent. """ mylog.info("Parsing Halo Catalog %04i", output1_id) HC1 = HaloCatalog(output1_id, False, external_FOF=external_FOF, \ FOF_directory=FOF_directory) mylog.info("Parsing Halo Catalog %04i", output2_id) HC2 = HaloCatalog(output2_id, True, external_FOF=external_FOF, \ FOF_directory=FOF_directory) mylog.info("Calculating fractions") pfrac = HC1.calculate_parentage_fractions(HC2) if output_basename is not None and pfrac != {}: f = open("%s.txt" % (output_basename), "w") for hid1 in sorted(pfrac): for hid2 in sorted(pfrac[hid1]): if not str(hid2).isdigit(): continue p1, p2, npart = pfrac[hid1][hid2] if p1 == 0.0: continue f.write( "Halo %s (%s) contributed %0.3e of its particles to %s (%s), which makes up %0.3e of that halo\n" % ( hid1, output1_id, p2, hid2, output2_id, p1)) f.close() cPickle.dump(pfrac, open("%s.cpkl" % (output_basename), "wb")) return HC1.redshift, HC2.redshift, pfrac