def generate_geom_meta(geom): """Generate geometry metadata dict. Currently, this sinmply hashes on the geometry coordinates. Note that the values are rounded to the nearest centimeter for hashing purposes. (Also, the values are converted to integers at this precision to eliminate any possible float32 / float64 issues that could cause discrepancies in hash values for what we consider to be equal geometries.) Parameters ---------- geom : shape (n_strings, n_depths, 3) numpy ndarray, dtype float{32,64} Returns ------- metadata : OrderedDict Contains the item: 'hash' : length-8 str Hex characters convert to a string of length 8 """ assert len(geom.shape) == 3 assert geom.shape[2] == 3 rounded_ints = np.round(geom * 100).astype(np.int) geom_hash = hash_obj(rounded_ints, fmt='hex')[:8] return OrderedDict([('hash', geom_hash)])
def generate_clsim_table_meta(r_binning_kw, t_binning_kw, costheta_binning_kw, costhetadir_binning_kw, deltaphidir_binning_kw, tray_kw_to_hash): """ Returns ------- hash_val : string 8 hex characters indicating hash value for the table metaname : string Filename for file that will contain the complete metadata used to define this set of tables """ linear_binning_keys = sorted(['min', 'max', 'n_bins']) power_binning_keys = sorted(['min', 'max', 'power', 'n_bins']) for binning_kw in [t_binning_kw, costheta_binning_kw, costhetadir_binning_kw, deltaphidir_binning_kw]: assert sorted(binning_kw.keys()) == linear_binning_keys assert sorted(r_binning_kw.keys()) == power_binning_keys tray_keys = sorted(['PhotonSource', 'Zenith', 'Azimuth', 'NEvents', 'IceModel', 'DisableTilt', 'PhotonPrescale', 'Sensor']) assert sorted(tray_kw_to_hash.keys()) == tray_keys hashable_params = dict( r_binning_kw=r_binning_kw, t_binning_kw=t_binning_kw, costheta_binning_kw=costheta_binning_kw, costhetadir_binning_kw=costhetadir_binning_kw, deltaphidir_binning_kw=deltaphidir_binning_kw, tray_kw_to_hash=tray_kw_to_hash ) hash_val = hash_obj(hashable_params, fmt='hex')[:8] metaname = CLSIM_TABLE_METANAME_PROTO[-1].format(hash_val=hash_val) return hash_val, metaname
def generate_tdi_table_meta(binmap_hash, geom_hash, dom_tables_hash, times_str, x_min, x_max, y_min, y_max, z_min, z_max, binwidth, anisotropy, ic_dom_quant_eff, dc_dom_quant_eff, ic_exponent, dc_exponent): """Generate a metadata dict for a time- and DOM-independent Cartesian (x,y,z)-binned table. Parameters ---------- binmap_hash : string geom_hash : string dom_tables_hash : string times_str : string x_lims, y_lims, z_lims : 2-tuples of floats binwidth : float anisotropy : None or tuple ic_dom_quant_eff : float in [0, 1] dc_dom_quant_eff : float in [0, 1] ic_exponent : float >= 0 dc_exponent : float >= 0 Returns ------- metadata : OrderedDict Contains keys 'fbasename' : string 'hash' : string 'kwargs' : OrderedDict """ if dom_tables_hash is None: dom_tables_hash = 'none' kwargs = OrderedDict([('geom_hash', geom_hash), ('binmap_hash', binmap_hash), ('dom_tables_hash', dom_tables_hash), ('times_str', times_str), ('x_min', x_min), ('x_max', x_max), ('y_min', y_min), ('y_max', y_max), ('z_min', z_min), ('z_max', z_max), ('binwidth', binwidth), ('anisotropy', anisotropy), ('ic_dom_quant_eff', ic_dom_quant_eff), ('dc_dom_quant_eff', dc_dom_quant_eff), ('ic_exponent', ic_exponent), ('dc_exponent', dc_exponent)]) hash_params = deepcopy(kwargs) for param in ['x_min', 'x_max', 'y_min', 'y_max', 'z_min', 'z_max']: rounded_int = int(np.round(hash_params[param] * 100)) hash_params[param] = rounded_int kwargs[param] = float(rounded_int) / 100 for param in [ 'ic_dom_quant_eff', 'dc_dom_quant_eff', 'ic_exponent', 'dc_exponent' ]: rounded_int = int(np.round(hash_params[param] * 10000)) hash_params[param] = rounded_int kwargs[param] = float(rounded_int) / 10000 hash_params['binwidth'] = int(np.round(hash_params['binwidth'] * 1e10)) tdi_hash = hash_obj(hash_params, fmt='hex') anisotropy_str = generate_anisotropy_str(anisotropy) fname = TDI_TABLE_FNAME_PROTO[-1].format(tdi_hash=tdi_hash, anisotropy_str=anisotropy_str, table_name='', **kwargs) fbasename = fname.rsplit('_.fits')[0] metadata = OrderedDict([('fbasename', fbasename), ('hash', tdi_hash), ('kwargs', kwargs)]) return metadata
def generate_binmap_meta(r_max, r_power, n_rbins, n_costhetabins, n_phibins, cart_binwidth, oversample, antialias): """Generate metadata dict for spherical to Cartesian bin mapping, including the file name, hash string, and a dict with all of the parameters that contributed to these which can be passed via ``**binmap_kw`` to the `sphbin2cartbin` function. Parameters ---------- r_max : float Maximum radius in Retro (t,r,theta)-binned DOM table (meters) r_power : float Binning in radial direction is regular in the inverse of this power. I.e., every element of `np.diff(r**(1/r_power))` is equal. n_rbins, n_costhetabins, n_phibins : int cart_binwidth : float Cartesian bin widths, same in x, y, and z (meters) oversample : int Oversample factor, same in x, y, and z antialias : int Antialias factor Returns ------- metadata : OrderedDict Contains following items: 'fname' : string File name for the specified bin mapping 'hash' : length-8 string Hex digits represented as a string. 'kwargs' : OrderedDict The keyword args used for the hash. """ kwargs = OrderedDict([ ('r_max', r_max), ('r_power', r_power), ('n_rbins', n_rbins), ('n_costhetabins', n_costhetabins), ('n_phibins', n_phibins), ('cart_binwidth', cart_binwidth), ('oversample', oversample), ('antialias', antialias) ]) binmap_hash = hash_obj(kwargs, fmt='hex') print('kwargs:', kwargs) fname = ( 'sph2cart_binmap' '_%s' '_nr{n_rbins:d}_ncostheta{n_costhetabins:d}_nphi{n_phibins:d}' '_rmax{r_max:f}_rpwr{r_power}' '_bw{cart_binwidth:.6f}' '_os{oversample:d}' '_aa{antialias:d}' '.pkl'.format(**kwargs) ) % binmap_hash metadata = OrderedDict([ ('fname', fname), ('hash', binmap_hash), ('kwargs', kwargs) ]) return metadata
def generate_clsim_table( outdir, gcd, ice_model, angular_sensitivity, disable_tilt, disable_anisotropy, string, dom, n_events, seed, coordinate_system, binning, tableset_hash=None, tile=None, overwrite=False, compress=False, ): """Generate a CLSim table. See wiki.icecube.wisc.edu/index.php/Ice for information about ice models. Parameters ---------- outdir : string gcd : string ice_model : str E.g. "spice_mie", "spice_lea", ... angular_sensitivity : str E.g. "h2-50cm", "9" (which is equivalent to "new25" because, like, duh) disable_tilt : bool Whether to force no layer tilt in simulation (if tilt is present in bulk ice model; otherwise, this has no effect) disable_anisotropy : bool Whether to force no bulk ice anisotropy (if anisotropy is present in bulk ice model; otherwise, this has no effect) string : int in [1, 86] dom : int in [1, 60] n_events : int > 0 Note that the number of photons is much larger than the number of events (related to the "brightness" of the defined source). seed : int in [0, 2**32) Seed for CLSim's random number generator coordinate_system : string in {"spherical", "cartesian"} If spherical, base coordinate system is .. :: (r, theta, phi, t, costhetadir, (optionally abs)deltaphidir) If Cartesian, base coordinate system is .. :: (x, y, z, costhetadir, phidir) but if any of the coordinate axes are specified to have 0 bins, they will be omitted (but the overall order is maintained). binning : mapping If `coordinate_system` is "spherical", keys should be: "n_r_bins" "n_t_bins" "n_costheta_bins" "n_phi_bins" "n_costhetadir_bins" "n_deltaphidir_bins" "r_max" "r_power" "t_max" "t_power" "deltaphidir_power" If `coordinate_system` is "cartesian", keys should be: "n_x_bins" "n_y_bins" "n_z_bins" "n_costhetadir_bins" "n_phidir_bins" "x_min" "x_max" "y_min" "y_max" "z_min" "z_max" tableset_hash : str, optional Specify if the table is a tile used to generate a larger table tile : int >= 0, optional Specify if the table is a tile used to generate a larger table overwrite : bool, optional Whether to overwrite an existing table (default: False) compress : bool, optional Whether to pass the resulting table through zstandard compression (default: True) Raises ------ ValueError If `compress` is True but `zstd` command-line utility cannot be found AssertionError, ValueError If illegal argument values are passed ValueError If `overwrite` is False and a table already exists at the target path Notes ----- Binnings are as follows: * Radial binning is regular in the space of r**(1/r_power), with `n_r_bins` spanning from 0 to `r_max` meters. * Time binning is regular in the space of t**(1/t_power), with `n_t_bins` spanning from 0 to `t_max` nanoseconds. * Position zenith angle is binned regularly in the cosine of the zenith angle with `n_costhetadir_bins` spanning from -1 to +1. * Position azimuth angle is binned regularly, with `n_phi_bins` spanning from -pi to pi radians. * Photon directionality zenith angle (relative to IcedCube coordinate system) is binned regularly in cosine-zenith space, with `n_costhetadir_bins` spanning from `costhetadir_min` to `costhetadir_max` * Photon directionality azimuth angle; sometimes assumed to be symmetric about line from DOM to the center of the bin, so is binned as an absolute value, i.e., from 0 to pi radians. Otherwise, binned from -np.pi to +np.pi The following are forced upon the above binning specifications (and remaining parameters are specified as arguments to the function) * t_min = 0 (ns) * r_min = 0 (m) * costheta_min = -1 * costheta_max = 1 * phi_min = -pi (rad) * phi_max = pi (rad) * costhetadir_min = -1 * costhetadir_max = 1 * deltaphidir_min = 0 (rad) * deltaphidir_min = pi (rad) """ assert isinstance(n_events, Integral) and n_events > 0 assert isinstance(seed, Integral) and 0 <= seed < 2**32 assert ((tableset_hash is not None and tile is not None) or (tableset_hash is None and tile is None)) n_bins_per_dim = [] for key, val in binning.items(): if not key.startswith('n_'): continue assert isinstance(val, Integral), '{} not an integer'.format(key) assert val >= 0, '{} must be >= 0'.format(key) n_bins_per_dim.append(val) # Note: + 2 accounts for under & overflow bins in each dimension n_bins = np.product([n + 2 for n in n_bins_per_dim if n > 0]) assert n_bins > 0 #if n_bins > 2**32: # raise ValueError( # 'The flattened bin index in CLSim is represented by uint32 which' # ' has a max of 4 294 967 296, but the binning specified comes to' # ' {} bins ({} times too many).' # .format(n_bins, n_bins / 2**32) # ) ice_model = ice_model.strip() angular_sensitivity = angular_sensitivity.strip() # For now, hole ice model is hard-coded in our CLSim branch; see # clsim/private/clsim/I3CLSimLightSourceToStepConverterFlasher.cxx # in the branch you're using to check that this is correct assert angular_sensitivity == 'flasher_p1_0.30_p2_-1' gcd_info = extract_gcd(gcd) if compress and not any( access(join(path, 'zstd'), X_OK) for path in environ['PATH'].split(pathsep)): raise ValueError('`zstd` command not found in path') outdir = expand(outdir) mkdir(outdir) axes = OrderedDict() binning_kw = OrderedDict() # Note that the actual binning in CLSim is performed using float32, so we # first "truncate" all values to that precision. However, the `LinearAxis` # function requires Python floats (which are 64 bits), so we have to # convert all values to to `float` when passing as kwargs to `LinearAxis` # (and presumably the values will be re-truncated to float32 within the # CLsim code somewhere). Hopefully following this procedure, the values # actually used within CLSim are what we want...? CLSim is stupid. ftype = np.float32 if coordinate_system == 'spherical': binning['t_min'] = ftype(0) # ns binning['r_min'] = ftype(0) # meters costheta_min = ftype(-1.0) costheta_max = ftype(1.0) # See # clsim/resources/kernels/spherical_coordinates.c.cl # in the branch you're using to check that the following are correct phi_min = ftype(3.0543261766433716e-01) phi_max = ftype(6.5886182785034180e+00) binning['costhetadir_min'] = ftype(-1.0) binning['costhetadir_max'] = ftype(1.0) binning['deltaphidir_min'] = ftype(-3.1808626651763916e+00) binning['deltaphidir_max'] = ftype(3.1023228168487549e+00) if binning['n_r_bins'] > 0: assert isinstance(binning['r_power'], Integral) and binning['r_power'] > 0 r_binning_kw = OrderedDict([ ('min', float(binning['r_min'])), ('max', float(binning['r_max'])), ('n_bins', int(binning['n_r_bins'])), ]) if binning['r_power'] == 1: axes['r'] = LinearAxis(**r_binning_kw) else: r_binning_kw['power'] = int(binning['r_power']) axes['r'] = PowerAxis(**r_binning_kw) binning_kw['r'] = r_binning_kw if binning['n_costheta_bins'] > 0: costheta_binning_kw = OrderedDict([ ('min', float(costheta_min)), ('max', float(costheta_max)), ('n_bins', int(binning['n_costheta_bins'])), ]) axes['costheta'] = LinearAxis(**costheta_binning_kw) binning_kw['costheta'] = costheta_binning_kw if binning['n_phi_bins'] > 0: phi_binning_kw = OrderedDict([ ('min', float(phi_min)), ('max', float(phi_max)), ('n_bins', int(binning['n_phi_bins'])), ]) axes['phi'] = LinearAxis(**phi_binning_kw) binning_kw['phi'] = phi_binning_kw if binning['n_t_bins'] > 0: assert isinstance(binning['t_power'], Integral) and binning['t_power'] > 0 t_binning_kw = OrderedDict([ ('min', float(binning['t_min'])), ('max', float(binning['t_max'])), ('n_bins', int(binning['n_t_bins'])), ]) if binning['t_power'] == 1: axes['t'] = LinearAxis(**t_binning_kw) else: t_binning_kw['power'] = int(binning['t_power']) axes['t'] = PowerAxis(**t_binning_kw) binning_kw['t'] = t_binning_kw if binning['n_costhetadir_bins'] > 0: costhetadir_binning_kw = OrderedDict([ ('min', float(binning['costhetadir_min'])), ('max', float(binning['costhetadir_max'])), ('n_bins', int(binning['n_costhetadir_bins'])), ]) axes['costhetadir'] = LinearAxis(**costhetadir_binning_kw) binning_kw['costhetadir'] = costhetadir_binning_kw if binning['n_deltaphidir_bins'] > 0: assert (isinstance(binning['deltaphidir_power'], Integral) and binning['deltaphidir_power'] > 0) deltaphidir_binning_kw = OrderedDict([ ('min', float(binning['deltaphidir_min'])), ('max', float(binning['deltaphidir_max'])), ('n_bins', int(binning['n_deltaphidir_bins'])), ]) if binning['deltaphidir_power'] == 1: axes['deltaphidir'] = LinearAxis(**deltaphidir_binning_kw) else: deltaphidir_binning_kw['power'] = int( binning['deltaphidir_power']) axes['deltaphidir'] = PowerAxis(**deltaphidir_binning_kw) binning_kw['deltaphidir'] = deltaphidir_binning_kw elif coordinate_system == 'cartesian': binning['t_min'] = ftype(0) # ns binning['costhetadir_min'], binning['costhetadir_max'] = ftype( -1.0), ftype(1.0) binning['phidir_min'], binning['phidir_max'] = ftype(-np.pi), ftype( np.pi) # rad if binning['n_x_bins'] > 0: x_binning_kw = OrderedDict([ ('min', float(binning['x_min'])), ('max', float(binning['x_max'])), ('n_bins', int(binning['n_x_bins'])), ]) axes['x'] = LinearAxis(**x_binning_kw) binning_kw['x'] = x_binning_kw if binning['n_y_bins'] > 0: y_binning_kw = OrderedDict([ ('min', float(binning['y_min'])), ('max', float(binning['y_max'])), ('n_bins', int(binning['n_y_bins'])), ]) axes['y'] = LinearAxis(**y_binning_kw) binning_kw['y'] = y_binning_kw if binning['n_z_bins'] > 0: z_binning_kw = OrderedDict([ ('min', float(binning['z_min'])), ('max', float(binning['z_max'])), ('n_bins', int(binning['n_z_bins'])), ]) axes['z'] = LinearAxis(**z_binning_kw) binning_kw['z'] = z_binning_kw if binning['n_t_bins'] > 0: assert isinstance(binning['t_power'], Integral) and binning['t_power'] > 0 t_binning_kw = OrderedDict([ ('min', float(binning['t_min'])), ('max', float(binning['t_max'])), ('n_bins', int(binning['n_t_bins'])), ]) if binning['t_power'] == 1: axes['t'] = LinearAxis(**t_binning_kw) else: t_binning_kw['power'] = int(binning['t_power']) axes['t'] = PowerAxis(**t_binning_kw) binning_kw['t'] = t_binning_kw if binning['n_costhetadir_bins'] > 0: costhetadir_binning_kw = OrderedDict([ ('min', float(binning['costhetadir_min'])), ('max', float(binning['costhetadir_max'])), ('n_bins', int(binning['n_costhetadir_bins'])), ]) axes['costhetadir'] = LinearAxis(**costhetadir_binning_kw) binning_kw['costhetadir'] = costhetadir_binning_kw if binning['n_phidir_bins'] > 0: phidir_binning_kw = OrderedDict([ ('min', float(binning['phidir_min'])), ('max', float(binning['phidir_max'])), ('n_bins', int(binning['n_phidir_bins'])), ]) axes['phidir'] = LinearAxis(**phidir_binning_kw) binning_kw['phidir'] = phidir_binning_kw binning_order = BINNING_ORDER[coordinate_system] missing_dims = set(axes.keys()).difference(binning_order) if missing_dims: raise ValueError( '`binning_order` specified is {} but is missing dimension(s) {}'. format(binning_order, missing_dims)) axes_ = OrderedDict() binning_kw_ = OrderedDict() for dim in binning_order: if dim in axes: axes_[dim] = axes[dim] binning_kw_[dim] = binning_kw[dim] axes = axes_ binning_kw = binning_kw_ # NOTE: use SphericalAxes even if we're actually binning Cartesian since we # don't care how it handles e.g. volumes, and Cartesian isn't implemented # in CLSim yet axes = SphericalAxes(axes.values()) # Construct metadata initially with items that will be hashed metadata = OrderedDict([ ('source_gcd_i3_md5', gcd_info['source_gcd_i3_md5']), ('coordinate_system', coordinate_system), ('binning_kw', binning_kw), ('ice_model', ice_model), ('angular_sensitivity', angular_sensitivity), ('disable_tilt', disable_tilt), ('disable_anisotropy', disable_anisotropy) ]) # TODO: this is hard-coded in our branch of CLSim; make parameter & fix here! if 't' in binning: metadata['t_is_residual_time'] = True if tableset_hash is None: hash_val = hash_obj(metadata, fmt='hex')[:8] print('derived hash:', hash_val) else: hash_val = tableset_hash print('tableset_hash:', hash_val) metadata['hash_val'] = hash_val if tile is not None: metadata['tile'] = tile dom_spec = OrderedDict([('string', string), ('dom', dom)]) if 'depth_idx' in dom_spec and ('subdet' in dom_spec or 'string' in dom_spec): if 'subdet' in dom_spec: dom_spec['string'] = dom_spec.pop('subdet') string = dom_spec['string'] depth_idx = dom_spec['depth_idx'] if isinstance(string, str): subdet = dom_spec['subdet'].lower() dom_x, dom_y = 0, 0 ic_avg_z, dc_avg_z = get_average_dom_z_coords(gcd_info['geo']) if string == 'ic': dom_z = ic_avg_z[depth_idx] elif string == 'dc': dom_z = dc_avg_z[depth_idx] else: raise ValueError('Unrecognized subdetector {}'.format(subdet)) else: dom_x, dom_y, dom_z = gcd_info['geo'][string - 1, depth_idx] metadata['string'] = string metadata['depth_idx'] = depth_idx if tile is not None: raise ValueError( 'Cannot produce tiled tables using "depth_idx"-style table groupings;' ' use "string"/"dom"-style tables instead.') clsim_table_fname_proto = CLSIM_TABLE_FNAME_PROTO[1] clsim_table_metaname_proto = CLSIM_TABLE_METANAME_PROTO[0] print('Subdetector {}, depth index {} (z_avg = {} m)'.format( subdet, depth_idx, dom_z)) elif 'string' in dom_spec and 'dom' in dom_spec: string = dom_spec['string'] dom = dom_spec['dom'] dom_x, dom_y, dom_z = gcd_info['geo'][string - 1, dom - 1] metadata['string'] = string metadata['dom'] = dom if tile is None: clsim_table_fname_proto = CLSIM_TABLE_FNAME_PROTO[2] clsim_table_metaname_proto = CLSIM_TABLE_METANAME_PROTO[1] else: clsim_table_fname_proto = CLSIM_TABLE_TILE_FNAME_PROTO[-1] clsim_table_metaname_proto = CLSIM_TABLE_TILE_METANAME_PROTO[-1] print( 'GCD = "{}"\nString {}, dom {}: (x, y, z) = ({}, {}, {}) m'.format( gcd, string, dom, dom_x, dom_y, dom_z)) else: raise ValueError('Cannot understand `dom_spec` {}'.format(dom_spec)) # Until someone figures out DOM tilt and ice column / bubble column / cable # orientations for sure, we'll just set DOM orientation to zenith=pi, # azimuth=0. dom_zenith = np.pi dom_azimuth = 0.0 # Now add other metadata items that are useful but not used for hashing metadata['dom_x'] = dom_x metadata['dom_y'] = dom_y metadata['dom_z'] = dom_z metadata['dom_zenith'] = dom_zenith metadata['dom_azimuth'] = dom_azimuth metadata['seed'] = seed metadata['n_events'] = n_events metapath = join(outdir, clsim_table_metaname_proto.format(**metadata)) tablepath = join(outdir, clsim_table_fname_proto.format(**metadata)) # Save metadata as a JSON file (so it's human-readable by any tool, not # just Python--in contrast to e.g. pickle files) json.dump(metadata, file(metapath, 'w'), sort_keys=False, indent=4) print('=' * 80) print('Metadata for the table set was written to\n "{}"'.format(metapath)) print('Table will be written to\n "{}"'.format(tablepath)) print('=' * 80) exists_at = [] for fpath in [tablepath, tablepath + '.zst']: if isfile(fpath): exists_at.append(fpath) if exists_at: names = ', '.join('"{}"'.format(fp) for fp in exists_at) if overwrite: print('WARNING! Deleting existing table(s) at ' + names) for fpath in exists_at: remove(fpath) else: raise ValueError('Table(s) already exist at {}; not' ' overwriting.'.format(names)) print('') tray = I3Tray() tray.AddSegment( TabulateRetroSources, 'TabulateRetroSources', source_gcd_i3_md5=gcd_info['source_gcd_i3_md5'], binning_kw=binning_kw, axes=axes, ice_model=ice_model, angular_sensitivity=angular_sensitivity, disable_tilt=disable_tilt, disable_anisotropy=disable_anisotropy, hash_val=hash_val, dom_spec=dom_spec, dom_x=dom_x, dom_y=dom_y, dom_z=dom_z, dom_zenith=dom_zenith, dom_azimuth=dom_azimuth, seed=seed, n_events=n_events, tablepath=tablepath, tile=tile, record_errors=False, ) logging.set_level_for_unit('I3CLSimStepToTableConverter', 'TRACE') logging.set_level_for_unit('I3CLSimTabulatorModule', 'DEBUG') logging.set_level_for_unit('I3CLSimLightSourceToStepConverterGeant4', 'TRACE') logging.set_level_for_unit('I3CLSimLightSourceToStepConverterFlasher', 'TRACE') tray.Execute() tray.Finish() if compress: print('Compressing table with zstandard via command line') print(' zstd -1 --rm "{}"'.format(tablepath)) subprocess.check_call(['zstd', '-1', '--rm', tablepath]) print('done.')