def train_hmm(clr: cooler.Cooler, mix_num: int = 3, discore_fn=di_score): """ :param clr: :param mix_num: :param discore_fn: :return: """ model = ghmm_model(STATES, INIT_TRANSITION, INIT_PROB, END_PROB, init_mean_fn(mix_num), init_var_fn(mix_num)) di_dict = {} for chrom in clr.chromnames: matrix = clr.matrix(sparse=True).fetch(chrom).tocsr() di_array = discore_fn(matrix) gap_mask = remove_small_gap( np.isnan(clr.bins().fetch(chrom)['weight'].values)) di_dict[chrom] = split_diarray(di_array, gap_mask) train_data = [] for chrom_di in di_dict.values(): train_data.extend(di for di in chrom_di.values()) model.fit( train_data, algorithm='baum-welch', max_iterations=10000, stop_threshold=1e-5, n_jobs=CPU_CORE - 5, verbose=False ) return model
def make_slices( clr: Cooler, regions: Dict[str, np.ndarray], names:Optional[dict] = {}, force_disjoint:Optional[bool] = False ) -> List[np.ndarray]: # Fetch relevant bin_ids from the cooler file b_ids, n_ids = fetch_bins_from_cooler(cooler=clr, regions=regions, names=names) if force_disjoint: # Identify unique bin_ids and isolate disjoint regions slices = {chrom: get_unique_bins(b_ids=b_ids[chrom]) for chrom in b_ids} n_ids = {} for chrom in slices: n_ids[chrom] = [] for sl in slices[chrom]: # start, end, bins and node names for region stl = clr.bins()[sl[0]]["start"].values[0] el = clr.bins()[sl[-1] + 1]["end"].values[0] sl_id = f"{chrom}:{stl}-{el}" n_ids[chrom].append(sl_id) else: slices = {chrom:[np.array(item) for item in b_ids[chrom]] for chrom in b_ids} return slices, n_ids
def __init__(self, fan_pin, peltier_pin, step_pin, thermometer_pin): self.cooler = Cooler(Pin(fan_pin, Pin.OUT), Pin(peltier_pin, Pin.OUT)) self.cooler.fanOn() self.cooler.coolerHigh() self.pid = Pid(17, 1, 0.02, 0, 0.995) #(temperature, P, I, D, memoryFactor) self.thermometer = Thermometer(thermometer_pin) self.pump = PWMPump(Pin(step_pin, Pin.OUT)) for i in range(10): self.pump.pwm.freq(i * 1500) utime.sleep(0.1) self.pump.pwm.freq(15000)
def make_bins( clr: Cooler, sites: Dict[str,np.ndarray], names: Optional[Dict[str,str]]=None ) -> Dict[str, np.ndarray]: bins = {} outnames = {} bad_sites = {} for chrom in sites: cbins = clr.bins().fetch(chrom) start = cbins['start'].values[0] site_locs = ((sites[chrom] - start)/clr.binsize).astype('int') good_sites = site_locs < cbins.shape[0] bad_sites[chrom] = np.where(~good_sites)[0] bins[chrom] = site_locs[good_sites] if names is not None: outnames[chrom] = np.array(names[chrom])[good_sites] if names is not None: return bins, outnames, bad_sites else: return bins, bad_sites
def __init__(self, source_uri, bins, chunksize, batchsize, map=map): from cooler.api import Cooler self._map = map self.source_uri = source_uri self.chunksize = chunksize self.batchsize = batchsize clr = Cooler(source_uri) self._size = clr.info['nnz'] self.old_binsize = clr.binsize self.old_chrom_offset = clr._load_dset('indexes/chrom_offset') self.old_bin1_offset = clr._load_dset('indexes/bin1_offset') self.gs = GenomeSegmentation(clr.chromsizes, bins) self.new_binsize = get_binsize(bins) assert self.new_binsize % self.old_binsize == 0 self.factor = self.new_binsize // self.old_binsize
def get_coolers(self, table, res=1000000): names = table['lib_name'].values cool_dict = defaultdict(list) for name in names: if name not in self.metadata['lib_name'].values: print(f'Name: {name} not found in metadata. Skipping') continue cool_dict['lib_name'].append(name) flag = True for cpath in self.cooler_paths: if f'{name}.hg38.mapq_30.1000.mcool' in os.listdir(cpath): flag = False cool = Cooler( cpath + f'{name}.hg38.mapq_30.1000.mcool::/resolutions/{res}') cool_dict[f'cooler_{res}'].append(cool) if flag: print( f'Cooler not found matching {name}. Appending np.nan to appropriate row' ) cool_dict[f'cooler_{res}'].append(np.nan) df = pd.DataFrame(cool_dict) df = table.copy(deep=True).merge(df, on='lib_name', how='outer') return df
def fetch_bins_from_cooler( cooler: Cooler, regions: Dict[str, np.ndarray], names: Optional[dict] = {} ) -> List[List[np.int64]]: # Fetch relevant bin_ids from the cooler file b_ids = {} n_ids = {} for chrom in regions: b_ids[chrom] = [] n_ids[chrom] = [] for idx, row in enumerate(regions[chrom]): b_add = list( cooler.bins() .fetch("{}:{}-{}".format(chrom, row[0], row[1])) .index.values ) try: n_ids[chrom].append(names[chrom][idx]) except: n_ids[chrom].append("{}:{}-{}".format(chrom, row[0], row[1])) b_ids[chrom].append( b_add ) return b_ids, n_ids
def coords_to_bins(clr: cooler.Cooler, coords: pd.DataFrame) -> np.ndarray: """ Converts genomic coordinates to a list of bin ids based on the whole genome contact map. Parameters ---------- coords : pandas.DataFrame Table of genomic coordinates, with columns chrom, pos. Returns ------- numpy.array of ints : Indices in the whole genome matrix contact map. """ coords.pos = (coords.pos // clr.binsize) * clr.binsize # Coordinates are merged with bins, both indices are kept in memory so that # the indices of matching bins can be returned in the order of the input # coordinates idx = (clr.bins()[:].reset_index().rename(columns={ "index": "bin_idx" }).merge( coords.reset_index().rename(columns={"index": "coord_idx"}), left_on=["chrom", "start"], right_on=["chrom", "pos"], how="right", ).set_index("bin_idx").sort_values("coord_idx").index.values) return idx
def matrix_balance(cool_uri, nproc=1, chunksize=int(1e7), mad_max=5, min_nnz=10, min_count=0, ignore_diags=1, tol=1e-5, max_iters=1000): ''' Perform separate matrix balancing for regions with different copy numbers and output the bias vector in the "sweight" column. ''' cool_path, group_path = util.parse_cooler_uri(cool_uri) # Overwrite the existing sweight column with h5py.File(cool_path, 'r+') as h5: grp = h5[group_path] if 'sweight' in grp['bins']: del grp['bins']['sweight'] clr = Cooler(cool_uri) try: if nproc > 1: pool = balance.Pool(nproc) map_ = pool.imap_unordered else: map_ = map bias, stats = iterative_correction(clr, chunksize=chunksize, tol=tol, min_nnz=min_nnz, min_count=min_count, mad_max=mad_max, max_iters=max_iters, ignore_diags=ignore_diags, rescale_marginals=True, use_lock=False, map=map_) finally: if nproc > 1: pool.close() if not stats['converged']: logger.error('Iteration limit reached without convergence') logger.error('Storing final result. Check log to assess convergence.') with h5py.File(cool_path, 'r+') as h5: grp = h5[group_path] # add the bias column to the file h5opts = dict(compression='gzip', compression_opts=6) grp['bins'].create_dataset('sweight', data=bias, **h5opts) grp['bins']['sweight'].attrs.update(stats)
def correlate(x_mcool, y_mcool, output_prefix, resolution): from pore_c.analyses.matrix import correlate from cooler import Cooler file_paths = catalogs.MatrixCorrelationCatalog.generate_paths( output_prefix) x_cool = Cooler(str(x_mcool) + f"::/resolutions/{resolution}") y_cool = Cooler(str(y_mcool) + f"::/resolutions/{resolution}") x_chrom_names = set(x_cool.chromnames) y_chrom_names = set(y_cool.chromnames) if x_chrom_names != y_chrom_names: x_not_y = x_chrom_names - y_chrom_names y_not_x = y_chrom_names - x_chrom_names if x_not_y and y_not_x: raise ValueError( f"Chromosomes are not sub/supersets x:{x_not_y}, y:{y_not_x}") elif x_not_y: logger.warning( f"Extra chromosomes in x, will not be included in calculations: {x_not_y}" ) else: logger.warning( f"Extra chromosomes in y, will not be included in calculations: {y_not_x}" ) metadata = correlate(x_cool, y_cool, xy_path=file_paths["xy"], coefficients_path=file_paths["coefficients"], resolution=resolution) metadata["resolution"] = resolution metadata["x"]["path"] = str(x_mcool) metadata["y"]["path"] = str(y_mcool) matrix_cat = catalogs.MatrixCorrelationCatalog.create( file_paths, metadata, {}) logger.info(str(matrix_cat))
def parse_cooler( cooler_file: str, regions: Dict[str, np.ndarray] ) -> Tuple[Cooler, List[np.ndarray]]: # Load cooler c = Cooler(cooler_file) # Fetch relevant bin_ids from the cooler file b_ids = fetch_bins_from_cooler(cooler, regions) # Identify unique bin_ids and isolate disjoint regions slices = get_unique_bins(b_ids) return c, slices
def preprocess_hic( clr: cooler.Cooler, min_contacts: Optional[int] = None, region: Optional[str] = None, ) -> sp.csr_matrix: """ Given an input cooler object, returns the preprocessed Hi-C matrix. Preprocessing involves (in that order): subsetting region, subsampling contacts, normalisation, detrending (obs / exp). Balancing weights must be pre-computer in the referenced cool file. Region must be in UCSC format. """ # Load raw matrix and subset region if requested mat = clr.matrix(sparse=True, balance=False) bins = clr.bins() if region is None: mat = mat[:] bins = bins[:] else: mat = mat.fetch(region) bins = bins.fetch(region) try: biases = bins["weight"].values except KeyError as err: sys.stderr.write("Error: Input cooler must be balanced.\n") raise err # get to same coverage if requested and matrix is not empty if mat.sum() and (min_contacts is not None): mat = cup.subsample_contacts(mat, min_contacts).tocoo() valid = cup.get_detectable_bins(mat, n_mads=5) # balance region with weights precomputed on the whole matrix mat.data = mat.data * biases[mat.row] * biases[mat.col] # Detrend for P(s) mat = cup.detrend(mat.tocsr(), smooth=False, detectable_bins=valid[0]) # Replace NaNs by 0s mat.data = np.nan_to_num(mat.data) mat.eliminate_zeros() return mat
def _aggregate(self, span): from cooler.api import Cooler lo, hi = span clr = Cooler(self.source_uri) # convert_enum=False returns chroms as raw ints table = clr.pixels(join=True, convert_enum=False) chunk = table[lo:hi] # logger.info('{} {}'.format(lo, hi)) print('{} {}'.format(lo, hi)) # use the "start" point as anchor for re-binning # XXX - alternatives: midpoint anchor, proportional re-binning binsize = self.gs.binsize chrom_binoffset = self.gs.chrom_binoffset chrom_abspos = self.gs.chrom_abspos start_abspos = self.gs.start_abspos chrom_id1 = chunk['chrom1'].values chrom_id2 = chunk['chrom2'].values start1 = chunk['start1'].values start2 = chunk['start2'].values if binsize is None: abs_start1 = chrom_abspos[chrom_id1] + start1 abs_start2 = chrom_abspos[chrom_id2] + start2 chunk['bin1_id'] = np.searchsorted( start_abspos, abs_start1, side='right') - 1 chunk['bin2_id'] = np.searchsorted( start_abspos, abs_start2, side='right') - 1 else: rel_bin1 = np.floor(start1 / binsize).astype(int) rel_bin2 = np.floor(start2 / binsize).astype(int) chunk['bin1_id'] = chrom_binoffset[chrom_id1] + rel_bin1 chunk['bin2_id'] = chrom_binoffset[chrom_id2] + rel_bin2 grouped = chunk.groupby(['bin1_id', 'bin2_id'], sort=False) return grouped['count'].sum().reset_index()
def main(): """Balance of plant of a boiling water nuclear reactor. Attributes ---------- end_time: float End of the flow time in SI unit. time_step: float Size of the time step between port communications in SI unit. use_mpi: bool If set to `True` use MPI otherwise use Python multiprocessing. """ # Preamble end_time = 30.0 * unit.minute time_step = 30.0 # seconds show_time = (True, 5 * unit.minute) use_mpi = False # True for MPI; False for Python multiprocessing plot_results = True # True for enabling plotting section below params = get_params() # parameters for BoP BWR #***************************************************************************** # Define Cortix system # System top level plant = Cortix(use_mpi=use_mpi, splash=True) # Network plant_net = plant.network = Network() params['start-time'] = 0.0 params['end-time'] = end_time params['shutdown-time'] = 999.0 * unit.hour params['shutdown-mode'] = False #***************************************************************************** # Create reactor module reactor = BWR(params) reactor.name = 'BWR' reactor.save = True reactor.time_step = time_step reactor.end_time = end_time reactor.show_time = show_time reactor.RCIS = True # Add reactor module to network plant_net.module(reactor) #***************************************************************************** # Create turbine high pressure module params['turbine_inlet_pressure'] = 2 params['turbine_outlet_pressure'] = 0.5 params['high_pressure_turbine'] = True #params_turbine = reactor.params #params_turbine.inlet_pressure = 2 #params.turbine_outlet_pressure = 0.5 turbine_hp = Turbine(params) turbine_hp.name = 'High Pressure Turbine' turbine_hp.save = True turbine_hp.time_step = time_step turbine_hp.end_time = end_time # Add turbine high pressure module to network plant_net.module(turbine_hp) #***************************************************************************** # Create turbine low pressure module params['turbine_inlet_pressure'] = 0.5 params['turbine_outlet_pressure'] = 0.005 params['high_pressure_turbine'] = False params['steam flowrate'] = params['steam flowrate'] / 2 turbine_lp1 = Turbine(params) turbine_lp1.name = 'Low Pressure Turbine 1' turbine_lp1.save = True turbine_lp1.time_step = time_step turbine_lp1.end_time = end_time plant_net.module(turbine_lp1) #***************************************************************************** # Create turbine low pressure module params['turbine_inlet_pressure'] = 0.5 params['turbine_outlet_pressure'] = 0.005 params['high_pressure_turbine'] = False turbine_lp2 = Turbine(params) turbine_lp2.name = 'Low Pressure Turbine 2' turbine_lp2.save = True turbine_lp2.time_step = time_step turbine_lp2.end_time = end_time plant_net.module(turbine_lp2) #***************************************************************************** # Create condenser module params['steam flowrate'] = params['steam flowrate'] * 2 condenser = Condenser() condenser.name = 'Condenser' condenser.save = True condenser.time_step = time_step condenser.end_time = end_time plant_net.module(condenser) #***************************************************************************** params['RCIS-shutdown-time'] = 5 * unit.minute rcis = Cooler(params) rcis.name = 'RCIS' rcis.save = True rcis.time_step = time_step rcis.end_time = end_time plant_net.module(rcis) #***************************************************************************** # Create the BoP network connectivity plant_net.connect([reactor, 'coolant-outflow'], [turbine_hp, 'inflow']) plant_net.connect([turbine_hp, 'outflow-1'], [turbine_lp1, 'inflow']) plant_net.connect([turbine_hp, 'outflow-2'], [turbine_lp2, 'inflow']) plant_net.connect([turbine_lp1, 'outflow-1'], [condenser, 'inflow-1']) plant_net.connect([turbine_lp2, 'outflow-1'], [condenser, 'inflow-2']) plant_net.connect([condenser, 'outflow'], [reactor, 'coolant-inflow']) plant_net.connect([reactor, 'RCIS-outflow'], [rcis, 'coolant-inflow']) plant_net.connect([rcis, 'coolant-outflow'], [reactor, 'RCIS-inflow']) #plant_net.connect([rcis, 'signal-in'], [reactor, 'signal-out']) plant_net.draw(engine='dot', node_shape='folder') #***************************************************************************** # Run network dynamics simulation plant.run() #***************************************************************************** # Plot results if plot_results and (plant.use_multiprocessing or plant.rank == 0): # Reactor plots reactor = plant_net.modules[0] (quant, time_unit ) = reactor.neutron_phase.get_quantity_history('neutron-dens') quant.plot(x_scaling=1 / unit.minute, x_label='Time [m]', y_label=quant.latex_name + ' [' + quant.unit + ']') plt.grid() plt.savefig('startup-neutron-dens.png', dpi=300) (quant, time_unit ) = reactor.neutron_phase.get_quantity_history('delayed-neutrons-cc') quant.plot(x_scaling=1 / unit.minute, x_label='Time [m]', y_label=quant.latex_name + ' [' + quant.unit + ']') plt.grid() plt.savefig('startup-delayed-neutrons-cc.png', dpi=300) (quant, time_unit ) = reactor.coolant_outflow_phase.get_quantity_history('temp') quant.plot(x_scaling=1 / unit.minute, x_label='Time [m]', y_label=quant.latex_name + ' [' + quant.unit + ']') plt.grid() plt.savefig('startup-coolant-outflow-temp.png', dpi=300) (quant, time_unit) = reactor.reactor_phase.get_quantity_history('fuel-temp') quant.plot(x_scaling=1 / unit.minute, x_label='Time [m]', y_label=quant.latex_name + ' [' + quant.unit + ']') plt.grid() plt.savefig('startup-fuel-temp.png', dpi=300) # Turbine high pressure plots turbine_hp = plant_net.modules[1] (quant, time_unit) = turbine_hp.outflow_phase.get_quantity_history('power') quant.plot(x_scaling=1 / unit.minute, x_label='Time [m]', y_label=quant.latex_name + ' [' + quant.unit + ']', title='High Pressure Turbine Power') plt.grid() plt.savefig('startup-turbine-hp-power.png', dpi=300) (quant, time_unit) = turbine_hp.outflow_phase.get_quantity_history('temp') quant.plot(x_scaling=1 / unit.minute, x_label='Time [m]', y_label=quant.latex_name + ' [' + quant.unit + ']', title='High Pressure Turbine Outflow Temperature') plt.grid() plt.savefig('startup-turbine-hp-outflow-temp.png', dpi=300) # Turbine low pressure graphs turbine_lp1 = plant_net.modules[2] (quant, time_unit) = turbine_lp1.outflow_phase.get_quantity_history('power') quant.plot(x_scaling=1 / unit.minute, x_label='Time [m]', y_label=quant.latex_name + ' [' + quant.unit + ']', title='Lower Pressure Turbine 1 Power') plt.grid() plt.savefig('startup-turbine-lp1-power.png', dpi=300) (quant, time_unit) = turbine_lp1.outflow_phase.get_quantity_history('temp') quant.plot(x_scaling=1 / unit.minute, x_label='Time [m]', y_label=quant.latex_name + ' [' + quant.unit + ']', title='Lower Pressure Turbine 1 Outflow Temperature') plt.grid() plt.savefig('startup-turbine-lp1-outflow-temp.png', dpi=300) # Condenser graphs condenser = plant_net.modules[3] (quant, time_unit) = condenser.outflow_phase.get_quantity_history('temp') quant.plot(x_scaling=1 / unit.minute, x_label='Time [m]', y_label=quant.latex_name + ' [' + quant.unit + ']') plt.grid() plt.savefig('startup-condenser-outflow-temp.png', dpi=300) #setup initial values for simulation turbine1_outflow_temp = turbine_hp.outflow_phase.get_value( 'temp', end_time) turbine1_chi = turbine_hp.outflow_phase.get_value('quality', end_time) turbine1_power = turbine_hp.outflow_phase.get_value('power', end_time) turbine2_outflow_temp = turbine_lp1.outflow_phase.get_value( 'temp', end_time) turbine2_chi = turbine_lp1.outflow_phase.get_value('quality', end_time) turbine2_power = turbine_lp1.outflow_phase.get_value('power', end_time) condenser_runoff_temp = condenser.outflow_phase.get_value('temp', end_time) delayed_neutron_cc = reactor.neutron_phase.get_value( 'delayed-neutrons-cc', end_time) n_dens = reactor.neutron_phase.get_value('neutron-dens', end_time) fuel_temp = reactor.reactor_phase.get_value('fuel-temp', end_time) coolant_temp = reactor.coolant_outflow_phase.get_value('temp', end_time) # Values loaded into params when they are needed (module instantiation) # Properly shutdown simulation plant.close() # Now we run shutdown as a seperate simulation with starting parameters equal to the ending # values of the startup simulation #************************************************************************************************** # Preamble start_time = 0.0 * unit.minute end_time = 60 * unit.minute time_step = 30.0 # seconds show_time = (True, 5 * unit.minute) use_mpi = False # True for MPI; False for Python multiprocessing plot_results = True # True for enabling plotting section below params = get_params() # clear params, just to be safe #***************************************************************************** # Define Cortix system # System top level plant = Cortix(use_mpi=use_mpi, splash=True) # Network plant_net = plant.network = Network() params['start-time'] = start_time params['end-time'] = end_time params['shutdown time'] = 0.0 params['shutdown-mode'] = True #***************************************************************************** # Create reactor module params['delayed-neutron-cc'] = delayed_neutron_cc params['n-dens'] = n_dens params['fuel-temp'] = fuel_temp params['coolant-temp'] = coolant_temp params['operating-mode'] = 'shutdown' reactor = BWR(params) reactor.name = 'BWR' reactor.save = True reactor.time_step = time_step reactor.end_time = end_time reactor.show_time = show_time reactor.RCIS = False # Add reactor module to network plant_net.module(reactor) #***************************************************************************** # Create turbine high pressure module params['turbine_inlet_pressure'] = 2 params['turbine_outlet_pressure'] = 0.5 params['high_pressure_turbine'] = True params['turbine-outflow-temp'] = turbine1_outflow_temp params['turbine-chi'] = turbine1_chi params['turbine-work'] = turbine1_power params['turbine-inflow-temp'] = coolant_temp #params_turbine = reactor.params #params_turbine.inlet_pressure = 2 #params.turbine_outlet_pressure = 0.5 turbine_hp = Turbine(params) turbine_hp.name = 'High Pressure Turbine' turbine_hp.save = True turbine_hp.time_step = time_step turbine_hp.end_time = end_time # Add turbine high pressure module to network plant_net.module(turbine_hp) #***************************************************************************** # Create turbine low pressure 1 module params['turbine_inlet_pressure'] = 0.5 params['turbine_outlet_pressure'] = 0.005 params['high_pressure_turbine'] = False params['steam flowrate'] = params['steam flowrate'] / 2 params['turbine-outflow-temp'] = turbine2_outflow_temp params['turbine-inflow-temp'] = turbine1_outflow_temp params['turbine-chi'] = turbine2_chi params['turbine-work'] = turbine2_power turbine_lp1 = Turbine(params) turbine_lp1.name = 'Low Pressure Turbine 1' turbine_lp1.save = True turbine_lp1.time_step = time_step turbine_lp1.end_time = end_time plant_net.module(turbine_lp1) #***************************************************************************** # Create turbine low pressure 2 module params['turbine_inlet_pressure'] = 0.5 params['turbine_outlet_pressure'] = 0.005 params['high_pressure_turbine'] = False turbine_lp2 = Turbine(params) turbine_lp2.name = 'Low Pressure Turbine 2' turbine_lp2.save = True turbine_lp2.time_step = time_step turbine_lp2.end_time = end_time plant_net.module(turbine_lp2) #***************************************************************************** # Create condenser module params['steam flowrate'] = params['steam flowrate'] * 2 params['condenser-runoff-temp'] = condenser_runoff_temp condenser = Condenser() condenser.name = 'Condenser' condenser.save = True condenser.time_step = time_step condenser.end_time = end_time plant_net.module(condenser) #***************************************************************************** params['RCIS-shutdown-time'] = -1 * unit.minute rcis = Cooler(params) rcis.name = 'RCIS' rcis.save = True rcis.time_step = time_step rcis.end_time = end_time plant_net.module(rcis) #***************************************************************************** # Create the BoP network connectivity plant_net.connect([reactor, 'coolant-outflow'], [turbine_hp, 'inflow']) plant_net.connect([turbine_hp, 'outflow-1'], [turbine_lp1, 'inflow']) plant_net.connect([turbine_hp, 'outflow-2'], [turbine_lp2, 'inflow']) plant_net.connect([turbine_lp1, 'outflow-1'], [condenser, 'inflow-1']) plant_net.connect([turbine_lp2, 'outflow-1'], [condenser, 'inflow-2']) plant_net.connect([condenser, 'outflow'], [reactor, 'coolant-inflow']) plant_net.connect([reactor, 'RCIS-outflow'], [rcis, 'coolant-inflow']) plant_net.connect([rcis, 'coolant-outflow'], [reactor, 'RCIS-inflow']) #plant_net.connect([rcis, 'signal-in'], [reactor, 'signal-out']) plant_net.draw(engine='dot', node_shape='folder') #***************************************************************************** # Run network dynamics simulation plant.run() #***************************************************************************** # Plot results if plot_results and (plant.use_multiprocessing or plant.rank == 0): # Reactor plots reactor = plant_net.modules[0] (quant, time_unit ) = reactor.neutron_phase.get_quantity_history('neutron-dens') quant.plot(x_scaling=1 / unit.minute, x_label='Time [m]', y_label=quant.latex_name + ' [' + quant.unit + ']') plt.grid() plt.savefig('shutdown-neutron-dens.png', dpi=300) (quant, time_unit ) = reactor.neutron_phase.get_quantity_history('delayed-neutrons-cc') quant.plot(x_scaling=1 / unit.minute, x_label='Time [m]', y_label=quant.latex_name + ' [' + quant.unit + ']') plt.grid() plt.savefig('shutdown-delayed-neutrons-cc.png', dpi=300) (quant, time_unit ) = reactor.coolant_outflow_phase.get_quantity_history('temp') quant.plot(x_scaling=1 / unit.minute, x_label='Time [m]', y_label=quant.latex_name + ' [' + quant.unit + ']') plt.grid() plt.savefig('shutdown-coolant-outflow-temp.png', dpi=300) (quant, time_unit) = reactor.reactor_phase.get_quantity_history('fuel-temp') quant.plot(x_scaling=1 / unit.minute, x_label='Time [m]', y_label=quant.latex_name + ' [' + quant.unit + ']') plt.grid() plt.savefig('shutdown-fuel-temp.png', dpi=300) # Turbine high pressure plots turbine_hp = plant_net.modules[1] (quant, time_unit) = turbine_hp.outflow_phase.get_quantity_history('power') quant.plot(x_scaling=1 / unit.minute, x_label='Time [m]', y_label=quant.latex_name + ' [' + quant.unit + ']', title='High Pressure Turbine Power') plt.grid() plt.savefig('shutdown-turbine-hp-power.png', dpi=300) (quant, time_unit) = turbine_hp.outflow_phase.get_quantity_history('temp') quant.plot(x_scaling=1 / unit.minute, x_label='Time [m]', y_label=quant.latex_name + ' [' + quant.unit + ']', title='High Pressure Turbine Outflow Temperature') plt.grid() plt.savefig('shutdown-turbine-hp-outflow-temp.png', dpi=300) # Turbine low pressure graphs turbine_lp1 = plant_net.modules[2] (quant, time_unit) = turbine_lp1.outflow_phase.get_quantity_history('power') quant.plot(x_scaling=1 / unit.minute, x_label='Time [m]', y_label=quant.latex_name + ' [' + quant.unit + ']', title='Lower Pressure Turbine 1 Power') plt.grid() plt.savefig('shutdown-turbine-lp1-power.png', dpi=300) (quant, time_unit) = turbine_lp1.outflow_phase.get_quantity_history('temp') quant.plot(x_scaling=1 / unit.minute, x_label='Time [m]', y_label=quant.latex_name + ' [' + quant.unit + ']', title='Lower Pressure Turbine 1 Outflow Temperature') plt.grid() plt.savefig('shutdown-turbine-lp1-outflow-temp.png', dpi=300) # Condenser graphs condenser = plant_net.modules[4] (quant, time_unit) = condenser.outflow_phase.get_quantity_history('temp') quant.plot(x_scaling=1 / unit.minute, x_label='Time [m]', y_label=quant.latex_name + ' [' + quant.unit + ']') plt.grid() plt.savefig('shutdown-condenser-outflow-temp.png', dpi=300) # Shutdown The Simulation plant.close()
#!/usr/bin/env python # encoding: utf-8 from cooler import Cooler if __name__ == '__main__': near_cool = Cooler() near_cool.run()
def _single_clr_edge_and_node_info_from_sites(c: cooler.Cooler, sites: Dict[str, np.ndarray], balance: Optional[bool] = True, join: Optional[bool] = False): ''' Given some cooler and a dictionary of sites (with chromosomes as keys), return the submatrices retrieved from these slices within the Hi-C map. Submatrices are returned in sparse COO format with an edge_idxs dictionary, an edge_attrs dictionary and a node_info dictionary. Optionally users can balance the Hi-C matrix before retrieval of matrix information. SInce multiple chromosomes and slices per chromosome can be supplied, user may optionally join regions into one larger region consisting of the given slices concatenated together. This function does not actually do the joining procedure since the passed slices may not be disjoint. :param cooler: Cooler file object :type edge_index: cooler.Cooler :param slices: Dictionary with chromosomes as keys and lists of sites as values. Multiple sites are allowed per chromosome. :type slices: Dict[str,List[np.ndarray]] :param balance: Whether to perform matrix balancing on the Hi-C matrix before retrieving individual slices. :type balance: Optional[bool] :param join: Boolean determining whether to retrieve Hi-C martrix information corresponding to the interface between slices. This is only recommended if slices are disjoint since the interface isn't well defined if slices aren't disjoint. :type join: Optional[bool] ''' # Iterate through slices, adding in edge indexes and edge attributes edge_idxs = {} edge_attrs = {} sub_graph_nodes = {} chroms = list(sites.keys()) for idx, chrom1 in enumerate(chroms): edge_idxs[chrom1] = {} edge_attrs[chrom1] = {} sub_graph_nodes[chrom1] = {} for chrom2 in chroms[idx:]: if chrom1 != chrom2 and not join: continue mat = c.matrix(balance=balance).fetch(chrom1, chrom2) mat = mat[sites[chrom1], :] mat = mat[:, sites[chrom2]] mat = coo(mat) b1 = c.bins().fetch(chrom1).index.values[sites[chrom1]] b2 = c.bins().fetch(chrom2).index.values[sites[chrom2]] edge_index = np.concatenate( [b1[mat.row][None, :], b2[mat.col][None, :]], axis=0, ) edge_data = mat.data[:, None] if chrom1 != chrom2: edge_index = np.append(edge_index, edge_index[::-1, :], axis=1) edge_data = np.append(edge_data, edge_data, axis=0) edge_data[np.isnan(edge_data)] = 0 ind = np.lexsort((edge_index[0, :], edge_index[1, :])) edge_index = edge_index[:, ind] edge_data = edge_data[ind, :] edge_idxs[chrom1][chrom2] = [edge_index] edge_attrs[chrom1][chrom2] = [edge_data] if chrom1 == chrom2: sub_graph_nodes[chrom1][chrom2] = [b1] else: sub_graph_nodes[chrom1][chrom2] = [np.append(b1, b2)] return edge_idxs, edge_attrs, sub_graph_nodes
def _single_clr_edge_and_node_info_from_slices(c: cooler.Cooler, slices: Dict[str, List[np.ndarray]], balance: Optional[bool] = True, join: Optional[bool] = False): ''' Given some cooler and a dictionary of slices (with chromosomes as keys), return the submatrices retrieved from these slices within the Hi-C map. Submatrices are returned in sparse COO format with an edge_idxs dictionary, an edge_attrs dictionary and a node_info dictionary. Optionally users can balance the Hi-C matrix before retrieval of matrix information. SInce multiple chromosomes and slices per chromosome can be supplied, user may optionally join regions into one larger region consisting of the given slices concatenated together. This function does not actually do the joining procedure since the passed slices may not be disjoint. :param cooler: Cooler file object :type edge_index: cooler.Cooler :param slices: Dictionary with chromosomes as keys and lists of slices as values. Multiple slices are allowed per chromosome. :type slices: Dict[str,List[np.ndarray]] :param balance: Whether to perform matrix balancing on the Hi-C matrix before retrieving individual slices. :type balance: Optional[bool] :param join: Boolean determining whether to retrieve Hi-C martrix information corresponding to the interface between slices. This is only recommended if slices are disjoint since the interface isn't well defined if slices aren't disjoint. :type join: Optional[bool] ''' # Iterate through slices, adding in edge indexes and edge attributes edge_idxs = {} edge_attrs = {} sub_graph_nodes = {} chroms = list(slices.keys()) for cidx1, chrom1 in enumerate(chroms): edge_idxs[chrom1] = {} edge_attrs[chrom1] = {} sub_graph_nodes[chrom1] = {} for chrom2 in chroms[cidx1:]: if chrom1 != chrom2 and not join: continue edge_idxs[chrom1][chrom2] = [] edge_attrs[chrom1][chrom2] = [] sub_graph_nodes[chrom1][chrom2] = [] for idx, s1 in enumerate(slices[chrom1]): if chrom1 == chrom2: #don't want to repeat region pairings slist = slices[chrom1][idx:] else: slist = slices[chrom2] for jdx, s2 in enumerate(slist): if s1[0] == s2[0] and jdx != 0: continue if s1[0] != s2[0] and not join: continue mat = c.matrix(balance=balance, sparse=True)[s1[0]:s1[-1] + 1, s2[0]:s2[-1] + 1] edge_index = np.concatenate( [s1[mat.row][None, :], s2[mat.col][None, :]], axis=0, ) edge_data = mat.data[:, None] if np.sum(s1 - s2) != 0: edge_index, edge_data = make_edges_bidirectional( edge_index, edge_data) ind = np.lexsort((edge_index[0, :], edge_index[1, :])) edge_index = edge_index[:, ind] edge_data = edge_data[ind, :] edge_idxs[chrom1][chrom2].append(edge_index) edge_attrs[chrom1][chrom2].append(edge_data) if s1[0] == s2[0]: sub_graph_nodes[chrom1][chrom2].append(s1) else: sub_graph_nodes[chrom1][chrom2].append( np.append(s1, s2)) return edge_idxs, edge_attrs, sub_graph_nodes
def export_to_cooler( contact_table, output_prefix, cooler_resolution, fragment_table, chromsizes, query, query_columns=None, by_haplotype=False, ): results = [] if query_columns: columns = query_columns[:] else: columns = [] columns.extend(["align1_fragment_id", "align2_fragment_id"]) if by_haplotype: columns.extend(["align1_haplotype", "align2_haplotype"]) contact_df = dd.read_parquet(contact_table, engine=PQ_ENGINE, version=PQ_VERSION, columns=columns, index=False) if query: contact_df = contact_df.query(query) chrom_dict = pd.read_csv(chromsizes, sep="\t", header=None, names=["chrom", "size"], index_col=["chrom"], squeeze=True) # create even-widht bins using cooler bins_df = binnify(chrom_dict, cooler_resolution) bins_df.index.name = "bin_id" # convert to ranges for overlap bins = pr.PyRanges(bins_df.reset_index().rename(columns={ "start": "Start", "end": "End", "chrom": "Chromosome" })) fragment_df = dd.read_parquet(fragment_table, engine=PQ_ENGINE, version=PQ_VERSION).compute() midpoint_df = pr.PyRanges( fragment_df.reset_index()[[ "chrom", "start", "end", "fragment_id" ]].assign(start=lambda x: ((x.start + x.end) * 0.5).round(0).astype( int)).eval("end = start + 1").rename(columns={ "chrom": "Chromosome", "start": "Start", "end": "End" })) # use a pyranges joing to assign fragments to bins fragment_to_bin = midpoint_df.join( bins, how="left").df[["fragment_id", "bin_id"]] fragment_to_bin = fragment_to_bin.set_index( "fragment_id").sort_index() # .astype(np.uint32) nulls = fragment_to_bin["bin_id"] == -1 if nulls.any(): logger.warning( "Some fragments did not overlap bins, removing from analysis:\n{}". format(fragment_to_bin[nulls].join(fragment_df))) fragment_to_bin = fragment_to_bin[~nulls] # use a join to assign each end of a contact to a bin binned_contacts = (contact_df.merge( fragment_to_bin, how="inner", right_index=True, left_on="align1_fragment_id").merge( fragment_to_bin, how="inner", right_index=True, left_on="align2_fragment_id", suffixes=[None, "_2"]).rename(columns={ "bin_id": "bin1_id", "bin_id_2": "bin2_id" })) if not by_haplotype: cooler_path = output_prefix + ".cool" # group size == number of contacts per bin_pair pixels = binned_contacts.groupby( ["bin1_id", "bin2_id"]).size().rename("count").astype(np.int32).reset_index() create_cooler(cooler_path, bins_df, pixels, ordered=True, symmetric_upper=True, ensure_sorted=True) c = Cooler(cooler_path) logger.info(f"Created cooler: {c.info}") results.append(cooler_path) else: tmp_parquet = output_prefix + ".tmp.pq" pixels = ( # create a key to groupy by haplotype pair, order of haplotypes doesn't matter binned_contacts.assign( hap_key=lambda x: x[["align1_haplotype", "align2_haplotype"] ].apply(lambda y: "{}_{}".format(*sorted( y)).replace("-1", "nohap"), axis=1, meta="object") ).groupby(["hap_key", "bin1_id", "bin2_id"]).size().rename("count").astype( np.int32 ).reset_index().astype({"hap_key": "category"})) # save to a temporary parquet file, this might not be necessary # but want to avoid the whole contact matrix hitting memory pixels.to_parquet( tmp_parquet, write_metadata_file=True, partition_on=["hap_key"], write_index=False, engine=PQ_ENGINE, version=PQ_VERSION, ) pixels = dd.read_parquet(tmp_parquet, engine=PQ_ENGINE, version=PQ_VERSION, columns=["hap_key"], index=False) hap_keys = pixels["hap_key"].unique().compute() # create a cooler for each haplotype pair for hap_key in hap_keys: cooler_path = f"{output_prefix}.{hap_key}.cool" pixels = dd.read_parquet( tmp_parquet, filters=[("hap_key", "==", hap_key)], index=False, engine=PQ_ENGINE, version=PQ_VERSION, columns=["bin1_id", "bin2_id", "count"], ) create_cooler(cooler_path, bins_df, pixels, ordered=True, symmetric_upper=True, ensure_sorted=True) c = Cooler(cooler_path) logger.info(f"Created cooler: {c.info}") results.append(cooler_path) shutil.rmtree(tmp_parquet) return results
from cooler import Cooler # input_uri = "unzoomifiable_5kb.cool" input_uri = "missing_bin_cooler_40kb.cool" clr = Cooler(input_uri) print(clr.pixels(join=True)[17872226]) table = clr.pixels(join=True)[17800000:] print(table[table['chrom2'].isnull()]) # chrom1 start1 end1 chrom2 start2 end2 count # 17872226 chr88 2640000 2680000 NaN NaN NaN 1 ######################## # TODO ######################### # Now i'm on the same page with Nezar # bad pixel is found ... # FIND OUT - HOW that happened ?!?!?!?!?! ... # # processes that might be involved: # # pairix ${pairs_lib} # # cooler cload pairix \ # --nproc ${task.cpus} \ # --assembly ${params.input.genome.assembly} \
class T_control: def __init__(self, fan_pin, peltier_pin, step_pin, thermometer_pin): self.cooler = Cooler(Pin(fan_pin, Pin.OUT), Pin(peltier_pin, Pin.OUT)) self.cooler.fanOn() self.cooler.coolerHigh() self.pid = Pid(17, 1, 0.02, 0, 0.995) #(temperature, P, I, D, memoryFactor) self.thermometer = Thermometer(thermometer_pin) self.pump = PWMPump(Pin(step_pin, Pin.OUT)) for i in range(10): self.pump.pwm.freq(i * 1500) utime.sleep(0.1) self.pump.pwm.freq(15000) #self.pump = VariablePump(Pin(step_pin,Pin.OUT)) #self.pump.setSpeed(0.90) #self.pump.startMotor() def call(self): self.t = utime.time() read = -self.pid.update(self.thermometer.read()) if read > 2: self.cooler.coolerHigh() self.pump.pwm.freq(10000) elif read >= 1: self.cooler.coolerLow() self.pump.pwm.freq(10000) elif read >= -1: self.cooler.coolerLow() self.pump.pwm.freq(6666 + int(3333 * read)) else: self.cooler.coolerLow() self.pump.pwm.freq(200) def __loop(self): while self.isRunning: self.call() utime.sleep(10) def startPID(self): self.isRunning = True _thread.start_new_thread(self.__loop, ()) def stopPID(self): self.isRunning = False
def get_pairing_score_obs_exp( clr: cooler.Cooler, expected: pd.DataFrame, windowsize: int = 4 * 10**4, func: Callable = np.mean, regions: pd.DataFrame = pd.DataFrame(), norm: bool = True, arms: pd.DataFrame = pd.DataFrame(), ) -> pd.DataFrame: """Takes a cooler file (clr), an expected dataframe (expected; maybe generated by getExpected), a windowsize (windowsize), a summary function (func) and a set of genomic regions to calculate the pairing score as follows: A square with side-length windowsize is created for each of the entries in the supplied genomics regions and the summary function applied to the Hi-C pixels (obs/exp values) at the location in the supplied cooler file. The results are returned as a dataframe. If no regions are supplied, regions are constructed for each bin in the cooler file to construct a genome-wide pairing score.""" # Check whether genomic regions were supplied if len(regions) == 0: # If no regions are supplied, pregenerate all bins; drop bins with nan weights regions = clr.bins()[:].dropna() # find midpoint of each bin to assign windows to each midpoint regions.loc[:, "mid"] = (regions["start"] + regions["end"]) // 2 # check that norm is only set if genomewide pairingScore is calculated elif norm: raise ValueError( "Norm flag can only be set with genomeWide pairingScore!") # drop nan rows from regions regions = regions.dropna() # fix indices regions.index = range(len(regions)) regions.loc[:, "binID"] = range(len(regions)) # Chromosomal arms are needed so each process only extracts a subset from the file if len(arms) == 0: arms = get_arms_hg19() # extract all windows windows = assign_regions(windowsize, clr.binsize, regions["chrom"], regions["mid"], arms) # add binID to later merge piles windows.loc[:, "binID"] = regions["binID"] windows = windows.dropna() # generate pileup pile = do_pileup_obs_exp(clr, expected, windows, collapse=False) # convert to dataframe pile_frame = pile_to_frame(pile) # replace inf with nan pile_frame = pile_frame.replace([np.inf, -np.inf], np.nan) # apply function to each row (row = individual window) summarized = pile_frame.apply(func, axis=1) # subset regions with regions that were assigned windows output = pd.merge(regions, windows, on="binID", suffixes=("", "_w")).dropna() # add results output.loc[:, "PairingScore"] = summarized # normalize by median if norm: output.loc[:, "PairingScore"] = output["PairingScore"] - np.median( output.dropna()["PairingScore"]) return output[["chrom", "start", "end", "PairingScore"]]
for i in range(0, len(spans), batchsize): try: lock.acquire() print("right before collapse ...{} {}".format( i, spans[i:i + batchsize])) results = self._map(self.aggregate, spans[i:i + batchsize]) finally: lock.release() for df in results: # yield {k: v.values for k, v in six.iteritems(df)} yield df input_uri = "" c = Cooler(input_uri) new_bins = binnify(c.chromsizes, 2 * c.binsize) iterator = CoolerAggregator(input_uri, new_bins, 1000000, batchsize=1, map=map) # # last message before it fails ... # # INFO:cooler:17868809 17872380 # for ii in iterator: # print(ii) # from cooler.api import Cooler lo, hi = 17869999, 17872300 # lo, hi = 17868809, 17872380 clr = Cooler(input_uri)
from flask import Flask, redirect, url_for, render_template, request from cooler import Cooler # globals app = Flask(__name__) cooler = Cooler(21, 20, 16) @app.route('/', methods=('GET', 'POST')) def index(): if request.method == 'GET': return render_template('index.html', **cooler.__dict__) if request.method == 'POST': cooler.control_mode = request.form['mode'] if request.form['mode'] == 'automatic': # TODO: check rationality if thresholds try: cooler.min_threshold = float(request.form['min_threshold']) except Exception as E: pass try: cooler.max_threshold = float(request.form['max_threshold']) except Exception as E: pass cooler.set_speed(False) return redirect('/update') if request.form['mode'] == 'manual':
#!/usr/bin/env python # encoding: utf-8 from cooler import Cooler if __name__ == '__main__': near_cool = Cooler() near_cool.run()