コード例 #1
0
ファイル: modeling_step.py プロジェクト: davidzhan1995/igm
def modeling_step(model, cfg):

    with HssFile(cfg['mstep']['input_hss'], 'r') as f:
        radii = f.radii
        index = f.index
        genome = f.genome

    n_struct = cfg['mstep']['n_struct']
    n_beads = cfg['mstep']['n_beads']

    basepath = os.path.join(cfg['mstep']['workdir'], cfg['mstep']['run_name'])
    cfg['mstep']['templates'] = {
        'crd_out': basepath + '.outcrd.{}.npy',
        'info_out': basepath + '.info.{}.txt'
    }
    cfg.save(basepath + '.config')

    serial_function = partial(modeling_task, cfg_file=basepath + '.config')
    pctype = cfg['parallel_controller']
    pcopts = cfg['parallel_controller_options']
    controller = controller_class[pctype](**pcopts)
    argument_list = list(range(n_struct))

    controller.map(serial_function, argument_list)

    # write coordinates
    crd_shape = (n_beads, n_struct, 3)
    with HssFile(cfg['mstep']['output_hss'], 'w') as hss:
        hss.index = index
        hss.genome = genome
        hss.radii = radii
        all_crd = hss.create_dataset('coordinates',
                                     shape=crd_shape,
                                     dtype=COORD_DTYPE)
        for i in range(n_struct):
            local_vars = resolve_templates(cfg['mstep']['templates'], [i])
            crd = np.load(local_vars['crd_out'])
            all_crd[:,
                    i, :] = crd  # note that we discard all the positions of added atoms

        # write info
        kernel = kernel_class[cfg['mstep']['kernel']]
        with open(cfg['mstep']['info_out'], 'w') as outf:
            outf.write('#')
            for k in kernel.INFO_KEYS:
                outf.write(k + '\t')
            outf.write('\n')
            for i in range(n_struct):
                local_vars = resolve_templates(cfg['mstep']['templates'], [i])
                with open(local_vars['info_out']) as inf:
                    outf.write(inf.read() + '\n')

    # cleanup
    for i in range(n_struct):
        local_vars = resolve_templates(cfg['mstep']['templates'], [i])
        os.remove(local_vars['crd_out'])
        os.remove(local_vars['info_out'])
コード例 #2
0
    def setup(self):

        opt = self.cfg['restraints']['sprite']

        self.tmp_extensions = [".npy", ".npz"]

        self.tmp_dir = make_absolute_path(
            self.cfg.get('restraints/sprite/tmp_dir', 'sprite'),
            self.cfg.get('parameters/tmp_dir'))

        self.keep_temporary_files = self.cfg.get(
            'restraints/sprite/keep_temporary_files', False)

        if not os.path.isdir(self.tmp_dir):
            os.makedirs(self.tmp_dir)
        #---

        clusters_file = self.cfg.get('restraints/sprite/clusters')
        with h5py.File(clusters_file, 'r') as h5:
            n_clusters = len(h5['indptr']) - 1

        with HssFile(self.cfg.get("optimization/structure_output"),
                     'r') as hss:
            self.n_struct = hss.nstruct

        batch_size = self.cfg.get('restraints/sprite/batch_size', 10)
        n_batches = n_clusters // batch_size
        if n_clusters % batch_size != 0:
            n_batches += 1

        self.n_batches = n_batches
        self.n_clusters = n_clusters
        self.argument_list = range(n_batches)
コード例 #3
0
ファイル: _preprocess.py プロジェクト: bonimba87/igm
def prepareHss(fname,
               nbead,
               nstruct,
               genome,
               index,
               radii,
               nucleus_shape='sphere',
               nucleus_parameters=5000.0,
               nucleus_volume=0,
               coord_chunks=None):
    with HssFile(fname, 'w') as hss:
        #put everything into hssFile
        hss.set_nbead(nbead)
        hss.set_nstruct(nstruct)
        hss.set_genome(genome)
        hss.set_index(index)
        hss.set_radii(radii)
        if coord_chunks is None:
            hss.set_coordinates(np.zeros((nbead, nstruct, 3)))
        else:
            hss.create_dataset('coordinates',
                               shape=(nbead, nstruct, 3),
                               dtype=COORD_DTYPE,
                               chunks=coord_chunks)
        env = hss.create_group('envelope')
        env.create_dataset('shape', data=nucleus_shape)
        env.create_dataset('volume', data=nucleus_volume)
        env.create_dataset('params', data=nucleus_parameters)
コード例 #4
0
    def task(batch_id, cfg, tmp_dir):
        """ Compute activation distances for batch identified by parameter batch_id """

        dictHiC = cfg['restraints']['Hi-C']
        hss = HssFile(cfg.get("optimization/structure_output"), 'r')

        # read params
        fname = os.path.join(tmp_dir, '%d.in.npy' % batch_id)
        params = np.load(fname)

        # initialize result list
        results = []

        # compute activation distances for all pairs of locus indexes, append to "results" list
        for i, j, pwish, plast in params:
            res = get_actdist(int(i),
                              int(j),
                              pwish,
                              plast,
                              hss,
                              contactRange=dictHiC.get('contact_range', 2.0))

            for r in res:
                results.append(r)  # (i, j, actdist, p)
            # -

        hss.close()

        # save activation distances from current batch to a batch-unique output file, using format specifier 'actdist_fmt_str'
        fname = os.path.join(tmp_dir, '%d.out.tmp' % batch_id)
        with open(fname, 'w') as f:
            f.write('\n'.join([actdist_fmt_str % x for x in results]))
コード例 #5
0
ファイル: ModelingStep.py プロジェクト: davidzhan1995/igm
    def teardown_poller(self):
        """ Teardown poller, after all coodinates and statistics have been updated for the whole population """

        total_violations, total_restraints = self._summary_data[
            'n_violations'], self._summary_data['n_imposed']

        if total_restraints == 0:
            violation_score = 0
        else:
            # percentage of restraint violations
            violation_score = total_violations / total_restraints

        # open HSS population file
        self._hss = HssFile(self.hssfilename, 'r+')  # LB

        # store violation score, all the coordinates and the restraint violation summary into the HSS file
        self._hss.set_violation(violation_score)
        self._hss.set_coordinates(self._hss_crd)  # LB
        h5_create_or_replace_dataset(self._hss,
                                     'summary',
                                     data=json.dumps(
                                         self._summary_data,
                                         default=lambda a: a.tolist()))

        # close HSS file
        self._hss.close()
コード例 #6
0
ファイル: step.py プロジェクト: davidzhan1995/igm
    def reduce(self):
        """
        Collect all structure coordinates together to assemble a hssFile
        """

        hssfilename = self.cfg["optimization"]["structure_output"] + '.T'

        # bonimba: using changes as Nan
        with HssFile(hssfilename, 'r+') as hss:
            n_struct = hss.nstruct
            n_beads = hss.nbead
            #iterate all structure files and
            total_restraints = 0.0
            total_violations = 0.0

            # extract coordinates and put them in matrix
            master = hss.coordinates
            print('Collecting all the coordinates from all configurations....')

            for i in tqdm(range(hss.nstruct), desc='(REDUCE)'):
                fname = "{}_{}.hms".format(self.tmp_file_prefix, i)
                hms = HmsFile(os.path.join(self.tmp_dir, fname))
                crd = hms.get_coordinates()
                total_restraints += hms.get_total_restraints()
                total_violations += hms.get_total_violations()

                # edit master numpy matrix...
                master[:, i, :] = crd

            # in un colpo solo, chiudi il fil
            hss.set_coordinates(master)
            #-
            if (total_violations == 0) and (total_restraints == 0):
                hss.set_violation(np.nan)
            else:
                hss.set_violation(total_violations / total_restraints)

        hss.close()

        # repack
        PACK_SIZE = 1e6
        pack_beads = max(1, int(PACK_SIZE / n_struct / 3))
        pack_beads = min(pack_beads, n_beads)

        logger.info('repacking...')
        cmd = 'h5repack -l coordinates:CHUNK={:d}x{:d}x3 {:s} {:s}'.format(
            pack_beads, n_struct, hssfilename, hssfilename + '.swap')
        os.system(cmd)
        logger.info('done.')
        os.rename(hssfilename + '.swap',
                  self.cfg.get("optimization/structure_output"))

        if self.keep_intermediate_structures:
            copyfile(self.cfg["optimization"]["structure_output"],
                     self.intermediate_name())
コード例 #7
0
ファイル: ModelingStep.py プロジェクト: davidzhan1995/igm
    def reduce(self):
        """
        Collect all structure coordinates together to assemble a hssFile
        """

        # wait for poller to finish (see also RelaxInit.py script)
        for _ in tqdm(self.file_poller.enumerate(), desc='(REDUCE)'):
            pass

        # read and log details, and save the runtime variables
        with HssFile(self.hssfilename, 'r+') as hss:
            n_struct = hss.nstruct
            n_beads = hss.nbead
            violation_score = log_stats(hss, self.cfg)
            self.cfg['runtime']['violation_score'] = violation_score
            h5_create_or_replace_dataset(hss,
                                         'config_data',
                                         data=json.dumps(
                                             self.cfg,
                                             default=lambda a: a.tolist()))

        # repack hss file
        PACK_SIZE = 1e6
        pack_beads = max(1, int(PACK_SIZE / n_struct / 3))
        pack_beads = min(pack_beads, n_beads)
        cmd = [
            'h5repack', '-i', self.hssfilename, '-o',
            self.hssfilename + '.swap', '-l',
            'coordinates:CHUNK={:d}x{:d}x3'.format(pack_beads, n_struct), '-v'
        ]

        sp = Popen(cmd, stderr=PIPE, stdout=PIPE)
        logger.info('repacking...')
        stdout, stderr = sp.communicate()
        if sp.returncode != 0:
            print(' '.join(cmd))
            print('O:', stdout.decode('utf-8'))
            print('E:', stderr.decode('utf-8'))
            raise RuntimeError('repacking failed. error code: %d' %
                               sp.returncode)
        logger.info('done.')

        # save the output file with a unique file name if requested (see 'intermediate_name' function below)
        if self.keep_intermediate_structures:
            copyfile(self.hssfilename + '.swap',
                     self.intermediate_name() + '.hss')

        # finally replace output file
        shutil.move(self.hssfilename + '.swap',
                    self.cfg.get("optimization/structure_output"))
コード例 #8
0
ファイル: RelaxInit.py プロジェクト: davidzhan1995/igm
    def reduce(self):
        """
        Collect all structure coordinates together to assemble a hssFile, using the polling function, and repack
        """

        # update structure coordinates
        for i in tqdm(self.file_poller.enumerate(), desc='(REDUCE)'):
            pass

        with HssFile(self.hssfilename, 'r+') as hss:
            n_struct = hss.nstruct
            n_beads = hss.nbead

        logger.info(
            'Coordinates in master file updated for ALL structures; repacking starts...'
        )

        # repack hss file (this is a syntax proper to h5df files)
        PACK_SIZE = 1e6
        pack_beads = max(1, int(PACK_SIZE / n_struct / 3))
        pack_beads = min(pack_beads, n_beads)
        cmd = [
            'h5repack', '-i', self.hssfilename, '-o',
            self.hssfilename + '.swap', '-l',
            'coordinates:CHUNK={:d}x{:d}x3'.format(pack_beads, n_struct), '-v'
        ]

        sp = Popen(cmd, stderr=PIPE, stdout=PIPE)
        logger.info('repacking...')
        stdout, stderr = sp.communicate()
        if sp.returncode != 0:
            print(' '.join(cmd))
            print('O:', stdout.decode('utf-8'))
            print('E:', stderr.decode('utf-8'))
            raise RuntimeError('repacking failed. error code: %d' %
                               sp.returncode)
        logger.info('repacking done.')

        # save the output file with a unique file name if requested
        if self.keep_intermediate_structures:
            copyfile(self.hssfilename + '.swap',
                     self.intermediate_name() + '.hss')

        # get rid of temporary .swap file
        os.rename(self.hssfilename + '.swap',
                  self.cfg.get("optimization/structure_output"))
コード例 #9
0
ファイル: igm_test.py プロジェクト: davidzhan1995/igm
def checkViolations(cfg):
    hss = HssFile(cfg["structure_output"])
    vio = hss.get_violation()
    if "sigma" in cfg["restraints"]["Hi-C"]:
        print(cfg["restraints"]["Hi-C"]["sigma"], vio)
        if vio < 0.01:
            if len(cfg["restraints"]["Hi-C"]["sigma_list"]) > 0:
                cfg["restraints"]["Hi-C"]["sigma"] = cfg["restraints"]["Hi-C"][
                    "sigma_list"].pop(0)
            else:
                return False
    else:
        print("Start", vio)
        cfg["restraints"]["Hi-C"]["sigma"] = cfg["restraints"]["Hi-C"][
            "sigma_list"].pop(0)
    #-
    return True
コード例 #10
0
    def task(struct_id, cfg, tmp_dir):
        """
        generate one random structure with territories
        """
        k = np.random.randint(0, 2**32)
        np.random.seed((k * struct_id) % (2**32))
        hssfilename = cfg["optimization"]["structure_output"]
        nucleus_radius = cfg.get("model/init_radius")

        with HssFile(hssfilename, 'r') as hss:
            index = hss.index

        crd = generate_territories(index, nucleus_radius)

        ofname = os.path.join(tmp_dir, 'random_%d.hms' % struct_id)
        with HmsFile(ofname, 'w') as hms:
            hms.saveCoordinates(struct_id, crd)
コード例 #11
0
ファイル: ModelingStep.py プロジェクト: davidzhan1995/igm
    def setup_poller(self):
        """ Set up polling function: define coordinate master matrix and a dictionary summarizing statistics from run"""

        _hss = HssFile(self.hssfilename, 'r')
        #self._hss = HssFile(self.hssfilename, 'r+')

        self._hss_crd = _hss.coordinates
        _hss.close()

        self._summary_data = {
            'n_imposed': 0.0,
            'n_violations': 0.0,
            'histogram': {
                'counts':
                np.zeros(DEFAULT_HIST_BINS + 1),
                'edges':
                np.arange(0, DEFAULT_HIST_MAX,
                          DEFAULT_HIST_MAX / DEFAULT_HIST_BINS).tolist() +
                [DEFAULT_HIST_MAX, np.inf]
            },
            'bystructure': {
                'n_imposed':
                np.zeros(self.cfg["model"]["population_size"],
                         dtype=np.float32),
                'n_violations':
                np.zeros(self.cfg["model"]["population_size"],
                         dtype=np.float32),
                'total_energies':
                np.zeros(self.cfg["model"]["population_size"],
                         dtype=np.float32),
                'pair_energies':
                np.zeros(self.cfg["model"]["population_size"],
                         dtype=np.float32),
                'bond_energies':
                np.zeros(self.cfg["model"]["population_size"],
                         dtype=np.float32),
                'thermo': {}
            },
            'byrestraint': {}
        }
コード例 #12
0
    def task(batch_id, cfg, tmp_dir):
        """ Read in temporary in.tmp files, generated list of Damid activation distances, produce out.tmp files """

        nucleus_parameters = None
        shape = cfg.get('model/restraints/envelope/nucleus_shape')
        if shape == 'sphere':
            nucleus_parameters = cfg.get(
                'model/restraints/envelope/nucleus_radius')
        elif shape == 'ellipsoid':
            nucleus_parameters = cfg.get(
                'model/restraints/envelope/nucleus_semiaxes')
        else:
            raise NotImplementedError(
                'shape %s has not been implemented yet.' % shape)

        with HssFile(cfg.get("optimization/structure_output"), 'r') as hss:

            # read params from temporary damid.in.npy files
            fname = os.path.join(tmp_dir, '%d.damid.in.npy' % batch_id)
            params = np.load(fname)

            # compute the corresponding output to save to out.tmp file
            results = []
            for i, pwish, plast in params:
                res = get_damid_actdist(int(i),
                                        pwish,
                                        plast,
                                        hss,
                                        contact_range=cfg.get(
                                            'restraints/DamID/contact_range',
                                            0.05),
                                        shape=shape,
                                        nucleus_param=nucleus_parameters)
                results += res  #(i, damid_actdist, p)
            #-

        # save output for this chunk to file, using the format specified by string 'damid_actdist_fmt_str'
        fname = os.path.join(tmp_dir, '%d.out.tmp' % batch_id)
        with open(fname, 'w') as f:
            f.write('\n'.join([damid_actdist_fmt_str % x for x in results]))
コード例 #13
0
def get_structure(path, n, folder='.'):
    path = os.path.join(folder, path)

    with HssFile(path, 'r') as f:
        crd = f.get_struct_crd(n).tolist()
        chrom = f.genome.chroms[f.index.chrom].tolist()
        radius = f.radii.tolist()
        nstruct = f.nstruct
        cstarts = f.index.offset.tolist()

    return {
        'crd':
        [crd[cstarts[i]:cstarts[i + 1]] for i in range(len(cstarts) - 1)],
        'idx':
        chrom,
        'rad':
        [radius[cstarts[i]:cstarts[i + 1]] for i in range(len(cstarts) - 1)],
        'n':
        int(nstruct),
        'cstarts':
        cstarts,
        'chroms':
        [str(v) for i, v in enumerate(chrom) if i == 0 or v != chrom[i - 1]],
    }
コード例 #14
0
    def task(batch_id, cfg, tmp_dir):

        clusters_file = cfg.get('restraints/sprite/clusters')
        batch_size = cfg.get('restraints/sprite/batch_size', 10)
        keep_best = cfg.get('restraints/sprite/keep_best', 50)

        # read the clusters
        with h5py.File(clusters_file, 'r') as h5:
            start = batch_id * batch_size
            stop = (batch_id + 1) * batch_size + 1
            ii = h5['indptr'][start:stop][()]
            data = h5['data'][ii[0]:ii[-1]]  #load everything for performance
            ii -= ii[0]  # subtract offset (ii[0]) from the full ii array

            # generate a list of arrays of 'stop-start' length
            clusters = [data[ii[i - 1]:ii[i]] for i in range(1, len(ii))]
            del data

        # open the structure file and read index
        hss = HssFile(cfg.get("optimization/structure_output"), 'r')
        index = hss.index

        indexes, values, selected_beads = [], [], []

        for cluster in clusters:

            # effective number of different chromosomes involved in current cluster (no repetition)
            n_chrom = len(np.unique(index.chrom[cluster]))

            # if max_chrom_in_cluster exceeeded, then append arrays full of -1 entries
            if n_chrom > cfg.get('restraints/sprite/max_chrom_in_cluster', 6):
                selected_beads.append(
                    np.zeros((keep_best, len(cluster)), dtype='i4') -
                    1  # matrix (keep_best, len(cluster)) of -1s
                )
                indexes.append(np.array(
                    [-1] * keep_best))  # array with "keep_best" -1 entries
                values.append(np.array(
                    [-1] * keep_best))  # array with "keep_best" -1 entries
                continue

            # compute radius**2 of giration for a set of genomic segments, across population (three outputs)
            # NB: current_selected_beads = the bead indexes making up the cluster after considering the possible combinations of chromosome copies.
            rg2s, _, current_selected_beads = compute_gyration_radius(
                hss['coordinates'], cluster, index, index.copy_index)

            # rg2s = array of size (n_struct), one radius of gyration for each structure, for a given cluster
            # current_selected_beads = array/list of (n_struct, len(cluster)), each row pertains to a different configuration

            # ind is the array of indexes which would sort the array, aka rg2s[ind[0]] <= rgs2[ind[1]] <= ...
            # sorting out radii of gyration array, from smallest to largest
            # (using argpartition & argsort is suggested on stackoverflow, in order to maximize efficiency)

            ind = np.argpartition(rg2s,
                                  keep_best)[:keep_best]  # this is O(n_struct)
            ind = ind[np.argsort(
                rg2s[ind])]  # sorting is O(keep_best ln(keep_best))

            # extract sorted arrays (smallest keep_best entries)
            best_rg2s = rg2s[ind]
            current_selected_beads = current_selected_beads[
                ind]  # select the indices of "keep_best" configurations, and correpsonding beads
            # for each configuration, we have the bead indexes associated to rg.. the numnber
            # of beads is determined by the size of the cluster
            # append quantities to master lists, which sweeps over the different clusters in batch
            selected_beads.append(current_selected_beads)
            indexes.append(ind)
            values.append(
                best_rg2s)  # append keep_best best values of radii of gyration

        #------- saving step into a batch-dependent set of files
        sel_file = os.path.join(tmp_dir, 'tmp.%d.selected.npz' % batch_id)
        idx_file = os.path.join(tmp_dir, 'tmp.%d.idx.npy' % batch_id)
        val_file = os.path.join(tmp_dir, 'tmp.%d.values.npy' % batch_id)

        np.savez(
            sel_file, *selected_beads
        )  # for each configuration in batch, save list of beads making up the cluster (batch_size, keep_best, n_beads)
        # n_beads is different from cluster to cluster (different cluster sizes)
        np.save(
            idx_file, np.array(indexes, dtype=np.int32)
        )  # save indices of configurations explored in current batch (batch_size, keep_best)
        np.save(
            val_file, values
        )  # save radius of gyration values for each of the configutations explored in current batch (batch_size, rg)

        # verify pickle integrity, sometimes weird io problems happen
        selected_beads = np.load(sel_file)
        indexes = np.load(idx_file)
        values = np.load(val_file)
コード例 #15
0
ファイル: ModelingStep.py プロジェクト: davidzhan1995/igm
    def task(struct_id, cfg, tmp_dir):
        """
        Do single structure modeling with restraint assignment from A-step
        """

        # the static method modifications to the cfg should only be local,
        # use a copy of the config file
        cfg = deepcopy(cfg)

        # extract structure information
        step_id = cfg.get('runtime/step_hash', 'xxxx')

        readyfile = os.path.join(tmp_dir, '%s.%d.ready' % (step_id, struct_id))

        # if the ready file exists it does nothing, unless it is a clear run
        if not cfg.get('optimization/clean_restart', False):
            if os.path.isfile(readyfile):
                return

        hssfilename = cfg["optimization"]["structure_output"]

        # read index, radii, coordinates
        with HssFile(hssfilename, 'r') as hss:
            index = hss.index
            radii = hss.radii
            if cfg.get('optimization/random_shuffling', False):
                crd = generate_random_in_sphere(
                    radii, cfg.get('model/restraints/envelope/nucleus_radius'))
            else:
                crd = hss.get_struct_crd(struct_id)

        # init Model class (igm.model)
        model = Model(uid=struct_id)

        # get the chain ids
        chain_ids = np.concatenate([[i] * s
                                    for i, s in enumerate(index.chrom_sizes)])

        # add particles into model
        n_particles = len(crd)
        for i in range(n_particles):
            model.addParticle(crd[i],
                              radii[i],
                              Particle.NORMAL,
                              chainID=chain_ids[i])

        # Add restraints
        monitored_restraints = []

        # ---- POLYMER STRUCTURAL INTEGRITY INTRINSIC restraints ----- #

        # add excluded volume restraint
        ex = Steric(cfg.get("model/restraints/excluded/evfactor"))
        model.addRestraint(ex)

        # add nucleus envelope restraint
        shape = cfg.get('model/restraints/envelope/nucleus_shape')
        envelope_k = cfg.get('model/restraints/envelope/nucleus_kspring')
        radius = 0
        semiaxes = (0, 0, 0)

        if shape == 'sphere':
            radius = cfg.get('model/restraints/envelope/nucleus_radius')
            ev = Envelope(shape, radius, envelope_k)
        elif cfg['model']['restraints']['envelope'][
                'nucleus_shape'] == 'ellipsoid':
            semiaxes = cfg.get('model/restraints/envelope/nucleus_semiaxes')
            ev = Envelope(shape, semiaxes, envelope_k)
        elif cfg['model']['restraints']['envelope'][
                'nucleus_shape'] == 'exp_map':
            volume_file = cfg.get('model/restraints/envelope/input_map')
            ev = GenEnvelope(shape, volume_file, envelope_k)

        model.addRestraint(ev)
        monitored_restraints.append(ev)

        # add consecutive bead polymer restraint to ensure chain connectivity
        if cfg.get('model/restraints/polymer/polymer_bonds_style') != 'none':
            contact_probabilities = cfg['runtime'].get(
                'consecutive_contact_probabilities', None)
            pp = Polymer(
                index,
                cfg['model']['restraints']['polymer']['contact_range'],
                cfg['model']['restraints']['polymer']['polymer_kspring'],
                contact_probabilities=contact_probabilities)
            model.addRestraint(pp)
            monitored_restraints.append(pp)

        # LB: add nuclear body excluded volume restraints
        if "nucleolus" in cfg['restraints']:

            # read in nucle lus coordinates and radius from cfg file

            for mappa in cfg['restraints']['nucleolus']['input_map']:

                nucl = GenEnvelope(cfg['restraints']['nucleolus']['shape'],
                                   mappa,
                                   cfg['restraints']['nucleolus']['k_spring'])
                model.addRestraint(nucl)
                monitored_restraints.append(nucl)

        # ---- IGM MODELING RESTRAINTS FROM EXPERIMENTAL DATA (FISH MISSING) ---- #

        # add Hi-C restraint
        if "Hi-C" in cfg['restraints']:

            # read parameters from cfg file
            actdist_file = cfg.get('runtime/Hi-C/actdist_file')
            contact_range = cfg.get('restraints/Hi-C/contact_range', 2.0)
            k = cfg.get('restraints/Hi-C/contact_kspring', 0.05)

            # effectively add HiC restraints (bonds)
            hic = HiC(actdist_file, contact_range, k)
            model.addRestraint(hic)
            monitored_restraints.append(hic)

        # add DAMID restraint
        if "DamID" in cfg['restraints']:

            # read parameters from cfg file
            actdist_file = cfg.get('runtime/DamID/damid_actdist_file')
            contact_range = cfg.get('restraints/DamID/contact_range', 2.0)
            k = cfg.get('restraints/DamID/contact_kspring', 0.05)

            # effectively add DAMID restraints
            damid = Damid(damid_file=actdist_file,
                          contact_range=contact_range,
                          nuclear_radius=radius,
                          k=k,
                          shape=shape,
                          semiaxes=semiaxes)
            model.addRestraint(damid)
            monitored_restraints.append(damid)

        # add SPRITE restraint
        if "sprite" in cfg['restraints']:

            # read parameters from cfg file
            sprite_tmp = make_absolute_path(
                cfg.get('restraints/sprite/tmp_dir', 'sprite'),
                cfg.get('parameters/tmp_dir'))
            assignment_filename = make_absolute_path(
                cfg.get('restraints/sprite/assignment_file', 'assignment.h5'),
                sprite_tmp)

            # effectively add SPRITE retraints
            sprite = Sprite(assignment_filename,
                            cfg.get('restraints/sprite/volume_fraction',
                                    0.05), struct_id,
                            cfg.get('restraints/sprite/kspring', 1.0))
            model.addRestraint(sprite)
            monitored_restraints.append(sprite)

        # ========Optimization
        cfg['runtime']['run_name'] = cfg.get('runtime/step_hash') + '_' + str(
            struct_id)
        optinfo = model.optimize(cfg)

        # tolerance parameter: if violation score is smaller than tolerance, then restraint is satisfied
        tol = cfg.get('optimization/violation_tolerance', 0.01)

        # save optimization results to .hms file
        ofname = os.path.join(tmp_dir, 'mstep_%d.hms' % struct_id)
        with HmsFile(ofname, 'w') as hms:
            hms.saveModel(struct_id, model)

            # create violations statistics and save all of that into the "vstat" dictionary
            vstat = {}
            for r in monitored_restraints:
                vs = []
                n_imposed = 0
                for fid in r.forceID:
                    f = model.forces[fid]
                    n_imposed += f.rnum
                    if f.rnum > 1:
                        # a list of values is appended to vs = [] at once
                        vs += f.getViolationRatios(model.particles).tolist()
                    else:
                        # one value is appended at the time to vs = []
                        vs.append(f.getViolationRatio(model.particles))
                vs = np.array(vs)
                H, edges = get_violation_histogram(vs)
                num_violations = np.count_nonzero(
                    vs > tol
                )  # the same 'tol' value is used to compute the number of violations across different restraint kinds...is that too easy?
                vstat[repr(r)] = {
                    'histogram': {
                        'edges': edges.tolist(),
                        'counts': H.tolist()
                    },
                    'n_violations': num_violations,
                    'n_imposed': n_imposed
                }

            # add violation dictionary to hms file
            h5_create_or_replace_dataset(hms, 'violation_stats',
                                         json.dumps(vstat))

            if isinstance(optinfo, dict):
                grp = h5_create_group_if_not_exist(hms, 'opt_info')
                for k, v in optinfo.items():
                    if not isinstance(v, dict):
                        h5_create_or_replace_dataset(grp, k, data=v)
                h5_create_or_replace_dataset(hms,
                                             'opt_info_dict',
                                             data=json.dumps(optinfo))

        # double check it has been written correctly
        with HmsFile(ofname, 'r') as hms:
            if not np.all(hms.get_coordinates() == model.getCoordinates()):
                raise RuntimeError('error writing the file %s' % ofname)

        # generat;e the .ready file, which signals to the poller that optimization for that structure has been completed
        readyfile = os.path.join(tmp_dir, '%s.%d.ready' % (step_id, struct_id))
        open(readyfile, 'w').close()  # touch the ready-file
コード例 #16
0
ファイル: RelaxInit.py プロジェクト: davidzhan1995/igm
    def task(struct_id, cfg, tmp_dir):
        """
        relax one random structure chromosome structures, SERIAL
        """
        cfg = deepcopy(cfg)

        readyfile = os.path.join(tmp_dir, 'relax_%d.hms.ready' % struct_id)

        # if the ready file exists it does nothing, unless it is a clear run
        if not cfg.get('optimization/clean_restart', False):
            if os.path.isfile(readyfile):
                return

        # extract structure information
        hssfilename = cfg["optimization"]["structure_output"]

        # read index, radii, coordinates
        with HssFile(hssfilename, 'r') as hss:
            index = hss.index
            radii = hss.radii
            crd = hss.get_struct_crd(struct_id)

        # init Model
        model = Model(uid=struct_id)

        # add particles into model
        n_particles = len(crd)
        for i in range(n_particles):
            model.addParticle(crd[i], radii[i], Particle.NORMAL)

        # ========Add polymer/nucleoli restraints =========

        # add excluded volume restraint
        ex = Steric(cfg.get("model/restraints/excluded/evfactor"))
        model.addRestraint(ex)

        # add nucleus envelope restraint (spherical, ellipsoidal OR from data)
        if cfg['model']['restraints']['envelope']['nucleus_shape'] == 'sphere':
            ev = Envelope(
                cfg['model']['restraints']['envelope']['nucleus_shape'],
                cfg['model']['restraints']['envelope']['nucleus_radius'],
                cfg['model']['restraints']['envelope']['nucleus_kspring'])
        elif cfg['model']['restraints']['envelope'][
                'nucleus_shape'] == 'ellipsoid':
            ev = Envelope(
                cfg['model']['restraints']['envelope']['nucleus_shape'],
                cfg['model']['restraints']['envelope']['nucleus_semiaxes'],
                cfg['model']['restraints']['envelope']['nucleus_kspring'])
        elif cfg['model']['restraints']['envelope'][
                'nucleus_shape'] == 'exp_map':
            ev = GenEnvelope(
                cfg['model']['restraints']['envelope']['nucleus_shape'],
                cfg['model']['restraints']['envelope']['input_map'],
                cfg['model']['restraints']['envelope']['nucleus_kspring'])
        model.addRestraint(ev)

        # add consecutive polymer restraint
        contact_probabilities = cfg['runtime'].get(
            'consecutive_contact_probabilities', None)
        pp = Polymer(index,
                     cfg['model']['restraints']['polymer']['contact_range'],
                     cfg['model']['restraints']['polymer']['polymer_kspring'],
                     contact_probabilities=contact_probabilities)
        model.addRestraint(pp)

        # LB: add nuclear body "excluded volume" restraints (keep chromosomes out of nucleolar region)
        if 'nucleolus' in cfg['restraints']:

            for mappa in cfg['restraints']['nucleolus']['input_map']:

                nucl = GenEnvelope(cfg['restraints']['nucleolus']['shape'],
                                   mappa,
                                   cfg['restraints']['nucleolus']['k_spring'])
                model.addRestraint(nucl)
                logger.info(nucl)

        # ========Optimization

        # set "run_name" variable into "runtime" dictionary
        cfg['runtime']['run_name'] = cfg['runtime']['step_hash'] + '_' + str(
            struct_id)

        # run optimization of the structures, by enforcing excluded volume, polymer and envelope restraints
        model.optimize(cfg)

        # save optimization results (both optimized coordinates and violations) into a .hms file
        ofname = os.path.join(tmp_dir, 'relax_%d.hms' % struct_id)
        with HmsFile(ofname, 'w') as hms:
            hms.saveModel(struct_id, model)
            hms.saveViolations(pp)

        # make sure write was successful
        with HmsFile(ofname, 'r') as hms:
            if not np.all(hms.get_coordinates() == model.getCoordinates()):
                raise RuntimeError('error writing the file %s' % ofname)

        # create .ready file, which signals to the poller that optimization went to completion
        readyfile = ofname + '.ready'
        open(readyfile, 'w').close()  # touch the ready-file
コード例 #17
0
ファイル: RelaxInit.py プロジェクト: davidzhan1995/igm
    def teardown_poller(self):
        """ Reopen HSS file, overwrite ALL coordinates, close file """

        _hss = HssFile(self.hssfilename, 'r+')
        _hss.set_coordinates(self._hss_crd)
        _hss.close()
コード例 #18
0
ファイル: modeling_step.py プロジェクト: davidzhan1995/igm
def modeling_task(struct_id, cfg_file):
    '''
    Serial function to be mapped in parallel. //
    It is a wrapper intended to be used only internally by the parallel map
    function. Will be called as a partial with all the constant variables
    set, except i.
    Resolve the templates, obtains input data,
    runs the minimization routines and finally communicates back results.

    Parameters
    ---------- 
    i : int
        number of the structure 
    cfg_file : str 
        configuration filename for the task
    
    Returns
    -------
    None
    '''
    cfg = Config(cfg_file)

    # importing here so it will be called on the parallel workers
    local_vars = resolve_templates(cfg['mstep']['templates'], [struct_id])

    model = Model()
    with HssFile(cfg['mstep']['input_hss'], 'r') as f:
        radii = f.radii
        index = f.index
        crd = f['coordinates'][:, struct_id, :][()]

    n_particles = len(crd)
    for i in range(n_particles):
        model.addParticle(crd[i], radii[i], Particle.NORMAL)

    ee = Envelope(cfg['model']['nucleus_geometry'])
    model.addRestraint(ee)

    ex = Steric(cfg['model']['evfactor'])
    model.addRestraint(ex)

    pp = Polymer(index, cfg['model']['contact_range'],
                 cfg['model']['contact_kspring'])
    model.addRestraint(pp)

    kernel = kernel_class[cfg['mstep']['kernel']]
    info = kernel.optimize(model, cfg['optimization'])

    new_crd = np.array([p.pos for p in model.particles], dtype=COORD_DTYPE)
    np.save(local_vars['crd_out'], new_crd)

    # make sure that is readable
    np.load(local_vars['crd_out'])

    with open(local_vars['info_out'], 'w') as f:
        for k in kernel.INFO_KEYS:
            if isinstance(info[k], float):
                out_str = '{:9.2f}'.format(info[k])
            elif isinstance(info[k], int):
                out_str = '{:7d}'.format(info[k])
            else:
                out_str = str(info[k])
            f.write(out_str + '\t')
コード例 #19
0
ファイル: ModelingStep.py プロジェクト: davidzhan1995/igm
 def skip(self):
     fn = self.intermediate_name() + '.hss'
     if os.path.isfile(fn):
         with HssFile(fn, 'r') as hss:
             violation_score = log_stats(hss, self.cfg)
             self.cfg['runtime']['violation_score'] = violation_score
コード例 #20
0
    def reduce(self):

        # reconstruct contacts full map
        # ii = []
        # jj = []
        # data = []
        # with HssFile(self.cfg.get('optimization/structure_output')) as structure_output:

        out_dir = self.out_dir
        sigma = self.cfg.get('runtime/Hi-C/sigma')

        input_matrix = Contactmatrix(
            self.cfg.get('restraints/Hi-C/input_matrix'))
        with HssFile(self.cfg.get(
                'optimization/structure_output')) as structure_output:
            output_matrix = structure_output.buildContactMap(
                contactRange=self.cfg.get('restraints/Hi-C/contact_range') *
                (1 +
                 eps))  # give some tolerance. only in one direction though.
        output_matrix.save(os.path.join(out_dir, 'full_matrix.hcs'))
        output_matrix = output_matrix.sumCopies()
        output_matrix.matrix.data[:] = output_matrix.matrix.data.clip(0, 1)
        output_matrix.save(os.path.join(out_dir, 'out_matrix.hcs'))
        plot_comparison(input_matrix,
                        output_matrix,
                        labels=['input', 'output'],
                        file=os.path.join(out_dir, 'matrix_comparison.pdf'),
                        vmax=0.2)
        for c in input_matrix.index.get_chrom_names():
            plot_comparison(input_matrix[c],
                            output_matrix[c],
                            labels=['input', 'output'],
                            file=os.path.join(out_dir,
                                              'matrix_comparison_%s.pdf' % c),
                            title=c,
                            vmax=0.2)
        with np.errstate(divide='ignore', invalid='ignore'):
            diffmat = np.log2(output_matrix.matrix.toarray() /
                              input_matrix.matrix.toarray())
        maxv = np.percentile(np.abs(diffmat[np.isfinite(diffmat)]), 99)
        plt.figure()
        plt.imshow(diffmat, vmax=maxv, vmin=-maxv, cmap='RdBu_r')
        plt.title('difference_matrix')
        plt.colorbar()
        plt.savefig(os.path.join(out_dir, 'diffmat.pdf'))
        plt.close()
        for c in input_matrix.genome.chroms:
            ii = input_matrix.index.chrom == input_matrix.genome.getchrnum(c)
            plt.figure()
            xmat = diffmat[ii][:, ii]
            plt.imshow(xmat, vmax=maxv, vmin=-maxv, cmap='RdBu_r')
            plt.colorbar()
            plt.savefig(os.path.join(out_dir, 'diffmap_' + c + '.pdf'))
            plt.close()

        input_matrix = {(i, j): pwish
                        for i, j, pwish in input_matrix.matrix.coo_generator()
                        if pwish >= sigma and i != j}
        diffs = []
        reldiffs = []
        totp = 0
        for i, j, pout in output_matrix.matrix.coo_generator():
            p = input_matrix.get((i, j))
            if p is not None:
                diffs.append(pout - p)
                reldiffs.append((pout - p) / p)
                totp += p
        del output_matrix
        del input_matrix
        diffs = np.array(diffs)
        reldiffs = np.array(reldiffs)

        f, ax = plt.subplots(2, 2)
        ax[0, 0].set_title('Absolute matrix differences')
        ax[0, 0].hist(diffs, bins=100, range=(-1, 1))
        ax[0, 1].set_title('Relative matrix differences')
        ax[0, 1].hist(reldiffs, bins=100, range=(-1, 1))
        ax[1, 0].set_title('Absolute matrix differences (log)')
        ax[1, 0].hist(diffs, bins=100, log=True, range=(-1, 1))
        ax[1, 1].set_title('Relative matrix differences (log)')
        ax[1, 1].hist(reldiffs, bins=100, log=True, range=(-1, 1))

        plt.tight_layout()
        plt.savefig(os.path.join(out_dir, 'difference_histograms.pdf'))

        tol = self.cfg.get('restraints/Hi-C/evaluation_tolerance', 0.01)
        n = np.count_nonzero(np.abs(diffs) > tol)
        self.score = np.abs(reldiffs).mean()
        #self.ok = n < 0.01 * len(diffs)
        #self.score = float(n)/len(diffs)
        #self.cfg['runtime']['violation_score'] = self.score
        with open(os.path.join(out_dir, 'stats.txt'), 'w') as f:
            print("#score ave_differences ave_relative_differences", file=f)
            print(self.score, np.average(diffs), np.average(reldiffs), file=f)
        logger.info('>>>  Average relative difference: {:6.3f}%  <<<'.format(
            self.score * 100))
コード例 #21
0
ファイル: RelaxInit.py プロジェクト: davidzhan1995/igm
    def setup_poller(self):
        """ Load Hss population file, store all coordinates into numpy array, close file"""

        _hss = HssFile(self.hssfilename, 'r')
        self._hss_crd = _hss.coordinates
        _hss.close()
コード例 #22
0
ファイル: kernel_testing.py プロジェクト: davidzhan1995/igm
def debug_minimization(cfg, struct_id, rname, **kwargs):
    if not isinstance(cfg, dict):
        cfg = Config(cfg)
    if os.path.isfile(cfg['step_db']):
        db = StepDB(cfg)
        h = db.get_history()
        cfg.update(h[-1])

    cfg['optimization']['optimizer_options'].update(kwargs)
    cfg['optimization']['keep_temporary_files'] = True

    step_id = rname

    hssfilename    = cfg['structure_output']

    #read index, radii, coordinates
    with HssFile(hssfilename,'r') as hss:
        index = hss.index
        radii = hss.radii
        if cfg.get('random_shuffling', False):
            crd = generate_random_in_sphere(radii, cfg['model']['nucleus_radius'])
        else:
            crd = hss.get_struct_crd(struct_id)

    #init Model
    model = Model(uid=struct_id)

    # get the chain ids
    chain_ids = np.concatenate( [ [i]*s for i, s in enumerate(index.chrom_sizes) ] )

    #add particles into model
    n_particles = len(crd)
    for i in range(n_particles):
        model.addParticle(crd[i], radii[i], Particle.NORMAL, chainID=chain_ids[i])

    #========Add restraint
    monitored_restraints = []

    #add excluded volume restraint
    ex = Steric(cfg['model']['evfactor'])
    model.addRestraint(ex)

    #add nucleus envelop restraint
    if cfg['model']['nucleus_shape'] == 'sphere':
        ev = Envelope(cfg['model']['nucleus_shape'],
                      cfg['model']['nucleus_radius'],
                      cfg['model']['contact_kspring'])
    elif cfg['model']['nucleus_shape'] == 'ellipsoid':
        ev = Envelope(cfg['model']['nucleus_shape'],
                      cfg['model']['nucleus_semiaxes'],
                      cfg['model']['contact_kspring'])
    else:
        raise NotImplementedError('Invalid nucleus shape')
    model.addRestraint(ev)

    #add consecutive polymer restraint
    pp = Polymer(index,
                 cfg['model']['contact_range'],
                 cfg['model']['contact_kspring'])
    model.addRestraint(pp)
    monitored_restraints.append(pp)

    #add Hi-C restraint
    # if "Hi-C" in cfg['restraints']:
    #     dictHiC = cfg['restraints']['Hi-C']
    #     actdist_file = cfg['runtime']['Hi-C']['actdist_file']
    #     contact_range = dictHiC.get( 'contact_range', 2.0 )
    #     k = dictHiC.get( 'contact_kspring', 1.0 )

    #     hic = HiC(actdist_file, contact_range, k)
    #     model.addRestraint(hic)
    #     monitored_restraints.append(hic)

    # if "sprite" in cfg['restraints']:
    #     sprite_opt = cfg['restraints']['sprite']
    #     sprite = Sprite(
    #         sprite_opt['assignment_file'],
    #         sprite_opt['volume_fraction'],
    #         struct_id,
    #         sprite_opt['kspring']
    #     )
    #     model.addRestraint(sprite)
    #     monitored_restraints.append(sprite)

    #========Optimization
    #optimize model
    cfg['runtime']['run_name'] = rname
    model.optimize(cfg)

    tol = cfg.get('violation_tolerance', 0.01)
    lockfile = os.path.join('.', '%s.%d.ready' % (step_id, struct_id) )
    with FileLock(lockfile):
        open(lockfile, 'w').close() # touch the ready-file
        ofname = os.path.join('.', 'mstep_%d.hms' % struct_id)
        with HmsFile(ofname, 'w') as hms:
            hms.saveModel(struct_id, model)

            for r in monitored_restraints:
                hms.saveViolations(r, tolerance=tol)

        # double check it has been written correctly
        with HmsFile(ofname, 'r') as hms:
            if np.all( hms.get_coordinates() == model.getCoordinates() ):
                raise RuntimeError('error writing the file %s' % ofname)