Esempio n. 1
0
    def _Read_Lineage(self):
        ''' Read in Lineage object and import all the GalPop objects within 
        lineage to Inherit object. 
        '''
        # read in the lineage (< 0.05 seconds for one snapshot)
        if not self.quiet:
            read_time = time.time()
        bloodline = Lineage(nsnap_ancestor=self.nsnap_ancestor,
                            subhalo_prop=self.subhalo_prop,
                            quiet=self.quiet)
        bloodline.Read(self.nsnap_descendants, quiet=self.quiet)

        #DEFUNCT
        #if 'subhalogrowth' in self.sfr_prop.keys():
        #    # depending on whether SFR assign includes subhalo growth AM
        #    sfr_prop['subhalogrowth']['nsnap_descendant'] = nsnap_descendant

        # assign SFR to ancestor object
        bloodline.AssignSFR_ancestor(sfr_prop=self.sfr_prop, quiet=self.quiet)
        if not self.quiet:
            print 'Lineage Read Time = ', time.time() - read_time

        self.ancestor = bloodline.ancestor  # ancestor object
        self.ancestor.tQ[np.where(self.ancestor.tQ != 999.)] = 0.
        self.MshamEvol = self.ancestor.Msham_evol[0]

        self._ImportDescendants(bloodline)

        return None
def AncestorPlots(**sfinh_kwargs):
    ''' FQ for the SF Inherited Ancestor galaxy population. This test is mainly 
    to see what the discrepancy between the empirical SF/Q classifcation and the
    parameterized FQ model. Ideally, the model and the 
    '''
    bloodline = Lineage(nsnap_ancestor=sfinh_kwargs['nsnap_ancestor'],
                        subhalo_prop=sfinh_kwargs['subhalo_prop'])
    bloodline.Read([1])
    bloodline.AssignSFR_ancestor(sfr_prop=sfinh_kwargs['sfr_prop'])

    ancestor = getattr(bloodline, 'ancestor')
    ancestor.sfr_prop = sfinh_kwargs['sfr_prop']
    fig_file = lambda prop: ''.join([
        'figure/test/', 'Ancestor',
        str(sfinh_kwargs['nsnap_ancestor']), '.',
        prop.upper(), '.png'
    ])
    ancestor.plotSsfr(savefig=fig_file('ssfr'))
    plt.close()
    # FQ
    ancestor.plotFq(model=True, savefig=fig_file('fq'))
    ancestor.plotSFMS(sfqcut=True,
                      gal_class='all',
                      bovyplot=False,
                      sigSFR=False,
                      model=False,
                      savefig=fig_file('sfms'))
    plt.close()
    return None
def DescendantSSFR(nsnap_descendants, **kwargs):
    ''' Halo Mass Function composition at z = z_final by initial halo mass at 
    nsnap_ancestor. 
    '''
    # import SF inherited Lineage
    sf_inherit_file = InheritSF_file(nsnap_descendants, **kwargs)
    bloodline = Lineage(nsnap_ancestor=kwargs['nsnap_ancestor'],
                        subhalo_prop=kwargs['subhalo_prop'])
    if not isinstance(nsnap_descendants, list):
        nsnap_descendants = [nsnap_descendants]
    bloodline.Read(nsnap_descendants, filename=sf_inherit_file)
    # descendant
    for nsnap_descendant in nsnap_descendants:
        descendant = getattr(bloodline,
                             'descendant_snapshot' + str(nsnap_descendant))
        fig_name = ''.join([
            'figure/test/', 'SSFR.', '.nsnap',
            str(nsnap_descendant), '.'.join(
                (sf_inherit_file.rsplit('/')[-1]).rsplit('.')[:-1]), '.png'
        ])

        descendant.plotSsfr(line_color='red',
                            line_width=4,
                            sfms_prop=kwargs['sfr_prop']['sfms'],
                            z=descendant.zsnap,
                            groupcat=True,
                            savefig=fig_name)
        plt.close()
    return None
Esempio n. 4
0
class LineageTests(unittest.TestCase):

    def setUp(self):
        
        self.biodb_selector= s= Selector("ncbi")
        #self.feature= self.biodb_selector.getFeatureByID(781)
        self.feature= self.biodb_selector.getFeatureByID(89)
        self.lineage = Lineage(self.feature, self.biodb_selector)

    def test_lineage_loading(self):
        self.failIf(len(self.lineage.existing_list) > len(self.lineage.default_levels))
     
    def test_levels(self):
        self.failIf(self.lineage.get_levels() != [t.level for t in self.lineage.taxon_list])

    def test_taxon_retrieval(self):
        self.failIf(self.lineage.get_taxa_by_level(2)[0].taxon.level != 2)
   
    def test_level_exists(self):
        self.failIf(self.lineage.level_exists(3) != True)

    def test_closest_taxon_retrieval(self):
        self.failIf(self.lineage._get_closest_taxa_by_level(4, self.lineage.taxon_list)[0].taxon.id < 0)

    def test_closest_unn_retrieval(self):
        unn= self.lineage.get_closest_unnecessary_taxon_by_level(4)
        self.failIf(unn is None)
Esempio n. 5
0
def LineageAncestorSFMS(nsnap_ancestor,
                        subhalo_prop={
                            'scatter': 0.0,
                            'source': 'li-march'
                        },
                        sfr_prop={
                            'fq': {
                                'name': 'wetzelsmooth'
                            },
                            'sfr': {
                                'name': 'average'
                            }
                        }):
    '''Plot the SF-MS of the lineage ancestor object. This is mainly to make sure 
    that the AssignSFR routine is working properly 
    '''
    # read in lineage
    bloodline = Lineage(nsnap_ancestor=nsnap_ancestor,
                        subhalo_prop=subhalo_prop)
    bloodline.Read([1], sfr_prop=sfr_prop)
    ancestor = getattr(bloodline, 'ancestor')

    sfms_plot = plots.PlotSFMS()
    sfms_plot.cenque(ancestor, justsf=True)  # lineage ancestor
    sfms_plot.param_sfms(nsnap=nsnap_ancestor)

    sfms_plot_file = ''.join([
        'figure/test/', 'LineageAncestorSFMS',
        str(nsnap_ancestor),
        bloodline._file_spec(subhalo_prop=subhalo_prop, sfr_prop=sfr_prop),
        '.png'
    ])
    sfms_plot.save_fig(sfms_plot_file)
    return None
Esempio n. 6
0
def LineageFinalDescendantSMF(nsnap_ancestor,
                              subhalo_prop={
                                  'scatter': 0.0,
                                  'source': 'li-march'
                              },
                              sfr_prop={
                                  'fq': {
                                      'name': 'wetzelsmooth'
                                  },
                                  'sfr': {
                                      'name': 'average'
                                  }
                              }):
    ''' Plot SMF of final descendant. Also mark the composition of the final SMF from Subhalos 
    that gain stellar mass at different snapshots in the middle of the simulation. 
    '''
    prettyplot()
    pretty_colors = prettycolors()

    bloodline = Lineage(nsnap_ancestor=nsnap_ancestor,
                        subhalo_prop=subhalo_prop)
    bloodline.Read([1], sfr_prop=sfr_prop)

    final_desc = bloodline.descendant_snapshot1

    mf = SMF()

    smf_plot = plots.PlotSMF()
    smf_plot.cgpop(final_desc, line_color='k')

    for isnap in range(2, nsnap_ancestor + 1)[::-1]:
        started_here = np.where(final_desc.nsnap_genesis == isnap)
        mass, phi = mf._smf(final_desc.mass[started_here])

        try:
            phi_tot += phi
            smf_plot.sub.fill_between(mass,
                                      phi_tot - phi,
                                      phi_tot,
                                      color=pretty_colors[isnap],
                                      label='Nsnap=' + str(isnap))
        except UnboundLocalError:
            phi_tot = phi
            smf_plot.sub.fill_between(mass,
                                      np.zeros(len(phi)),
                                      phi,
                                      color=pretty_colors[isnap],
                                      label='Nsnap=' + str(isnap))

    smf_plot.set_axes()
    fig_file = ''.join([
        'figure/test/', 'LineageFinalDescendantSMF', '.ancestor',
        str(nsnap_ancestor),
        bloodline._file_spec(subhalo_prop=subhalo_prop, sfr_prop=sfr_prop),
        '.png'
    ])
    smf_plot.save_fig(fig_file)

    return None
Esempio n. 7
0
    def remap_snps(
            self,
            target_assembly,
            complement_bases=True,
            parallelize=True,
            processes=os.cpu_count(),
    ):
        """ Remap the SNP coordinates of this ``Individual`` from one assembly to another.

        This method is a wrapper for `remap_snps` in the ``Lineage`` class.

        This method uses the assembly map endpoint of the Ensembl REST API service to convert SNP
        coordinates / positions from one assembly to another. After remapping, the coordinates /
        positions for the ``Individual``'s SNPs will be that of the target assembly.

        If the SNPs are already mapped relative to the target assembly, remapping will not be
        performed.

        Parameters
        ----------
        target_assembly : {'NCBI36', 'GRCh37', 'GRCh38', 36, 37, 38}
            assembly to remap to
        complement_bases : bool
            complement bases when remapping SNPs to the minus strand
        parallelize : bool
            utilize multiprocessing to speedup calculations
        processes : int
            processes to launch if multiprocessing

        Returns
        -------
        chromosomes_remapped : list of str
            chromosomes remapped; empty if None
        chromosomes_not_remapped : list of str
            chromosomes not remapped; empty if None

        Notes
        -----
        An assembly is also know as a "build." For example:

        Assembly NCBI36 = Build 36
        Assembly GRCh37 = Build 37
        Assembly GRCh38 = Build 38

        See https://www.ncbi.nlm.nih.gov/assembly for more information about assemblies and
        remapping.

        References
        ----------
        ..[1] Ensembl, Assembly Map Endpoint,
          http://rest.ensembl.org/documentation/info/assembly_map
        """
        from lineage import Lineage

        l = Lineage(parallelize=parallelize, processes=processes)
        return l.remap_snps(self, target_assembly, complement_bases)
Esempio n. 8
0
    def merge_snps(self):
        if not self.snps_can_be_merged:
            return

        snps = self.snps.filter(generated_by_lineage=True)
        if len(snps) == 1:
            # remove SNPs generated by lineage since we're remaking that file
            snps[0].delete()

        if self.get_discrepant_snps():
            # remove discrepant SNPs since we'll be refreshing that data
            self.discrepant_snps.delete()

        with tempfile.TemporaryDirectory() as tmpdir:
            l = Lineage(output_dir=tmpdir, parallelize=False)

            ind = l.create_individual("ind")
            for snps in self.snps.all():
                if snps.build != 37:
                    temp = l.create_individual("temp", snps.file.path)
                    temp.remap_snps(37, parallelize=False)
                    temp_snps = temp.save_snps()
                    ind.load_snps(temp_snps)
                    del temp
                else:
                    ind.load_snps(snps.file.path)

                snps.merged = True
                snps.save()

            if ind.snp_count != 0:
                if len(ind.discrepant_snps) != 0:
                    dsnps = DiscrepantSnps.objects.create(
                        user=self.user,
                        individual=self,
                        snp_count=len(ind.discrepant_snps),
                    )
                    discrepant_snps_file = ind.save_discrepant_snps()
                    dsnps.file.name = dsnps.get_relative_path()
                    dsnps.save()
                    shutil.move(discrepant_snps_file, dsnps.file.path)

                merged_snps_file = ind.save_snps()
                summary_info, snps_is_valid = parse_snps(merged_snps_file)

                if snps_is_valid:
                    summary_info["generated_by_lineage"] = True
                    summary_info["merged"] = True
                    self.add_snps(merged_snps_file, summary_info)
def DescendantSMHM_composition(nsnap_descendant, abc_step=29, **sfinh_kwargs):
    ''' Plot the Stellar Mass to Halo Mass relation. 
    '''
    sfinherit_file = InheritSF_file(nsnap_descendant,
                                    abc_step=abc_step,
                                    **sfinh_kwargs)
    bloodline = Lineage(nsnap_ancestor=sfinh_kwargs['nsnap_ancestor'],
                        subhalo_prop=sfinh_kwargs['subhalo_prop'])
    bloodline.Read([nsnap_descendant], filename=sfinherit_file)

    descendant = getattr(bloodline,
                         'descendant_snapshot' + str(nsnap_descendant))
    smhm_plot = descendant.plotSMHM(bovyplot=False, scatter=False)

    # SMF composition based on their initial mass at nsnap_ancestor
    started_here = np.where(
        descendant.nsnap_genesis == sfinh_kwargs['nsnap_ancestor'])
    start_mass = descendant.mass_genesis[started_here]

    mass_bins = np.arange(7., 12.5, 0.5)
    mass_bin_low = mass_bins[:-1]
    mass_bin_high = mass_bins[1:]

    for i_m in range(len(mass_bin_low)):

        mbin = np.where((start_mass > mass_bin_low[i_m])
                        & (start_mass <= mass_bin_high[i_m]))

        smhm_plot.plot(stellarmass=descendant.mass[started_here[0][mbin]],
                       halomass=descendant.halo_mass[started_here[0][mbin]],
                       bovyplot=False,
                       color=i_m,
                       label=r"$\mathtt{M_{*,i}=}$" +
                       str(round(mass_bin_low[i_m], 2)) + "-" +
                       str(round(mass_bin_high[i_m], 2)))

    smhm_plot.plotSummary(stellarmass=descendant.mass,
                          halomass=descendant.halo_mass)

    smhm_plot.sub.legend(loc='upper left', scatterpoints=1)
    smhm_plot.set_axes()
    fig_file = ''.join([
        'figure/test/', 'SMHM_composition.', ''.join(
            (sfinherit_file.rsplit('/')[-1]).rsplit('.')[:-1]), '.png'
    ])
    smhm_plot.save_fig(fig_file)
    return None
Esempio n. 10
0
def predict(addr, model_name, input_lin, batch=False):
	url = "http://%s/%s/predict" % (addr, model_name) 
	if batch:
		req_json = json.dumps({'input_batch': [[x.val for x in input_lin]]})
	else:
		req_json = json.dumps({'input': [input_lin.val]})
	headers = {'Content-type': 'application/json'}
	r = requests.post(url, headers=headers, data=req_json)
	input_lin.make_prediction()
	return Lineage.add_node(input_lin, model_name, json.loads(r.text)["output"][0])
def DescendantFQ(nsnap_descendant, abc_step=29, **sfinh_kwargs):
    ''' FQ for the SF Inherited Descendant galaxy population  
    '''
    sfinherit_file = InheritSF_file(nsnap_descendant,
                                    abc_step=abc_step,
                                    **sfinh_kwargs)
    bloodline = Lineage(nsnap_ancestor=sfinh_kwargs['nsnap_ancestor'],
                        subhalo_prop=sfinh_kwargs['subhalo_prop'])
    bloodline.Read([nsnap_descendant], filename=sfinherit_file)

    descendant = getattr(bloodline,
                         'descendant_snapshot' + str(nsnap_descendant))
    descendant.sfr_prop = sfinh_kwargs['sfr_prop']

    fig_file = ''.join([
        'figure/test/', 'Fq.', '.'.join(
            (sfinherit_file.rsplit('/')[-1]).rsplit('.')[:-1]), '.png'
    ])
    descendant.plotFq(model=sfinh_kwargs['sfr_prop']['fq']['name'],
                      savefig=fig_file)
    return None
def Read_InheritSF(nsnap_descendant,
                   nsnap_ancestor=20,
                   n_step=29,
                   subhalo_prop={
                       'scatter': 0.2,
                       'source': 'li-march'
                   },
                   sfr_prop={
                       'fq': {
                           'name': 'wetzelsmooth'
                       },
                       'sfr': {
                           'name': 'average'
                       }
                   },
                   evol_prop={
                       'sfr': {
                           'dutycycle': {
                               'name': 'notperiodic'
                           }
                       },
                       'mass': {
                           'name': 'sham'
                       }
                   },
                   flag=None):

    sf_inherit_file = InheritSF_file(nsnap_descendant,
                                     nsnap_ancestor=nsnap_ancestor,
                                     abc_step=n_step,
                                     subhalo_prop=subhalo_prop,
                                     sfr_prop=sfr_prop,
                                     evol_prop=evol_prop,
                                     flag=flag)

    bloodline = Lineage(nsnap_ancestor=nsnap_ancestor,
                        subhalo_prop=subhalo_prop)
    bloodline.Read([nsnap_descendant], filename=sf_inherit_file)

    return bloodline
def LineageFinalDescendantSMF(nsnap_ancestor, 
        subhalo_prop={'scatter': 0.0, 'source': 'li-march'}, 
        sfr_prop={'fq': {'name': 'wetzelsmooth'}, 'sfr': {'name': 'average'}}): 
    ''' Plot SMF of final descendant. Also mark the composition of the final SMF from Subhalos 
    that gain stellar mass at different snapshots in the middle of the simulation. 
    '''
    prettyplot()
    pretty_colors=prettycolors()

    bloodline = Lineage(nsnap_ancestor=nsnap_ancestor, subhalo_prop=subhalo_prop)
    bloodline.Read([1], sfr_prop=sfr_prop)

    final_desc = bloodline.descendant_snapshot1
    
    mf = SMF()
    
    smf_plot = plots.PlotSMF()
    smf_plot.cgpop(final_desc, line_color='k')
    
    for isnap in range(2, nsnap_ancestor+1)[::-1]:
        started_here = np.where(final_desc.nsnap_genesis == isnap)
        mass, phi = mf._smf(final_desc.mass[started_here])

        try: 
            phi_tot += phi 
            smf_plot.sub.fill_between(mass, phi_tot - phi, phi_tot, color=pretty_colors[isnap], label='Nsnap='+str(isnap))
        except UnboundLocalError: 
            phi_tot = phi 
            smf_plot.sub.fill_between(mass, np.zeros(len(phi)), phi, color=pretty_colors[isnap], label='Nsnap='+str(isnap))

    smf_plot.set_axes() 
    fig_file = ''.join([
        'figure/test/',
        'LineageFinalDescendantSMF', 
        '.ancestor', str(nsnap_ancestor),
        bloodline._file_spec(subhalo_prop=subhalo_prop, sfr_prop=sfr_prop), 
        '.png'])
    smf_plot.save_fig(fig_file)

    return None
Esempio n. 14
0
    def remap_snps(self):
        # SNPs already remapped
        if len(self.snps.filter(generated_by_lineage=True)) == 3:
            return

        if len(self.snps.filter(generated_by_lineage=True)) == 1:
            snps = self.snps.filter(generated_by_lineage=True).get()
        else:
            # TODO: merge SNPs here, but for now just get canonical SNPs; assume Build 37
            snps = self.get_canonical_snps()

        if not snps:
            return

        with tempfile.TemporaryDirectory() as tmpdir:
            l = Lineage(output_dir=tmpdir, parallelize=False)

            ind = l.create_individual("lineage_NCBI36", snps.file.path)
            ind.remap_snps(36, parallelize=False)
            file = ind.save_snps()

            summary_info, snps_is_valid = parse_snps(file)

            if snps_is_valid:
                summary_info["generated_by_lineage"] = True
                summary_info["merged"] = True
                self.add_snps(file, summary_info)

            ind = l.create_individual("lineage_GRCh38", snps.file.path)
            ind.remap_snps(38, parallelize=False)
            file = ind.save_snps()

            summary_info, snps_is_valid = parse_snps(file)

            if snps_is_valid:
                summary_info["generated_by_lineage"] = True
                summary_info["merged"] = True
                self.add_snps(file, summary_info)
def LineageAncestorSFMS(nsnap_ancestor, 
        subhalo_prop = {'scatter': 0.0, 'source': 'li-march'}, 
        sfr_prop = { 'fq': {'name': 'wetzelsmooth'}, 'sfr': {'name': 'average'}}
        ): 
    '''Plot the SF-MS of the lineage ancestor object. This is mainly to make sure 
    that the AssignSFR routine is working properly 
    '''
    # read in lineage
    bloodline = Lineage(nsnap_ancestor=nsnap_ancestor, subhalo_prop=subhalo_prop)
    bloodline.Read([1], sfr_prop=sfr_prop)
    ancestor = getattr(bloodline, 'ancestor')

    sfms_plot = plots.PlotSFMS()
    sfms_plot.cenque(ancestor, justsf=True)   # lineage ancestor
    sfms_plot.param_sfms(nsnap=nsnap_ancestor) 

    sfms_plot_file = ''.join([
        'figure/test/', 
        'LineageAncestorSFMS', str(nsnap_ancestor), 
        bloodline._file_spec(subhalo_prop=subhalo_prop, sfr_prop=sfr_prop),
        '.png'])
    sfms_plot.save_fig(sfms_plot_file)
    return None
def predict(addr, model_name, input_lin, batch=False):
    url = "http://%s/%s/predict" % (addr, model_name)
    if batch:
        req_json = json.dumps({'input_batch': [[x.val for x in input_lin]]})
    else:
        req_json = json.dumps({'input': [input_lin.val]})
    headers = {'Content-type': 'application/json'}
    r = requests.post(url, headers=headers, data=req_json)
    # print(json.loads(r.text))
    str_r = json.loads(r.text)["output"].replace("[", "").replace("]",
                                                                  "").split()
    vals = [float(i) for i in str_r]
    new_lineage_objs = [
        Lineage.add_node(input_lin[i], model_name, vals[i])
        for i in range(len(input_lin))
    ]
    return new_lineage_objs
def LineageSMF(nsnap_ancestor, descendants=None, 
        subhalo_prop={'scatter': 0.0, 'source': 'li-march'}, 
        sfr_prop={'fq': {'name': 'wetzelsmooth'}, 'sfr': {'name': 'average'}}): 
    ''' Plot the SMF of lineage galaxy population (both ancestor and descendants).
    Compare the lineage SMF to the central subhalo and analytic SMFs for all 
    subhalos. The main agreement, is between the lineage SMF and the central subhalo
    SMF. If nsnap_ancestor is high, then there should be more discrepancy 
    between LIneage SMF and CentralSubhalos SMF because more galaxies are lost. 
    '''
    # read in desendants from the lineage object
    if descendants is None: 
        descendants = range(1, nsnap_ancestor)
    bloodline = Lineage(nsnap_ancestor=nsnap_ancestor, subhalo_prop=subhalo_prop)
    bloodline.Read(descendants) 
    
    smf_plot = plots.PlotSMF()
    # plot ancestor against the analytic SMF 
    ancestor = getattr(bloodline, 'ancestor')
    #smf_plot.cenque(ancestor)   # SMF of lineage ancestor
    #smf_plot.analytic(
    #        ancestor.zsnap, 
    #        source=ancestor.subhalo_prop['source'], 
    #        line_style='-', lw=1,
    #        label='All Subhalos')

    #smf = SMF()
    #subh = CentralSubhalos()
    #subh.Read(
    #        nsnap_ancestor, 
    #        scatter=ancestor.subhalo_prop['scatter'], 
    #        source=ancestor.subhalo_prop['source'])
    #subh_mass, subh_phi = smf.centralsubhalos(subh)
    #smf_plot.sub.plot(subh_mass, subh_phi, lw=4, ls='--', c='gray', label='Central Subhalos')

    for isnap in descendants: 
        descendant = getattr(bloodline, 'descendant_snapshot'+str(isnap))
        smf = SMF()
        subh = CentralSubhalos()
        subh.Read(
                isnap, 
                scatter=descendant.subhalo_prop['scatter'], 
                source=descendant.subhalo_prop['source'])
        subh_mass, subh_phi = smf.Obj(subh)
        smf_plot.sub.plot(subh_mass, subh_phi, lw=4, ls='--', c='gray')
        #print (subh_phi - d_phi)/d_phi

        #smf_plot.analytic(
        #        descendant.zsnap, 
        #        source=descendant.subhalo_prop['source'], 
        #        line_style='-', lw=1,
        #        label=None)

    smf_plot.set_axes()
    plt.show()
    smf_plot_file = ''.join([
        'figure/test/', 
        'LineageSMF', 
        '.ancestor', str(nsnap_ancestor), 
        bloodline._file_spec(subhalo_prop=bloodline.subhalo_prop, sfr_prop=bloodline.sfr_prop), 
        '.png'
        ])
    smf_plot.save_fig(smf_plot_file)
    return None
Esempio n. 18
0
 def setUp(self):
     
     self.biodb_selector= s= Selector("ncbi")
     #self.feature= self.biodb_selector.getFeatureByID(781)
     self.feature= self.biodb_selector.getFeatureByID(89)
     self.lineage = Lineage(self.feature, self.biodb_selector)
def DescendantSFMS_composition(nsnap_descendant,
                               abc_step=29,
                               bovyplot=False,
                               **sfinh_kwargs):
    ''' SFMS for the SF Inherited Descendant galaxy population  
    '''
    sfinherit_file = InheritSF_file(nsnap_descendant,
                                    abc_step=abc_step,
                                    **sfinh_kwargs)
    bloodline = Lineage(nsnap_ancestor=sfinh_kwargs['nsnap_ancestor'],
                        subhalo_prop=sfinh_kwargs['subhalo_prop'])
    bloodline.Read([nsnap_descendant], filename=sfinherit_file)

    descendant = getattr(bloodline,
                         'descendant_snapshot' + str(nsnap_descendant))
    if not bovyplot:
        sfms_plot = descendant.plotSFMS(bovyplot=False, scatter=False)

        # SMF composition based on their initial mass at nsnap_ancestor
        started_here = np.where(
            descendant.nsnap_genesis == sfinh_kwargs['nsnap_ancestor'])
        start_mass = descendant.mass_genesis[started_here]

        mass_bins = np.arange(7., 12.5, 0.5)
        mass_bin_low = mass_bins[:-1]
        mass_bin_high = mass_bins[1:]

        for i_m in range(len(mass_bin_low)):

            mbin = np.where((start_mass > mass_bin_low[i_m])
                            & (start_mass <= mass_bin_high[i_m]))

            sfms_plot.plot(
                mass=descendant.mass[started_here[0][mbin]],
                sfr=descendant.sfr[started_here[0][mbin]],
                sfr_class=descendant.sfr_class[started_here[0][mbin]],
                gal_class='quiescent',
                bovyplot=False,
                sigSFR=False,
                color=i_m,
                label=r"$\mathtt{M_{*,i}=}$" +
                str(round(mass_bin_low[i_m], 2)) + "-" +
                str(round(mass_bin_high[i_m], 2)))

        qfrac = Fq()
        m_arr = np.arange(9.0, 12.5, 0.5)
        sfms_plot.sub.plot(m_arr,
                           qfrac.SFRcut(
                               m_arr,
                               descendant.zsnap,
                               sfms_prop=(sfinh_kwargs['sfr_prop'])['sfms']),
                           c='k',
                           ls='--',
                           lw=4)
        bovyplot_str = ''
    else:
        sfms_plot = descendant.plotSFMS(bovyplot=True)
        bovyplot_str = '.bovy'

    sfms_plot.sub.legend(loc='lower right', scatterpoints=1)
    fig_file = ''.join([
        'figure/test/', 'SFMS_composition.', '.'.join(
            (sfinherit_file.rsplit('/')[-1]).rsplit('.')[:-1]), bovyplot_str,
        '.png'
    ])
    sfms_plot.save_fig(fig_file)
    return None
Esempio n. 20
0
    subprocess.call([
        'tar', '-xvf', 'dat/observations/primussdss.tar', '-C',
        'dat/observations/'
    ])
print 'Building Group Catalog hdf5 files'
[Obv.BuildGroupCat(Mrcut=Mr, position='central') for Mr in [18, 19, 20]]

# subhalos
subhalo_file = 'subhalo_sham.central.snapshot1.ancestor15.scatter0.0.li-march.hdf5'
if not os.path.exists('dat/wetzel_tree/' + subhalo_file):
    subprocess.call([
        'wget',
        'http://physics.nyu.edu/~chh327/data/subhalo_sham.ancestor15.li-march.tar',
        '-P', 'dat/wetzel_tree/'
    ])
    subprocess.call([
        'tar', '-xvf', 'dat/wetzel_tree/subhalo_sham.ancestor15.li-march.tar',
        '-C', 'dat/wetzel_tree/'
    ])

# lineage
for scat in [0.0, 0.2]:
    bloodline = Lineage(nsnap_ancestor=15,
                        subhalo_prop={
                            'scatter': scat,
                            'source': 'li-march'
                        },
                        clobber=True)
    bloodline.Descend(clobber=True)
    bloodline.Write()
def DescendantSMF_composition(nsnap_descendant,
                              mass_evol=True,
                              nsnap_ancestor=20,
                              abc_step=29,
                              subhalo_prop={
                                  'scatter': 0.2,
                                  'source': 'li-march'
                              },
                              sfr_prop={
                                  'fq': {
                                      'name': 'wetzelsmooth'
                                  },
                                  'sfr': {
                                      'name': 'average'
                                  }
                              },
                              evol_prop={
                                  'sfr': {
                                      'dutycycle': {
                                          'name': 'notperiodic'
                                      }
                                  },
                                  'mass': {
                                      'name': 'sham'
                                  }
                              }):

    sf_inherit_file = InheritSF_file(nsnap_descendant,
                                     nsnap_ancestor=nsnap_ancestor,
                                     abc_step=abc_step,
                                     subhalo_prop=subhalo_prop,
                                     sfr_prop=sfr_prop,
                                     evol_prop=evol_prop)

    bloodline = Lineage(nsnap_ancestor=nsnap_ancestor,
                        subhalo_prop=subhalo_prop)
    bloodline.Read([nsnap_descendant], filename=sf_inherit_file)

    descendant = getattr(bloodline,
                         'descendant_snapshot' + str(nsnap_descendant))
    smf_plot = descendant.plotSMF()

    mf = SMF()

    if mass_evol:
        # SMF composition based on their initial mass at nsnap_ancestor
        started_here = np.where(descendant.nsnap_genesis == nsnap_ancestor)
        start_mass = descendant.mass_genesis[started_here]

        mass_bins = np.arange(7., 12.5, 0.5)
        mass_bin_low = mass_bins[:-1]
        mass_bin_high = mass_bins[1:]

        for i_m in range(len(mass_bin_low)):

            mbin = np.where((start_mass > mass_bin_low[i_m])
                            & (start_mass <= mass_bin_high[i_m]))

            mass, phi = mf._smf(descendant.mass[started_here[0][mbin]])

            try:
                phi_tot += phi
                smf_plot.sub.fill_between(
                    mass,
                    phi_tot - phi,
                    phi_tot,
                    color=pretty_colors[i_m],
                    label=str(round(mass_bin_low[i_m], 2)) + '-' +
                    str(round(mass_bin_high[i_m], 2)))
            except UnboundLocalError:
                phi_tot = phi
                smf_plot.sub.fill_between(
                    mass,
                    np.zeros(len(phi)),
                    phi,
                    color=pretty_colors[i_m],
                    label=str(round(mass_bin_low[i_m], 2)) + '-' +
                    str(round(mass_bin_high[i_m], 2)))
    else:
        for isnap in range(2, nsnap_ancestor + 1)[::-1]:
            started_here = np.where(descendant.nsnap_genesis == isnap)
            mass, phi = mf._smf(descendant.mass[started_here])

            try:
                phi_tot += phi
                smf_plot.sub.fill_between(mass,
                                          phi_tot - phi,
                                          phi_tot,
                                          color=pretty_colors[isnap],
                                          label='Nsnap=' + str(isnap))
            except UnboundLocalError:
                phi_tot = phi
                smf_plot.sub.fill_between(mass,
                                          np.zeros(len(phi)),
                                          phi,
                                          color=pretty_colors[isnap],
                                          label='Nsnap=' + str(isnap))
    smf_plot.set_axes()
    mass_evol_str = ''
    if mass_evol:
        mass_evol_str = '.mass_evol'

    fig_file = ''.join([
        'figure/test/', ''.join(
            (sf_inherit_file.rsplit('/')[-1]).rsplit('.')[:-1]),
        '.SMF_composition', mass_evol_str, '.png'
    ])
    smf_plot.save_fig(fig_file)
    return None
def Compare_DescendantSMF_composition(nsnap_descendant, kwarg_list, n_step=29):
    ''' Compare the M* distribution at z_final of galaxies from same initial 
    M* bin for different prescriptions of SF evolution. More specifically
    this is to see how well the integrated SF stellar masses match the SHAM 
    stellar masses. 
    '''
    descendants = []
    for kwarg in kwarg_list:  # import SF inherited Lineages for different set of arguments
        sf_inherit_file = InheritSF_file(nsnap_descendant,
                                         abc_step=n_step,
                                         **kwarg)
        bloodline = Lineage(nsnap_ancestor=kwarg['nsnap_ancestor'],
                            subhalo_prop=kwarg['subhalo_prop'])
        bloodline.Read([nsnap_descendant], filename=sf_inherit_file)

        descendants.append(
            getattr(bloodline, 'descendant_snapshot' + str(nsnap_descendant)))

    mf = SMF()

    mass_bins = np.arange(7., 12.5, 0.5)
    mass_bin_low = mass_bins[:-1]
    mass_bin_high = mass_bins[1:]

    prettyplot()
    pretty_colors = prettycolors()
    lstyles = ['-', '--', '-.']
    for i_m in range(len(mass_bin_low)):
        fig = plt.figure(1)
        sub = fig.add_subplot(111)
        mass_evol_str = ''
        for i_desc, descendant in enumerate(descendants):
            # SMF composition based on their initial mass at nsnap_ancestor
            started_here = np.where(descendant.nsnap_genesis ==
                                    kwarg_list[i_desc]['nsnap_ancestor'])
            start_mass = descendant.mass_genesis[started_here]

            mbin = np.where((start_mass > mass_bin_low[i_m])
                            & (start_mass <= mass_bin_high[i_m]))

            mass, phi = mf._smf(descendant.mass[started_here[0][mbin]])

            sub.plot(mass,
                     phi,
                     c=pretty_colors[i_desc + 1],
                     lw=4,
                     ls=lstyles[i_desc],
                     label=(kwarg_list[i_desc])['evol_prop']['mass']['name'] +
                     ' Masses')
            mass_evol_str += (kwarg_list[i_desc])['evol_prop']['mass']['name']
        sub.set_xlim([7.5, 12.0])
        sub.set_ylim([0.0, 0.012])
        sub.set_ylabel(r'$\Phi$', fontsize=25)
        sub.set_xlabel(r'$\mathtt{log\;M_*}$', fontsize=25)
        sub.legend(loc='upper right')
        sub.set_title(''.join([
            'log M* = ',
            str(round(mass_bin_low[i_m], 2)), '-',
            str(round(mass_bin_high[i_m], 2)), ' at Snapshot ',
            str(kwarg_list[i_desc]['nsnap_ancestor'])
        ]),
                      fontsize=25)

        fig_file = ''.join([
            'figure/test/', 'DescendantSMF_composition', '.massevol_',
            mass_evol_str, '.Mbin',
            str(round(mass_bin_low[i_m], 2)), '_',
            str(round(mass_bin_high[i_m], 2)), '.png'
        ])
        fig.savefig(fig_file, bbox_inches='tight')
        plt.close()
    return None
Esempio n. 23
0
    def find_discordant_snps(self, progress_recorder=None):
        ind1_snps = self.individual1.get_canonical_snps()
        ind2_snps = self.individual2.get_canonical_snps()

        if not ind1_snps or not ind2_snps:
            self.delete()
            return

        if self.individual3:
            ind3_snps = self.individual3.get_canonical_snps()

            if not ind3_snps:
                self.delete()
                return

        with tempfile.TemporaryDirectory() as tmpdir:
            l = Lineage(output_dir=tmpdir, parallelize=False)

            ind1_snps_file = shutil.copy(
                ind1_snps.file.path,
                os.path.join(tmpdir, "ind1_snps" + ind1_snps.file_ext),
            )

            ind2_snps_file = shutil.copy(
                ind2_snps.file.path,
                os.path.join(tmpdir, "ind2_snps" + ind2_snps.file_ext),
            )

            if self.individual3:
                ind3_snps_file = shutil.copy(
                    ind3_snps.file.path,
                    os.path.join(tmpdir, "ind3_snps" + ind3_snps.file_ext),
                )

            ind1 = l.create_individual(self.individual1.name, ind1_snps_file)
            ind2 = l.create_individual(self.individual2.name, ind2_snps_file)

            if self.individual3:
                ind3 = l.create_individual(self.individual3.name,
                                           ind3_snps_file)
            else:
                ind3 = None

            discordant_snps = l.find_discordant_snps(ind1,
                                                     ind2,
                                                     ind3,
                                                     save_output=True)

            self.total_discordant_snps = len(discordant_snps)

            for root, dirs, files in os.walk(tmpdir):
                for file in files:
                    file_path = os.path.join(root, file)
                    if "discordant_snps" in file:
                        self.discordant_snps_csv.name = get_relative_user_dir_file(
                            self.user.uuid, uuid4())
                        compress_file(file_path, self.discordant_snps_csv.path)

                        self.discordant_snps_pickle = get_relative_user_dir_file(
                            self.user.uuid, uuid4(), ".pkl.gz")
                        discordant_snps.to_pickle(
                            self.discordant_snps_pickle.path)

                        break

        self.setup_complete = True
        self.save()
def DescendantHMF_composition(nsnap_descendant,
                              nsnap_ancestor=20,
                              n_step=29,
                              subhalo_prop={
                                  'scatter': 0.2,
                                  'source': 'li-march'
                              },
                              sfr_prop={
                                  'fq': {
                                      'name': 'wetzelsmooth'
                                  },
                                  'sfms': {
                                      'name': 'linear'
                                  }
                              },
                              evol_prop={
                                  'sfr': {
                                      'dutycycle': {
                                          'name': 'notperiodic'
                                      }
                                  },
                                  'mass': {
                                      'name': 'sham'
                                  }
                              }):
    ''' Halo Mass Function composition at z = z_final by initial halo mass at 
    nsnap_ancestor. 
    '''
    # import SF inherited Lineage
    sf_inherit_file = InheritSF_file(nsnap_descendant,
                                     nsnap_ancestor=nsnap_ancestor,
                                     abc_step=n_step,
                                     subhalo_prop=subhalo_prop,
                                     sfr_prop=sfr_prop,
                                     evol_prop=evol_prop)
    bloodline = Lineage(nsnap_ancestor=nsnap_ancestor,
                        subhalo_prop=subhalo_prop)
    bloodline.Read([nsnap_descendant], filename=sf_inherit_file)
    # descendant
    descendant = getattr(bloodline,
                         'descendant_snapshot' + str(nsnap_descendant))
    smf_plot = PlotSMF()
    mf = SMF()

    # SMF composition based on their initial mass at nsnap_ancestor
    started_here = np.where(descendant.nsnap_genesis == nsnap_ancestor)
    start_mass = descendant.halomass_genesis[started_here]

    mass_bins = np.arange(10., 17.5, 0.5)
    mass_bin_low = mass_bins[:-1]
    mass_bin_high = mass_bins[1:]

    for i_m in range(len(mass_bin_low)):

        mbin = np.where((start_mass > mass_bin_low[i_m])
                        & (start_mass <= mass_bin_high[i_m]))

        mass, phi = mf._smf(descendant.halo_mass[started_here[0][mbin]],
                            m_arr=np.arange(10., 17.1, 0.1))

        try:
            phi_tot += phi
            smf_plot.sub.fill_between(mass,
                                      phi_tot - phi,
                                      phi_tot,
                                      color=pretty_colors[i_m],
                                      label=str(round(mass_bin_low[i_m], 2)) +
                                      '-' + str(round(mass_bin_high[i_m], 2)))
        except UnboundLocalError:
            phi_tot = phi
            smf_plot.sub.fill_between(mass,
                                      np.zeros(len(phi)),
                                      phi,
                                      color=pretty_colors[i_m],
                                      label=str(round(mass_bin_low[i_m], 2)) +
                                      '-' + str(round(mass_bin_high[i_m], 2)))

    #smf_plot.set_axes()
    smf_plot.sub.set_yscale('log')
    smf_plot.sub.set_xlim([10.0, 17.0])
    smf_plot.sub.legend(loc='upper right')
    mass_evol_str = '.mass_evol'

    fig_file = ''.join([
        'figure/test/', ''.join(
            (sf_inherit_file.rsplit('/')[-1]).rsplit('.')[:-1]),
        '.HMF_composition', mass_evol_str, '.png'
    ])
    smf_plot.save_fig(fig_file)
    return None
def InheritSF(nsnap_descendant,
              nsnap_ancestor=20,
              subhalo_prop={
                  'scatter': 0.0,
                  'source': 'li-march'
              },
              sfr_prop={
                  'fq': {
                      'name': 'wetzelsmooth'
                  },
                  'sfms': {
                      'name': 'linear',
                      'mslope': 0.55,
                      'zslope': 1.1
                  }
              },
              evol_prop={
                  'pq': {
                      'slope': 0.05,
                      'yint': 0.0
                  },
                  'tau': {
                      'name': 'line',
                      'fid_mass': 10.75,
                      'slope': -0.6,
                      'yint': 0.6
                  },
                  'sfr': {
                      'dutycycle': {
                          'name': 'notperiodic'
                      }
                  },
                  'mass': {
                      'name': 'sham'
                  }
              }):
    ''' Evolve star formation properties of ancestor CentralGalaxyPopulation class within 
    the Lineage Class to descendant CentralGalaxlyPopulation object

    Parameters
    ----------
    nsnap_ancestor : int
        Snapshot number of ancestor CGPop object. The ancestor object is constructed 
        with from subhalo catalog with subhalo_prop properties. They are also assigned
        SFRs with sfr_prop properties. 
    subhalo_prop : dict
        Dictionary that describes the subhalo properties. The key 'scatter' corresponds
        to the M*-M_halo relation. The key 'soruce' describes the source SMF used for 
        the SHAM masses.
    sfr_prop : dict
        Dictionary that describes the SFR properties assigned to the ancestor CenQue object. 
        The key 'fq' describes the quiescent fraction used for the ancestor while the key
        'sfr' describes the properties of the SFR assignment. 
    evol_prop : dict
        Dictionary that consists of dictionaries which each describe paramter choices in 
        the model. 
        - evol_prop['pq'] dictates the quenching properties. 
        - evol_prop['tau'] dictates the quenching timescale. 
        - evol_prop['sfr'] dictates the SFR evolution. 
        - evol_prop['mass'] dictates the mass evolution. 
    '''
    # make sure that snapshot = 1 is included among imported descendants
    # and the first element of the list
    if isinstance(nsnap_descendant, list):
        raise ValueError('nsnap_descendant arg has to be an int')

    # evolution properties
    pq_prop = evol_prop['pq']
    tau_prop = evol_prop['tau']
    sfrevol_prop = evol_prop['sfr']
    massevol_prop = evol_prop['mass']

    # read in the lineage (< 0.05 seconds for one snapshot)
    read_time = time.time()
    bloodline = Lineage(nsnap_ancestor=nsnap_ancestor,
                        subhalo_prop=subhalo_prop)
    bloodline.Read(range(nsnap_descendant, nsnap_ancestor), sfr_prop='default')
    if 'subhalogrowth' in sfr_prop.keys():
        sfr_prop['subhalogrowth']['nsnap_descendant'] = nsnap_descendant
    bloodline.AssignSFR_ancestor(sfr_prop=sfr_prop)
    print 'Lineage Read Time = ', time.time() - read_time

    ancestor = bloodline.ancestor  # ancestor object
    t_init = ancestor.t_cosmic
    z_init = ancestor.zsnap
    # descendant object
    descendant = getattr(bloodline,
                         'descendant_snapshot' + str(nsnap_descendant))
    t_final = descendant.t_cosmic
    z_final = descendant.zsnap
    print 'Evolve until z = ', z_final

    # initialize SF properties of descendant
    n_descendant = len(descendant.snap_index)
    descendant.sfr = np.repeat(-999., n_descendant)
    descendant.ssfr = np.repeat(-999., n_descendant)
    descendant.min_ssfr = np.repeat(-999., n_descendant)
    descendant.tau = np.repeat(-999., n_descendant)
    descendant.sfr_class = np.chararray(n_descendant, itemsize=16)
    descendant.sfr_class[:] = ''

    succession, will = intersection_index(
        getattr(descendant, 'ancestor' + str(nsnap_ancestor)),
        ancestor.snap_index)
    if len(succession) != len(descendant.mass):
        raise ValueError('Something wrong with the lineage')
    q_ancestors = np.where(
        ancestor.sfr_class[will] == 'quiescent')[0]  # Q ancestors
    sf_ancestors = np.where(
        ancestor.sfr_class[will] == 'star-forming')[0]  # SF ancestors

    # Evolve queiscent ancestor galaxies
    q_time = time.time()
    descendant = _QuiescentEvol(ancestor,
                                descendant,
                                succession=succession,
                                will=will)
    print 'Quiescent evolution takes ', time.time() - q_time
    # Evolve Star Forming Galaxies
    sf_time = time.time()
    _StarformingEvol(ancestor,
                     descendant,
                     succession=succession,
                     will=will,
                     evol_prop=evol_prop,
                     sfr_prop=sfr_prop,
                     lineage=bloodline)
    print 'Star Forming Evolution takes ', time.time() - sf_time

    # Deal with over quenched galaxies
    overquenched = np.where(descendant.min_ssfr[succession[sf_ancestors]] >
                            descendant.ssfr[succession[sf_ancestors]])
    if len(overquenched[0]) > 0:
        descendant.ssfr[succession[
            sf_ancestors[overquenched]]] = descendant.min_ssfr[succession[
                sf_ancestors[overquenched]]]
        descendant.sfr[succession[sf_ancestors[overquenched]]] = descendant.ssfr[succession[sf_ancestors[overquenched]]] \
                + descendant.mass[succession[sf_ancestors[overquenched]]]
        #descendant.tau[succession[sf_ancestors[overquenched]]] = -999.

    descendant.data_columns = list(descendant.data_columns) + [
        'ssfr', 'sfr', 'min_ssfr', 'sfr_class'
    ]  #, 'tau'])
    setattr(bloodline, 'descendant_snapshot' + str(nsnap_descendant),
            descendant)

    return bloodline
Esempio n. 26
0
def LineageSMF(nsnap_ancestor,
               descendants=None,
               subhalo_prop={
                   'scatter': 0.0,
                   'source': 'li-march'
               },
               sfr_prop={
                   'fq': {
                       'name': 'wetzelsmooth'
                   },
                   'sfr': {
                       'name': 'average'
                   }
               }):
    ''' Plot the SMF of lineage galaxy population (both ancestor and descendants).
    Compare the lineage SMF to the central subhalo and analytic SMFs for all 
    subhalos. The main agreement, is between the lineage SMF and the central subhalo
    SMF. If nsnap_ancestor is high, then there should be more discrepancy 
    between LIneage SMF and CentralSubhalos SMF because more galaxies are lost. 
    '''
    # read in desendants from the lineage object
    if descendants is None:
        descendants = range(1, nsnap_ancestor)
    bloodline = Lineage(nsnap_ancestor=nsnap_ancestor,
                        subhalo_prop=subhalo_prop)
    bloodline.Read(descendants)

    smf_plot = plots.PlotSMF()
    # plot ancestor against the analytic SMF
    ancestor = getattr(bloodline, 'ancestor')
    #smf_plot.cenque(ancestor)   # SMF of lineage ancestor
    #smf_plot.analytic(
    #        ancestor.zsnap,
    #        source=ancestor.subhalo_prop['source'],
    #        line_style='-', lw=1,
    #        label='All Subhalos')

    #smf = SMF()
    #subh = CentralSubhalos()
    #subh.Read(
    #        nsnap_ancestor,
    #        scatter=ancestor.subhalo_prop['scatter'],
    #        source=ancestor.subhalo_prop['source'])
    #subh_mass, subh_phi = smf.centralsubhalos(subh)
    #smf_plot.sub.plot(subh_mass, subh_phi, lw=4, ls='--', c='gray', label='Central Subhalos')

    for isnap in descendants:
        descendant = getattr(bloodline, 'descendant_snapshot' + str(isnap))
        smf = SMF()
        subh = CentralSubhalos()
        subh.Read(isnap,
                  scatter=descendant.subhalo_prop['scatter'],
                  source=descendant.subhalo_prop['source'])
        subh_mass, subh_phi = smf.Obj(subh)
        smf_plot.sub.plot(subh_mass, subh_phi, lw=4, ls='--', c='gray')
        #print (subh_phi - d_phi)/d_phi

        #smf_plot.analytic(
        #        descendant.zsnap,
        #        source=descendant.subhalo_prop['source'],
        #        line_style='-', lw=1,
        #        label=None)

    smf_plot.set_axes()
    plt.show()
    smf_plot_file = ''.join([
        'figure/test/', 'LineageSMF', '.ancestor',
        str(nsnap_ancestor),
        bloodline._file_spec(subhalo_prop=bloodline.subhalo_prop,
                             sfr_prop=bloodline.sfr_prop), '.png'
    ])
    smf_plot.save_fig(smf_plot_file)
    return None
Esempio n. 27
0
#!/usr/local/bin/python3.8
import sys
import logging, sys

logger = logging.getLogger()
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler(sys.stdout))
from lineage import Lineage

l = Lineage(output_dir='storage/app/dna/output')

var1 = sys.argv[1]
var2 = sys.argv[2]

file1 = "storage/app/dna/" + sys.argv[3]
file2 = "storage/app/dna/" + sys.argv[4]

user662 = l.create_individual(var1, file1)
user663 = l.create_individual(var2, file2)
discordant_snps = l.find_discordant_snps(user662, user663, save_output=True)
len(discordant_snps.loc[discordant_snps['chrom'] != 'MT'])
results = l.find_shared_dna([user662, user663],
                            cM_threshold=0.75,
                            snp_threshold=1100)
Esempio n. 28
0
class TestSnps(BaseLineageTestCase):
    def setUp(self):
        self.l = Lineage()
        self.snps_GRCh38 = SNPs("tests/input/GRCh38.csv")
        self.snps = SNPs("tests/input/chromosomes.csv")
        self.snps_none = SNPs(None)
        self.del_output_dir_helper()

    def snps_discrepant_pos(self):
        return self.create_snp_df(rsid=["rs3094315"],
                                  chrom=["1"],
                                  pos=[1],
                                  genotype=["AA"])

    def test_assembly(self):
        assert self.snps_GRCh38.assembly == "GRCh38"

    def test_assembly_no_snps(self):
        assert self.snps_none.assembly == ""

    def test_snp_count(self):
        assert self.snps.snp_count == 6

    def test_snp_count_no_snps(self):
        assert self.snps_none.snp_count == 0

    def test_chromosomes(self):
        assert self.snps.chromosomes == ["1", "2", "3", "5", "PAR", "MT"]

    def test_chromosomes_no_snps(self):
        assert self.snps_none.chromosomes == []

    def test_chromosomes_summary(self):
        assert self.snps.chromosomes_summary == "1-3, 5, PAR, MT"

    def test_chromosomes_summary_no_snps(self):
        assert self.snps_none.chromosomes_summary == ""

    def test_build_no_snps(self):
        assert self.snps_none.build is None

    def test_build_detected_no_snps(self):
        assert not self.snps_none.build_detected

    def test_build_detected_PAR_snps(self):
        if os.getenv("DOWNLOADS_ENABLED"):
            snps = SNPs("tests/input/GRCh37_PAR.csv")
            assert snps.build == 37
            assert snps.build_detected

    def test_sex_no_snps(self):
        assert self.snps_none.sex == ""

    def test_sex_Male_Y_chrom(self):
        ind = self.simulate_snps(
            self.l.create_individual("test_snps_sex_Male_Y_chrom"),
            chrom="Y",
            pos_start=1,
            pos_max=59373566,
            pos_step=10000,
        )
        file = ind.save_snps()
        from lineage.snps import SNPs

        snps = SNPs(file)
        assert snps.sex == "Male"

    def test_get_summary(self):
        assert self.snps_GRCh38.get_summary() == {
            "source": "generic",
            "assembly": "GRCh38",
            "build": 38,
            "build_detected": True,
            "snp_count": 4,
            "chromosomes": "1, 3",
            "sex": "",
        }

    def test_get_summary_no_snps(self):
        assert self.snps_none.get_summary() is None

    def test_is_valid_True(self):
        assert self.snps_GRCh38.is_valid()

    def test_is_valid_False(self):
        assert not self.snps_none.is_valid()

    def test__read_raw_data(self):
        assert self.snps_none.snps is None
        assert self.snps_none.source == ""

    def test__lookup_build_with_snp_pos_None(self):
        snps = SNPs()
        snps._snps = self.snps_discrepant_pos()
        assert snps.detect_build() is None

    def test_get_assembly_None(self):
        snps = SNPs()
        snps._build = None
        assert snps.get_assembly() is ""
Esempio n. 29
0
 def setUp(self):
     self.l = Lineage()
     self.del_output_dir_helper()
Esempio n. 30
0
    def find_shared_dna_genes(self, progress_recorder=None):
        ind1_snps = self.individual1.get_canonical_snps()
        ind2_snps = self.individual2.get_canonical_snps()

        if not ind1_snps or not ind2_snps:
            self.delete()
            return

        with tempfile.TemporaryDirectory() as tmpdir:
            l = Lineage(output_dir=tmpdir, parallelize=False)

            ind1_snps_file = shutil.copy(
                ind1_snps.file.path,
                os.path.join(tmpdir, "ind1_snps" + ind1_snps.file_ext),
            )

            ind2_snps_file = shutil.copy(
                ind2_snps.file.path,
                os.path.join(tmpdir, "ind2_snps" + ind2_snps.file_ext),
            )

            ind1 = l.create_individual(self.individual1.name, ind1_snps_file)
            ind2 = l.create_individual(self.individual2.name, ind2_snps_file)

            shared_dna_one_chrom, shared_dna_two_chrom, shared_genes_one_chrom, shared_genes_two_chrom = l.find_shared_dna(
                ind1,
                ind2,
                cM_threshold=float(self.cM_threshold),
                snp_threshold=int(self.snp_threshold),
                shared_genes=True,
                save_output=True,
            )

            self.total_shared_segments_one_chrom = len(shared_dna_one_chrom)
            self.total_shared_segments_two_chrom = len(shared_dna_two_chrom)
            self.total_shared_cMs_one_chrom = Decimal(
                shared_dna_one_chrom["cMs"].sum())
            self.total_shared_cMs_two_chrom = Decimal(
                shared_dna_two_chrom["cMs"].sum())
            self.total_snps_one_chrom = shared_dna_one_chrom["snps"].sum()
            self.total_snps_two_chrom = shared_dna_two_chrom["snps"].sum()
            self.total_chrom_one_chrom = len(
                shared_dna_one_chrom["chrom"].unique())
            self.total_chrom_two_chrom = len(
                shared_dna_two_chrom["chrom"].unique())
            self.total_shared_genes_one_chrom = len(shared_genes_one_chrom)
            self.total_shared_genes_two_chrom = len(shared_genes_two_chrom)

            for root, dirs, files in os.walk(tmpdir):
                for file in files:
                    file_path = os.path.join(root, file)
                    if ".png" in file:
                        self.shared_dna_plot_png.name = get_relative_user_dir_file(
                            self.user.uuid, uuid4(), ".png")
                        shutil.move(file_path, self.shared_dna_plot_png.path)
                        os.chmod(self.shared_dna_plot_png.path, 0o640)

                    elif "shared_dna_one_chrom" in file:
                        self.shared_dna_one_chrom_csv = get_relative_user_dir_file(
                            self.user.uuid, uuid4())
                        compress_file(file_path,
                                      self.shared_dna_one_chrom_csv.path)

                        self.shared_dna_one_chrom_pickle = get_relative_user_dir_file(
                            self.user.uuid, uuid4(), ".pkl.gz")

                        shared_dna_one_chrom.to_pickle(
                            self.shared_dna_one_chrom_pickle.path)

                    elif "shared_genes_one_chrom" in file:
                        self.shared_genes_one_chrom_csv = get_relative_user_dir_file(
                            self.user.uuid, uuid4())
                        compress_file(file_path,
                                      self.shared_genes_one_chrom_csv.path)

                        self.shared_genes_one_chrom_pickle = get_relative_user_dir_file(
                            self.user.uuid, uuid4(), ".pkl.gz")

                        shared_genes_one_chrom.to_pickle(
                            self.shared_genes_one_chrom_pickle.path)

                    elif "shared_dna_two_chrom" in file:
                        self.shared_dna_two_chrom_csv = get_relative_user_dir_file(
                            self.user.uuid, uuid4())
                        compress_file(file_path,
                                      self.shared_dna_two_chrom_csv.path)

                        self.shared_dna_two_chrom_pickle = get_relative_user_dir_file(
                            self.user.uuid, uuid4(), ".pkl.gz")

                        shared_dna_two_chrom.to_pickle(
                            self.shared_dna_two_chrom_pickle.path)

                    elif "shared_genes_two_chrom" in file:
                        self.shared_genes_two_chrom_csv = get_relative_user_dir_file(
                            self.user.uuid, uuid4())
                        compress_file(file_path,
                                      self.shared_genes_two_chrom_csv.path)

                        self.shared_genes_two_chrom_pickle = get_relative_user_dir_file(
                            self.user.uuid, uuid4(), ".pkl.gz")

                        shared_genes_two_chrom.to_pickle(
                            self.shared_genes_two_chrom_pickle.path)

        self.setup_complete = True
        self.save()
Esempio n. 31
0
# print(output.val)
# print(output.graph)
# print(output.input_node)
# print(output.used)

# dot = Digraph(comment='Merge Example')
# for n in output.graph.nodes:
# 	dot.node(n)
# for e in output.graph.edges:
# 	dot.edge(e[0], e[1])
# dot.render('graphs/pipeline_merge.gv', view=True)

lineages = []
node_counts = {}
for i in range(20):
    rand_input = Lineage(np.random.random_sample())
    output = pipeline(rand_input, clipper_conn)
    lineages.append(output)
    for n in output.graph.nodes:
        if n in node_counts.keys():
            node_counts[n] += 1
        else:
            node_counts[n] = 1
    # total_counts = np.append(total_counts, counts)
    # print(output.graph.adj_list)

    # create a dict of counts instead of list

combined = Graph.merge_graphs([l for l in lineages])
print(node_counts)
Esempio n. 32
0
 def setUp(self):
     self.l = Lineage()
     self.snps_GRCh38 = SNPs("tests/input/GRCh38.csv")
     self.snps = SNPs("tests/input/chromosomes.csv")
     self.snps_none = SNPs(None)
     self.del_output_dir_helper()
def DescendantQAplot(nsnap_descendants, **sfinh_kwargs):
    ''' The ultimate QAplot t rule them all. 4 panels showing all the properties.
    '''
    sfinherit_file = InheritSF_file(nsnap_descendants, **sfinh_kwargs)
    bloodline = Lineage(nsnap_ancestor=sfinh_kwargs['nsnap_ancestor'],
                        subhalo_prop=sfinh_kwargs['subhalo_prop'])
    if not isinstance(nsnap_descendants, list):
        nsnap_descendants = [nsnap_descendants]
    bloodline.Read(nsnap_descendants, filename=sfinherit_file)

    for nsnap_descendant in nsnap_descendants:
        descendant = getattr(bloodline,
                             'descendant_snapshot' + str(nsnap_descendant))
        descendant.sfr_prop = sfinh_kwargs['sfr_prop']
        started_here = np.where(
            descendant.nsnap_genesis == sfinh_kwargs['nsnap_ancestor'])
        start_mass = descendant.mass_genesis[started_here]

        # Mass bins
        mass_bins = np.arange(7., 12.5, 0.5)
        mass_bin_low = mass_bins[:-1]
        mass_bin_high = mass_bins[1:]

        plt.close()
        prettyplot()
        fig = plt.figure(1, figsize=[25, 6])
        for i_sub in range(1, 5):
            sub_i = fig.add_subplot(1, 4, i_sub)

            if i_sub == 1:  # SMF
                mf = SMF()
                mass, phi = mf.Obj(descendant)

                sub_i.plot(mass,
                           phi,
                           lw=4,
                           c=pretty_colors[descendant.nsnap],
                           label=r'Simulated')

                censub = CentralSubhalos()
                censub.Read(descendant.nsnap,
                            scatter=sfinh_kwargs['subhalo_prop']['scatter'],
                            source=sfinh_kwargs['subhalo_prop']['source'],
                            nsnap_ancestor=sfinh_kwargs['nsnap_ancestor'])

                mass, phi = mf._smf(censub.mass)
                sub_i.plot(mass,
                           phi,
                           c='k',
                           lw=4,
                           ls='--',
                           label='Central Subhalos')

                sub_i.set_ylim([10**-5, 10**-1])
                sub_i.set_xlim([7.5, 12.0])
                plt.xticks([8., 9., 10., 11., 12.])
                sub_i.set_yscale('log')

                # x,y labels
                sub_i.set_xlabel(r'Mass $\mathtt{M_*}$', fontsize=25)
                sub_i.set_ylabel(r'Stellar Mass Function $\mathtt{\Phi}$',
                                 fontsize=25)
                sub_i.legend(loc='upper right', frameon=False)

            elif i_sub == 2:  # SFMS
                # SMF composition based on their initial mass at nsnap_ancestor
                for i_m in range(len(mass_bin_low)):
                    mbin = np.where((start_mass > mass_bin_low[i_m])
                                    & (start_mass <= mass_bin_high[i_m]))

                    sub_i.scatter(descendant.mass[started_here[0][mbin]],
                                  descendant.sfr[started_here[0][mbin]],
                                  color=pretty_colors[i_m],
                                  label=r"$\mathtt{M_{*,i}=}$" +
                                  str(round(mass_bin_low[i_m], 2)) + "-" +
                                  str(round(mass_bin_high[i_m], 2)))

                qfrac = Fq()
                m_arr = np.arange(9.0, 12.5, 0.5)
                sub_i.plot(m_arr,
                           qfrac.SFRcut(
                               m_arr,
                               descendant.zsnap,
                               sfms_prop=(sfinh_kwargs['sfr_prop'])['sfms']),
                           c='k',
                           ls='--',
                           lw=4)

                sub_i.set_xlim([9.0, 12.0])
                sub_i.set_ylim([-5.0, 2.0])
                sub_i.set_xlabel(r'$\mathtt{log\;M_*}$', fontsize=25)
                sub_i.set_ylabel(r'$\mathtt{log\;SFR}$', fontsize=25)

            elif i_sub == 3:  #SMHM
                for i_m in range(len(mass_bin_low)):
                    mbin = np.where((start_mass > mass_bin_low[i_m])
                                    & (start_mass <= mass_bin_high[i_m]))

                    sub_i.scatter(descendant.halo_mass[started_here[0][mbin]],
                                  descendant.mass[started_here[0][mbin]],
                                  color=pretty_colors[i_m],
                                  label=r"$\mathtt{M_{*,i}=}$" +
                                  str(round(mass_bin_low[i_m], 2)) + "-" +
                                  str(round(mass_bin_high[i_m], 2)))

                stellarmass = descendant.mass[started_here]
                halomass = descendant.halo_mass[started_here]
                mbin = np.arange(halomass.min(), halomass.max(), 0.25)
                mlow = mbin[:-1]
                mhigh = mbin[1:]

                muMstar = np.zeros(len(mlow))
                sigMstar = np.zeros(len(mlow))

                for im in range(len(mlow)):
                    mbin = np.where((halomass > mlow[im])
                                    & (halomass <= mhigh[im]))
                    muMstar[im] = np.mean(stellarmass[mbin])
                    sigMstar[im] = np.std(stellarmass[mbin])
                sub_i.errorbar(0.5 * (mlow + mhigh),
                               muMstar,
                               yerr=sigMstar,
                               color='k',
                               lw=3,
                               fmt='o',
                               capthick=2)

                sub_i.set_ylim([9.0, 12.0])
                sub_i.set_xlim([10.0, 15.0])
                sub_i.set_ylabel(r'Stellar Mass $\mathtt{M_*}$', fontsize=25)
                sub_i.set_xlabel(r'Halo Mass $\mathtt{M_{Halo}}$', fontsize=25)

                #sub_i.legend(loc='upper left', frameon=False, scatterpoints=1)
            elif i_sub == 4:  # Fq
                #mass, fq = descendant.Fq()
                sfq = qfrac.Classify(descendant.mass,
                                     descendant.sfr,
                                     descendant.zsnap,
                                     sfms_prop=descendant.sfr_prop['sfms'])
                gc = GroupCat(Mrcut=18, position='central')
                gc.Read()
                gc_sfq = qfrac.Classify(gc.mass,
                                        gc.sfr,
                                        np.mean(gc.z),
                                        sfms_prop=descendant.sfr_prop['sfms'])
                #sub_i.plot(mass, fq, color=pretty_colors[descendant.nsnap], lw=3, ls='--',
                #        label=r'$\mathtt{z =} '+str(descendant.zsnap)+'$')
                M_bin = np.array([9.7, 10.1, 10.5, 10.9, 11.3])
                M_low = M_bin[:-1]
                M_high = M_bin[1:]
                M_mid = 0.5 * (M_low + M_high)

                fq = np.zeros(len(M_low))
                gc_fq = np.zeros(len(M_low))
                for i_m in xrange(len(M_low)):
                    mlim = np.where((descendant.mass > M_low[i_m])
                                    & (descendant.mass <= M_high[i_m]))
                    gc_mlim = np.where((gc.mass > M_low[i_m])
                                       & (gc.mass <= M_high[i_m]))
                    ngal = np.float(len(mlim[0]))
                    gc_ngal = np.float(len(gc_mlim[0]))

                    if ngal != 0:  # no galaxy in mass bin
                        ngal_q = np.float(
                            len(np.where(sfq[mlim] == 'quiescent')[0]))
                        fq[i_m] = ngal_q / ngal
                    if gc_ngal != 0:
                        gc_ngal_q = np.float(
                            len(np.where(gc_sfq[gc_mlim] == 'quiescent')[0]))
                        gc_fq[i_m] = gc_ngal_q / gc_ngal

                sub_i.plot(M_mid,
                           fq,
                           color=pretty_colors[descendant.nsnap],
                           lw=3,
                           label=r'$\mathtt{z =} ' + str(descendant.zsnap) +
                           '$')

                fq_model = qfrac.model(
                    M_bin,
                    descendant.zsnap,
                    lit=sfinh_kwargs['sfr_prop']['fq']['name'])
                sub_i.plot(M_bin,
                           fq_model,
                           color='k',
                           lw=4,
                           ls='--',
                           label=sfinh_kwargs['sfr_prop']['fq']['name'])
                sub_i.scatter(M_mid,
                              gc_fq,
                              color='k',
                              s=100,
                              lw=0,
                              label='Group Catalog')

                sub_i.set_xlim([9.0, 12.0])
                sub_i.set_ylim([0.0, 1.0])

                sub_i.set_xlabel(r'Mass $\mathtt{M_*}$')
                sub_i.set_ylabel(r'Quiescent Fraction $\mathtt{f_Q}$',
                                 fontsize=20)

                sub_i.legend(loc='upper left',
                             frameon=False,
                             scatterpoints=1,
                             markerscale=0.75)

        plt.tight_layout(pad=0.4, w_pad=0.5, h_pad=1.0)
        fig_name = ''.join([
            'figure/test/', 'QAplot.', '.nsnap',
            str(nsnap_descendant), '.'.join(
                (sfinherit_file.rsplit('/')[-1]).rsplit('.')[:-1]), '.png'
        ])
        fig.savefig(fig_name, bbox_inches='tight')
        plt.close()
    return None
Esempio n. 34
0
def InheritSF(nsnap_descendant,
              nsnap_ancestor=20,
              subhalo_prop=None,
              sfr_prop=None,
              evol_prop=None,
              quiet=True):
    ''' Evolve star formation properties of 'ancestor' CentralGalaxyPopulation class at 
    redshift specified by nsnap_ancestor to descendant CentralGalaxlyPopulation object
    at redshift specified by nsnap_descendant. Both ancestor and descendant objects
    are attributes in the Lineage class, which contains all the 'lineage' information 
    i.e. all the halo tracking information. 

    Parameters
    ----------
    nsnap_ancestor : int
        Snapshot number of ancestor CGPop object. The ancestor object is constructed 
        with from subhalo catalog with subhalo_prop properties. They are also assigned
        SFRs with sfr_prop properties. 
    subhalo_prop : dict
        Dictionary that describes the subhalo properties. The key 'scatter' corresponds
        to the M*-M_halo relation. The key 'soruce' describes the source SMF used for 
        the SHAM masses.
    sfr_prop : dict
        Dictionary that describes the SFR properties assigned to the ancestor CenQue object. 
        The key 'fq' describes the quiescent fraction used for the ancestor while the key
        'sfr' describes the properties of the SFR assignment. 
    evol_prop : dict
        Dictionary that consists of dictionaries which each describe paramter choices in 
        the model. 
        - evol_prop['pq'] dictates the quenching properties. 
        - evol_prop['tau'] dictates the quenching timescale. 
        - evol_prop['sfr'] dictates the SFR evolution. 
        - evol_prop['mass'] dictates the mass evolution. 
    '''
    if isinstance(nsnap_descendant, list):
        d_list = True
    else:
        d_list = False
    # read in the lineage (< 0.05 seconds for one snapshot)
    #read_time = time.time()
    bloodline = Lineage(nsnap_ancestor=nsnap_ancestor,
                        subhalo_prop=subhalo_prop,
                        quiet=quiet)
    if not d_list:
        bloodline.Read(range(nsnap_descendant, nsnap_ancestor), quiet=quiet)
    else:
        bloodline.Read(range(np.min(nsnap_descendant), nsnap_ancestor),
                       quiet=quiet)
    if 'subhalogrowth' in sfr_prop.keys(
    ):  # depending on whether SFR assign includes subhalo growth AM
        sfr_prop['subhalogrowth']['nsnap_descendant'] = nsnap_descendant
    bloodline.AssignSFR_ancestor(sfr_prop=sfr_prop, quiet=quiet)
    #print 'Lineage Read Time = ', time.time() - read_time

    ancestor = bloodline.ancestor  # ancestor object
    t_init = ancestor.t_cosmic
    z_init = ancestor.zsnap
    ancestor.tQ[np.where(ancestor.tQ != 999.)] = 0.

    if not isinstance(nsnap_descendant, list):
        nsnap_descendant = [nsnap_descendant]

    descendants = []
    sf_ancestors, q_ancestors = [], []
    successions, wills = [], []
    for nd in nsnap_descendant:
        des = getattr(bloodline, 'descendant_snapshot' + str(nd))
        des._clean_initialize()  # initialize SF properties
        des.sfr_prop = ancestor.sfr_prop
        # match indices up with each other
        succession, will = intersection_index(
            getattr(des, 'ancestor' + str(nsnap_ancestor)),
            ancestor.snap_index)
        if len(succession) != len(des.mass):
            raise ValueError('Something wrong with the lineage')
        q_ancestor = np.where(
            ancestor.sfr_class[will] == 'quiescent')[0]  # Q ancestors
        sf_ancestor = np.where(
            ancestor.sfr_class[will] == 'star-forming')[0]  # SF ancestors
        if not quiet:
            print "nsnap_descendant = ", nd, "Ancestors: Nq = ", len(
                q_ancestor), ', Nsf = ', len(sf_ancestor)

        descendants.append(des)
        wills.append(will)
        successions.append(succession)
        q_ancestors.append(q_ancestor)
        sf_ancestors.append(sf_ancestor)

    # Evolve queiscent ancestor galaxies
    if not quiet:
        q_time = time.time()
    descendants = _QuiescentEvol(ancestor,
                                 descendants,
                                 successions=successions,
                                 wills=wills,
                                 q_ancestors=q_ancestors)
    if not quiet:
        print 'Quiescent evolution takes ', time.time() - q_time

    # Evolve Star Forming Galaxies
    if not quiet:
        sf_time = time.time()
    ################## PQ BASED DROPPED ##############################
    # if evol_prop['type'] == 'pq_based':
    #     _StarformingEvol_Pq(ancestor, descendant, succession=succession, will=will,
    #             evol_prop=evol_prop, sfr_prop=sfr_prop,
    #             lineage=bloodline)
    # elif evol_prop['type'] == 'simult':
    ################## PQ BASED DROPPED ##############################
    descendants = _StarformingEvol_SimulEvo(ancestor,
                                            descendants,
                                            successions=successions,
                                            wills=wills,
                                            evol_prop=evol_prop,
                                            sfr_prop=sfr_prop,
                                            lineage=bloodline,
                                            quiet=quiet)
    if not quiet:
        print 'Star Forming Evolution takes ', time.time() - sf_time

    descendants = _Overquenching(descendants,
                                 successions=successions,
                                 wills=wills,
                                 sf_ancestors=sf_ancestors)

    for descendant in descendants:
        descendant.data_columns = list(descendant.data_columns) + [
            'ssfr', 'sfr', 'min_ssfr', 'sfr_class'
        ]
        setattr(bloodline, 'descendant_snapshot' + str(descendant.nsnap),
                descendant)
    return bloodline

#setup Clipper connection
clipper_conn = ClipperConnection(
    KubernetesContainerManager(useInternalIP=True))
clipper_conn.connect()

batch_size = 2
# batches = np.array([])
# times = np.array([])
for i in range(20):
    # batch_size = np.random.randint(5, high=50)
    print("request " + str(i))
    if batch_size > 1:
        input_list = [
            Lineage(np.random.random_sample()) for i in range(batch_size)
        ]
        out_lin_1 = predict(clipper_conn.get_query_addr(),
                            "lineage1",
                            input_list,
                            batch=True)
        print(out_lin_1)
        out_lin_2 = predict(clipper_conn.get_query_addr(),
                            "lineage2",
                            out_lin_1,
                            batch=True)
        print(out_lin_2)
    else:
        input_lin = Lineage(np.random.random_sample())
        out_lin_1 = predict(clipper_conn.get_query_addr(), "lineage1",
                            input_lin)