def _Read_Lineage(self):
''' Read in Lineage object and import all the GalPop objects within
lineage to Inherit object.
'''
# read in the lineage (< 0.05 seconds for one snapshot)
if not self.quiet:
read_time = time.time()
bloodline = Lineage(nsnap_ancestor=self.nsnap_ancestor,
subhalo_prop=self.subhalo_prop,
quiet=self.quiet)
bloodline.Read(self.nsnap_descendants, quiet=self.quiet)
#DEFUNCT
#if 'subhalogrowth' in self.sfr_prop.keys():
# # depending on whether SFR assign includes subhalo growth AM
# sfr_prop['subhalogrowth']['nsnap_descendant'] = nsnap_descendant
# assign SFR to ancestor object
bloodline.AssignSFR_ancestor(sfr_prop=self.sfr_prop, quiet=self.quiet)
if not self.quiet:
print 'Lineage Read Time = ', time.time() - read_time
self.ancestor = bloodline.ancestor # ancestor object
self.ancestor.tQ[np.where(self.ancestor.tQ != 999.)] = 0.
self.MshamEvol = self.ancestor.Msham_evol[0]
self._ImportDescendants(bloodline)
return None
def AncestorPlots(**sfinh_kwargs):
''' FQ for the SF Inherited Ancestor galaxy population. This test is mainly
to see what the discrepancy between the empirical SF/Q classifcation and the
parameterized FQ model. Ideally, the model and the
'''
bloodline = Lineage(nsnap_ancestor=sfinh_kwargs['nsnap_ancestor'],
subhalo_prop=sfinh_kwargs['subhalo_prop'])
bloodline.Read([1])
bloodline.AssignSFR_ancestor(sfr_prop=sfinh_kwargs['sfr_prop'])
ancestor = getattr(bloodline, 'ancestor')
ancestor.sfr_prop = sfinh_kwargs['sfr_prop']
fig_file = lambda prop: ''.join([
'figure/test/', 'Ancestor',
str(sfinh_kwargs['nsnap_ancestor']), '.',
prop.upper(), '.png'
])
ancestor.plotSsfr(savefig=fig_file('ssfr'))
plt.close()
# FQ
ancestor.plotFq(model=True, savefig=fig_file('fq'))
ancestor.plotSFMS(sfqcut=True,
gal_class='all',
bovyplot=False,
sigSFR=False,
model=False,
savefig=fig_file('sfms'))
plt.close()
return None
def DescendantSSFR(nsnap_descendants, **kwargs):
''' Halo Mass Function composition at z = z_final by initial halo mass at
nsnap_ancestor.
'''
# import SF inherited Lineage
sf_inherit_file = InheritSF_file(nsnap_descendants, **kwargs)
bloodline = Lineage(nsnap_ancestor=kwargs['nsnap_ancestor'],
subhalo_prop=kwargs['subhalo_prop'])
if not isinstance(nsnap_descendants, list):
nsnap_descendants = [nsnap_descendants]
bloodline.Read(nsnap_descendants, filename=sf_inherit_file)
# descendant
for nsnap_descendant in nsnap_descendants:
descendant = getattr(bloodline,
'descendant_snapshot' + str(nsnap_descendant))
fig_name = ''.join([
'figure/test/', 'SSFR.', '.nsnap',
str(nsnap_descendant), '.'.join(
(sf_inherit_file.rsplit('/')[-1]).rsplit('.')[:-1]), '.png'
])
descendant.plotSsfr(line_color='red',
line_width=4,
sfms_prop=kwargs['sfr_prop']['sfms'],
z=descendant.zsnap,
groupcat=True,
savefig=fig_name)
plt.close()
return None
class LineageTests(unittest.TestCase):
def setUp(self):
self.biodb_selector= s= Selector("ncbi")
#self.feature= self.biodb_selector.getFeatureByID(781)
self.feature= self.biodb_selector.getFeatureByID(89)
self.lineage = Lineage(self.feature, self.biodb_selector)
def test_lineage_loading(self):
self.failIf(len(self.lineage.existing_list) > len(self.lineage.default_levels))
def test_levels(self):
self.failIf(self.lineage.get_levels() != [t.level for t in self.lineage.taxon_list])
def test_taxon_retrieval(self):
self.failIf(self.lineage.get_taxa_by_level(2)[0].taxon.level != 2)
def test_level_exists(self):
self.failIf(self.lineage.level_exists(3) != True)
def test_closest_taxon_retrieval(self):
self.failIf(self.lineage._get_closest_taxa_by_level(4, self.lineage.taxon_list)[0].taxon.id < 0)
def test_closest_unn_retrieval(self):
unn= self.lineage.get_closest_unnecessary_taxon_by_level(4)
self.failIf(unn is None)
def LineageAncestorSFMS(nsnap_ancestor,
subhalo_prop={
'scatter': 0.0,
'source': 'li-march'
},
sfr_prop={
'fq': {
'name': 'wetzelsmooth'
},
'sfr': {
'name': 'average'
}
}):
'''Plot the SF-MS of the lineage ancestor object. This is mainly to make sure
that the AssignSFR routine is working properly
'''
# read in lineage
bloodline = Lineage(nsnap_ancestor=nsnap_ancestor,
subhalo_prop=subhalo_prop)
bloodline.Read([1], sfr_prop=sfr_prop)
ancestor = getattr(bloodline, 'ancestor')
sfms_plot = plots.PlotSFMS()
sfms_plot.cenque(ancestor, justsf=True) # lineage ancestor
sfms_plot.param_sfms(nsnap=nsnap_ancestor)
sfms_plot_file = ''.join([
'figure/test/', 'LineageAncestorSFMS',
str(nsnap_ancestor),
bloodline._file_spec(subhalo_prop=subhalo_prop, sfr_prop=sfr_prop),
'.png'
])
sfms_plot.save_fig(sfms_plot_file)
return None
def LineageFinalDescendantSMF(nsnap_ancestor,
subhalo_prop={
'scatter': 0.0,
'source': 'li-march'
},
sfr_prop={
'fq': {
'name': 'wetzelsmooth'
},
'sfr': {
'name': 'average'
}
}):
''' Plot SMF of final descendant. Also mark the composition of the final SMF from Subhalos
that gain stellar mass at different snapshots in the middle of the simulation.
'''
prettyplot()
pretty_colors = prettycolors()
bloodline = Lineage(nsnap_ancestor=nsnap_ancestor,
subhalo_prop=subhalo_prop)
bloodline.Read([1], sfr_prop=sfr_prop)
final_desc = bloodline.descendant_snapshot1
mf = SMF()
smf_plot = plots.PlotSMF()
smf_plot.cgpop(final_desc, line_color='k')
for isnap in range(2, nsnap_ancestor + 1)[::-1]:
started_here = np.where(final_desc.nsnap_genesis == isnap)
mass, phi = mf._smf(final_desc.mass[started_here])
try:
phi_tot += phi
smf_plot.sub.fill_between(mass,
phi_tot - phi,
phi_tot,
color=pretty_colors[isnap],
label='Nsnap=' + str(isnap))
except UnboundLocalError:
phi_tot = phi
smf_plot.sub.fill_between(mass,
np.zeros(len(phi)),
phi,
color=pretty_colors[isnap],
label='Nsnap=' + str(isnap))
smf_plot.set_axes()
fig_file = ''.join([
'figure/test/', 'LineageFinalDescendantSMF', '.ancestor',
str(nsnap_ancestor),
bloodline._file_spec(subhalo_prop=subhalo_prop, sfr_prop=sfr_prop),
'.png'
])
smf_plot.save_fig(fig_file)
return None
def remap_snps(
self,
target_assembly,
complement_bases=True,
parallelize=True,
processes=os.cpu_count(),
):
""" Remap the SNP coordinates of this ``Individual`` from one assembly to another.
This method is a wrapper for `remap_snps` in the ``Lineage`` class.
This method uses the assembly map endpoint of the Ensembl REST API service to convert SNP
coordinates / positions from one assembly to another. After remapping, the coordinates /
positions for the ``Individual``'s SNPs will be that of the target assembly.
If the SNPs are already mapped relative to the target assembly, remapping will not be
performed.
Parameters
----------
target_assembly : {'NCBI36', 'GRCh37', 'GRCh38', 36, 37, 38}
assembly to remap to
complement_bases : bool
complement bases when remapping SNPs to the minus strand
parallelize : bool
utilize multiprocessing to speedup calculations
processes : int
processes to launch if multiprocessing
Returns
-------
chromosomes_remapped : list of str
chromosomes remapped; empty if None
chromosomes_not_remapped : list of str
chromosomes not remapped; empty if None
Notes
-----
An assembly is also know as a "build." For example:
Assembly NCBI36 = Build 36
Assembly GRCh37 = Build 37
Assembly GRCh38 = Build 38
See https://www.ncbi.nlm.nih.gov/assembly for more information about assemblies and
remapping.
References
----------
..[1] Ensembl, Assembly Map Endpoint,
http://rest.ensembl.org/documentation/info/assembly_map
"""
from lineage import Lineage
l = Lineage(parallelize=parallelize, processes=processes)
return l.remap_snps(self, target_assembly, complement_bases)
def merge_snps(self):
if not self.snps_can_be_merged:
return
snps = self.snps.filter(generated_by_lineage=True)
if len(snps) == 1:
# remove SNPs generated by lineage since we're remaking that file
snps[0].delete()
if self.get_discrepant_snps():
# remove discrepant SNPs since we'll be refreshing that data
self.discrepant_snps.delete()
with tempfile.TemporaryDirectory() as tmpdir:
l = Lineage(output_dir=tmpdir, parallelize=False)
ind = l.create_individual("ind")
for snps in self.snps.all():
if snps.build != 37:
temp = l.create_individual("temp", snps.file.path)
temp.remap_snps(37, parallelize=False)
temp_snps = temp.save_snps()
ind.load_snps(temp_snps)
del temp
else:
ind.load_snps(snps.file.path)
snps.merged = True
snps.save()
if ind.snp_count != 0:
if len(ind.discrepant_snps) != 0:
dsnps = DiscrepantSnps.objects.create(
user=self.user,
individual=self,
snp_count=len(ind.discrepant_snps),
)
discrepant_snps_file = ind.save_discrepant_snps()
dsnps.file.name = dsnps.get_relative_path()
dsnps.save()
shutil.move(discrepant_snps_file, dsnps.file.path)
merged_snps_file = ind.save_snps()
summary_info, snps_is_valid = parse_snps(merged_snps_file)
if snps_is_valid:
summary_info["generated_by_lineage"] = True
summary_info["merged"] = True
self.add_snps(merged_snps_file, summary_info)
def DescendantSMHM_composition(nsnap_descendant, abc_step=29, **sfinh_kwargs):
''' Plot the Stellar Mass to Halo Mass relation.
'''
sfinherit_file = InheritSF_file(nsnap_descendant,
abc_step=abc_step,
**sfinh_kwargs)
bloodline = Lineage(nsnap_ancestor=sfinh_kwargs['nsnap_ancestor'],
subhalo_prop=sfinh_kwargs['subhalo_prop'])
bloodline.Read([nsnap_descendant], filename=sfinherit_file)
descendant = getattr(bloodline,
'descendant_snapshot' + str(nsnap_descendant))
smhm_plot = descendant.plotSMHM(bovyplot=False, scatter=False)
# SMF composition based on their initial mass at nsnap_ancestor
started_here = np.where(
descendant.nsnap_genesis == sfinh_kwargs['nsnap_ancestor'])
start_mass = descendant.mass_genesis[started_here]
mass_bins = np.arange(7., 12.5, 0.5)
mass_bin_low = mass_bins[:-1]
mass_bin_high = mass_bins[1:]
for i_m in range(len(mass_bin_low)):
mbin = np.where((start_mass > mass_bin_low[i_m])
& (start_mass <= mass_bin_high[i_m]))
smhm_plot.plot(stellarmass=descendant.mass[started_here[0][mbin]],
halomass=descendant.halo_mass[started_here[0][mbin]],
bovyplot=False,
color=i_m,
label=r"$\mathtt{M_{*,i}=}$" +
str(round(mass_bin_low[i_m], 2)) + "-" +
str(round(mass_bin_high[i_m], 2)))
smhm_plot.plotSummary(stellarmass=descendant.mass,
halomass=descendant.halo_mass)
smhm_plot.sub.legend(loc='upper left', scatterpoints=1)
smhm_plot.set_axes()
fig_file = ''.join([
'figure/test/', 'SMHM_composition.', ''.join(
(sfinherit_file.rsplit('/')[-1]).rsplit('.')[:-1]), '.png'
])
smhm_plot.save_fig(fig_file)
return None
def predict(addr, model_name, input_lin, batch=False):
url = "http://%s/%s/predict" % (addr, model_name)
if batch:
req_json = json.dumps({'input_batch': [[x.val for x in input_lin]]})
else:
req_json = json.dumps({'input': [input_lin.val]})
headers = {'Content-type': 'application/json'}
r = requests.post(url, headers=headers, data=req_json)
input_lin.make_prediction()
return Lineage.add_node(input_lin, model_name, json.loads(r.text)["output"][0])
def DescendantFQ(nsnap_descendant, abc_step=29, **sfinh_kwargs):
''' FQ for the SF Inherited Descendant galaxy population
'''
sfinherit_file = InheritSF_file(nsnap_descendant,
abc_step=abc_step,
**sfinh_kwargs)
bloodline = Lineage(nsnap_ancestor=sfinh_kwargs['nsnap_ancestor'],
subhalo_prop=sfinh_kwargs['subhalo_prop'])
bloodline.Read([nsnap_descendant], filename=sfinherit_file)
descendant = getattr(bloodline,
'descendant_snapshot' + str(nsnap_descendant))
descendant.sfr_prop = sfinh_kwargs['sfr_prop']
fig_file = ''.join([
'figure/test/', 'Fq.', '.'.join(
(sfinherit_file.rsplit('/')[-1]).rsplit('.')[:-1]), '.png'
])
descendant.plotFq(model=sfinh_kwargs['sfr_prop']['fq']['name'],
savefig=fig_file)
return None
def Read_InheritSF(nsnap_descendant,
nsnap_ancestor=20,
n_step=29,
subhalo_prop={
'scatter': 0.2,
'source': 'li-march'
},
sfr_prop={
'fq': {
'name': 'wetzelsmooth'
},
'sfr': {
'name': 'average'
}
},
evol_prop={
'sfr': {
'dutycycle': {
'name': 'notperiodic'
}
},
'mass': {
'name': 'sham'
}
},
flag=None):
sf_inherit_file = InheritSF_file(nsnap_descendant,
nsnap_ancestor=nsnap_ancestor,
abc_step=n_step,
subhalo_prop=subhalo_prop,
sfr_prop=sfr_prop,
evol_prop=evol_prop,
flag=flag)
bloodline = Lineage(nsnap_ancestor=nsnap_ancestor,
subhalo_prop=subhalo_prop)
bloodline.Read([nsnap_descendant], filename=sf_inherit_file)
return bloodline
def LineageFinalDescendantSMF(nsnap_ancestor,
subhalo_prop={'scatter': 0.0, 'source': 'li-march'},
sfr_prop={'fq': {'name': 'wetzelsmooth'}, 'sfr': {'name': 'average'}}):
''' Plot SMF of final descendant. Also mark the composition of the final SMF from Subhalos
that gain stellar mass at different snapshots in the middle of the simulation.
'''
prettyplot()
pretty_colors=prettycolors()
bloodline = Lineage(nsnap_ancestor=nsnap_ancestor, subhalo_prop=subhalo_prop)
bloodline.Read([1], sfr_prop=sfr_prop)
final_desc = bloodline.descendant_snapshot1
mf = SMF()
smf_plot = plots.PlotSMF()
smf_plot.cgpop(final_desc, line_color='k')
for isnap in range(2, nsnap_ancestor+1)[::-1]:
started_here = np.where(final_desc.nsnap_genesis == isnap)
mass, phi = mf._smf(final_desc.mass[started_here])
try:
phi_tot += phi
smf_plot.sub.fill_between(mass, phi_tot - phi, phi_tot, color=pretty_colors[isnap], label='Nsnap='+str(isnap))
except UnboundLocalError:
phi_tot = phi
smf_plot.sub.fill_between(mass, np.zeros(len(phi)), phi, color=pretty_colors[isnap], label='Nsnap='+str(isnap))
smf_plot.set_axes()
fig_file = ''.join([
'figure/test/',
'LineageFinalDescendantSMF',
'.ancestor', str(nsnap_ancestor),
bloodline._file_spec(subhalo_prop=subhalo_prop, sfr_prop=sfr_prop),
'.png'])
smf_plot.save_fig(fig_file)
return None
def remap_snps(self):
# SNPs already remapped
if len(self.snps.filter(generated_by_lineage=True)) == 3:
return
if len(self.snps.filter(generated_by_lineage=True)) == 1:
snps = self.snps.filter(generated_by_lineage=True).get()
else:
# TODO: merge SNPs here, but for now just get canonical SNPs; assume Build 37
snps = self.get_canonical_snps()
if not snps:
return
with tempfile.TemporaryDirectory() as tmpdir:
l = Lineage(output_dir=tmpdir, parallelize=False)
ind = l.create_individual("lineage_NCBI36", snps.file.path)
ind.remap_snps(36, parallelize=False)
file = ind.save_snps()
summary_info, snps_is_valid = parse_snps(file)
if snps_is_valid:
summary_info["generated_by_lineage"] = True
summary_info["merged"] = True
self.add_snps(file, summary_info)
ind = l.create_individual("lineage_GRCh38", snps.file.path)
ind.remap_snps(38, parallelize=False)
file = ind.save_snps()
summary_info, snps_is_valid = parse_snps(file)
if snps_is_valid:
summary_info["generated_by_lineage"] = True
summary_info["merged"] = True
self.add_snps(file, summary_info)
def LineageAncestorSFMS(nsnap_ancestor,
subhalo_prop = {'scatter': 0.0, 'source': 'li-march'},
sfr_prop = { 'fq': {'name': 'wetzelsmooth'}, 'sfr': {'name': 'average'}}
):
'''Plot the SF-MS of the lineage ancestor object. This is mainly to make sure
that the AssignSFR routine is working properly
'''
# read in lineage
bloodline = Lineage(nsnap_ancestor=nsnap_ancestor, subhalo_prop=subhalo_prop)
bloodline.Read([1], sfr_prop=sfr_prop)
ancestor = getattr(bloodline, 'ancestor')
sfms_plot = plots.PlotSFMS()
sfms_plot.cenque(ancestor, justsf=True) # lineage ancestor
sfms_plot.param_sfms(nsnap=nsnap_ancestor)
sfms_plot_file = ''.join([
'figure/test/',
'LineageAncestorSFMS', str(nsnap_ancestor),
bloodline._file_spec(subhalo_prop=subhalo_prop, sfr_prop=sfr_prop),
'.png'])
sfms_plot.save_fig(sfms_plot_file)
return None
def predict(addr, model_name, input_lin, batch=False):
url = "http://%s/%s/predict" % (addr, model_name)
if batch:
req_json = json.dumps({'input_batch': [[x.val for x in input_lin]]})
else:
req_json = json.dumps({'input': [input_lin.val]})
headers = {'Content-type': 'application/json'}
r = requests.post(url, headers=headers, data=req_json)
# print(json.loads(r.text))
str_r = json.loads(r.text)["output"].replace("[", "").replace("]",
"").split()
vals = [float(i) for i in str_r]
new_lineage_objs = [
Lineage.add_node(input_lin[i], model_name, vals[i])
for i in range(len(input_lin))
]
return new_lineage_objs
def LineageSMF(nsnap_ancestor, descendants=None,
subhalo_prop={'scatter': 0.0, 'source': 'li-march'},
sfr_prop={'fq': {'name': 'wetzelsmooth'}, 'sfr': {'name': 'average'}}):
''' Plot the SMF of lineage galaxy population (both ancestor and descendants).
Compare the lineage SMF to the central subhalo and analytic SMFs for all
subhalos. The main agreement, is between the lineage SMF and the central subhalo
SMF. If nsnap_ancestor is high, then there should be more discrepancy
between LIneage SMF and CentralSubhalos SMF because more galaxies are lost.
'''
# read in desendants from the lineage object
if descendants is None:
descendants = range(1, nsnap_ancestor)
bloodline = Lineage(nsnap_ancestor=nsnap_ancestor, subhalo_prop=subhalo_prop)
bloodline.Read(descendants)
smf_plot = plots.PlotSMF()
# plot ancestor against the analytic SMF
ancestor = getattr(bloodline, 'ancestor')
#smf_plot.cenque(ancestor) # SMF of lineage ancestor
#smf_plot.analytic(
# ancestor.zsnap,
# source=ancestor.subhalo_prop['source'],
# line_style='-', lw=1,
# label='All Subhalos')
#smf = SMF()
#subh = CentralSubhalos()
#subh.Read(
# nsnap_ancestor,
# scatter=ancestor.subhalo_prop['scatter'],
# source=ancestor.subhalo_prop['source'])
#subh_mass, subh_phi = smf.centralsubhalos(subh)
#smf_plot.sub.plot(subh_mass, subh_phi, lw=4, ls='--', c='gray', label='Central Subhalos')
for isnap in descendants:
descendant = getattr(bloodline, 'descendant_snapshot'+str(isnap))
smf = SMF()
subh = CentralSubhalos()
subh.Read(
isnap,
scatter=descendant.subhalo_prop['scatter'],
source=descendant.subhalo_prop['source'])
subh_mass, subh_phi = smf.Obj(subh)
smf_plot.sub.plot(subh_mass, subh_phi, lw=4, ls='--', c='gray')
#print (subh_phi - d_phi)/d_phi
#smf_plot.analytic(
# descendant.zsnap,
# source=descendant.subhalo_prop['source'],
# line_style='-', lw=1,
# label=None)
smf_plot.set_axes()
plt.show()
smf_plot_file = ''.join([
'figure/test/',
'LineageSMF',
'.ancestor', str(nsnap_ancestor),
bloodline._file_spec(subhalo_prop=bloodline.subhalo_prop, sfr_prop=bloodline.sfr_prop),
'.png'
])
smf_plot.save_fig(smf_plot_file)
return None
def setUp(self):
self.biodb_selector= s= Selector("ncbi")
#self.feature= self.biodb_selector.getFeatureByID(781)
self.feature= self.biodb_selector.getFeatureByID(89)
self.lineage = Lineage(self.feature, self.biodb_selector)
def DescendantSFMS_composition(nsnap_descendant,
abc_step=29,
bovyplot=False,
**sfinh_kwargs):
''' SFMS for the SF Inherited Descendant galaxy population
'''
sfinherit_file = InheritSF_file(nsnap_descendant,
abc_step=abc_step,
**sfinh_kwargs)
bloodline = Lineage(nsnap_ancestor=sfinh_kwargs['nsnap_ancestor'],
subhalo_prop=sfinh_kwargs['subhalo_prop'])
bloodline.Read([nsnap_descendant], filename=sfinherit_file)
descendant = getattr(bloodline,
'descendant_snapshot' + str(nsnap_descendant))
if not bovyplot:
sfms_plot = descendant.plotSFMS(bovyplot=False, scatter=False)
# SMF composition based on their initial mass at nsnap_ancestor
started_here = np.where(
descendant.nsnap_genesis == sfinh_kwargs['nsnap_ancestor'])
start_mass = descendant.mass_genesis[started_here]
mass_bins = np.arange(7., 12.5, 0.5)
mass_bin_low = mass_bins[:-1]
mass_bin_high = mass_bins[1:]
for i_m in range(len(mass_bin_low)):
mbin = np.where((start_mass > mass_bin_low[i_m])
& (start_mass <= mass_bin_high[i_m]))
sfms_plot.plot(
mass=descendant.mass[started_here[0][mbin]],
sfr=descendant.sfr[started_here[0][mbin]],
sfr_class=descendant.sfr_class[started_here[0][mbin]],
gal_class='quiescent',
bovyplot=False,
sigSFR=False,
color=i_m,
label=r"$\mathtt{M_{*,i}=}$" +
str(round(mass_bin_low[i_m], 2)) + "-" +
str(round(mass_bin_high[i_m], 2)))
qfrac = Fq()
m_arr = np.arange(9.0, 12.5, 0.5)
sfms_plot.sub.plot(m_arr,
qfrac.SFRcut(
m_arr,
descendant.zsnap,
sfms_prop=(sfinh_kwargs['sfr_prop'])['sfms']),
c='k',
ls='--',
lw=4)
bovyplot_str = ''
else:
sfms_plot = descendant.plotSFMS(bovyplot=True)
bovyplot_str = '.bovy'
sfms_plot.sub.legend(loc='lower right', scatterpoints=1)
fig_file = ''.join([
'figure/test/', 'SFMS_composition.', '.'.join(
(sfinherit_file.rsplit('/')[-1]).rsplit('.')[:-1]), bovyplot_str,
'.png'
])
sfms_plot.save_fig(fig_file)
return None
subprocess.call([
'tar', '-xvf', 'dat/observations/primussdss.tar', '-C',
'dat/observations/'
])
print 'Building Group Catalog hdf5 files'
[Obv.BuildGroupCat(Mrcut=Mr, position='central') for Mr in [18, 19, 20]]
# subhalos
subhalo_file = 'subhalo_sham.central.snapshot1.ancestor15.scatter0.0.li-march.hdf5'
if not os.path.exists('dat/wetzel_tree/' + subhalo_file):
subprocess.call([
'wget',
'http://physics.nyu.edu/~chh327/data/subhalo_sham.ancestor15.li-march.tar',
'-P', 'dat/wetzel_tree/'
])
subprocess.call([
'tar', '-xvf', 'dat/wetzel_tree/subhalo_sham.ancestor15.li-march.tar',
'-C', 'dat/wetzel_tree/'
])
# lineage
for scat in [0.0, 0.2]:
bloodline = Lineage(nsnap_ancestor=15,
subhalo_prop={
'scatter': scat,
'source': 'li-march'
},
clobber=True)
bloodline.Descend(clobber=True)
bloodline.Write()
def DescendantSMF_composition(nsnap_descendant,
mass_evol=True,
nsnap_ancestor=20,
abc_step=29,
subhalo_prop={
'scatter': 0.2,
'source': 'li-march'
},
sfr_prop={
'fq': {
'name': 'wetzelsmooth'
},
'sfr': {
'name': 'average'
}
},
evol_prop={
'sfr': {
'dutycycle': {
'name': 'notperiodic'
}
},
'mass': {
'name': 'sham'
}
}):
sf_inherit_file = InheritSF_file(nsnap_descendant,
nsnap_ancestor=nsnap_ancestor,
abc_step=abc_step,
subhalo_prop=subhalo_prop,
sfr_prop=sfr_prop,
evol_prop=evol_prop)
bloodline = Lineage(nsnap_ancestor=nsnap_ancestor,
subhalo_prop=subhalo_prop)
bloodline.Read([nsnap_descendant], filename=sf_inherit_file)
descendant = getattr(bloodline,
'descendant_snapshot' + str(nsnap_descendant))
smf_plot = descendant.plotSMF()
mf = SMF()
if mass_evol:
# SMF composition based on their initial mass at nsnap_ancestor
started_here = np.where(descendant.nsnap_genesis == nsnap_ancestor)
start_mass = descendant.mass_genesis[started_here]
mass_bins = np.arange(7., 12.5, 0.5)
mass_bin_low = mass_bins[:-1]
mass_bin_high = mass_bins[1:]
for i_m in range(len(mass_bin_low)):
mbin = np.where((start_mass > mass_bin_low[i_m])
& (start_mass <= mass_bin_high[i_m]))
mass, phi = mf._smf(descendant.mass[started_here[0][mbin]])
try:
phi_tot += phi
smf_plot.sub.fill_between(
mass,
phi_tot - phi,
phi_tot,
color=pretty_colors[i_m],
label=str(round(mass_bin_low[i_m], 2)) + '-' +
str(round(mass_bin_high[i_m], 2)))
except UnboundLocalError:
phi_tot = phi
smf_plot.sub.fill_between(
mass,
np.zeros(len(phi)),
phi,
color=pretty_colors[i_m],
label=str(round(mass_bin_low[i_m], 2)) + '-' +
str(round(mass_bin_high[i_m], 2)))
else:
for isnap in range(2, nsnap_ancestor + 1)[::-1]:
started_here = np.where(descendant.nsnap_genesis == isnap)
mass, phi = mf._smf(descendant.mass[started_here])
try:
phi_tot += phi
smf_plot.sub.fill_between(mass,
phi_tot - phi,
phi_tot,
color=pretty_colors[isnap],
label='Nsnap=' + str(isnap))
except UnboundLocalError:
phi_tot = phi
smf_plot.sub.fill_between(mass,
np.zeros(len(phi)),
phi,
color=pretty_colors[isnap],
label='Nsnap=' + str(isnap))
smf_plot.set_axes()
mass_evol_str = ''
if mass_evol:
mass_evol_str = '.mass_evol'
fig_file = ''.join([
'figure/test/', ''.join(
(sf_inherit_file.rsplit('/')[-1]).rsplit('.')[:-1]),
'.SMF_composition', mass_evol_str, '.png'
])
smf_plot.save_fig(fig_file)
return None
def Compare_DescendantSMF_composition(nsnap_descendant, kwarg_list, n_step=29):
''' Compare the M* distribution at z_final of galaxies from same initial
M* bin for different prescriptions of SF evolution. More specifically
this is to see how well the integrated SF stellar masses match the SHAM
stellar masses.
'''
descendants = []
for kwarg in kwarg_list: # import SF inherited Lineages for different set of arguments
sf_inherit_file = InheritSF_file(nsnap_descendant,
abc_step=n_step,
**kwarg)
bloodline = Lineage(nsnap_ancestor=kwarg['nsnap_ancestor'],
subhalo_prop=kwarg['subhalo_prop'])
bloodline.Read([nsnap_descendant], filename=sf_inherit_file)
descendants.append(
getattr(bloodline, 'descendant_snapshot' + str(nsnap_descendant)))
mf = SMF()
mass_bins = np.arange(7., 12.5, 0.5)
mass_bin_low = mass_bins[:-1]
mass_bin_high = mass_bins[1:]
prettyplot()
pretty_colors = prettycolors()
lstyles = ['-', '--', '-.']
for i_m in range(len(mass_bin_low)):
fig = plt.figure(1)
sub = fig.add_subplot(111)
mass_evol_str = ''
for i_desc, descendant in enumerate(descendants):
# SMF composition based on their initial mass at nsnap_ancestor
started_here = np.where(descendant.nsnap_genesis ==
kwarg_list[i_desc]['nsnap_ancestor'])
start_mass = descendant.mass_genesis[started_here]
mbin = np.where((start_mass > mass_bin_low[i_m])
& (start_mass <= mass_bin_high[i_m]))
mass, phi = mf._smf(descendant.mass[started_here[0][mbin]])
sub.plot(mass,
phi,
c=pretty_colors[i_desc + 1],
lw=4,
ls=lstyles[i_desc],
label=(kwarg_list[i_desc])['evol_prop']['mass']['name'] +
' Masses')
mass_evol_str += (kwarg_list[i_desc])['evol_prop']['mass']['name']
sub.set_xlim([7.5, 12.0])
sub.set_ylim([0.0, 0.012])
sub.set_ylabel(r'$\Phi$', fontsize=25)
sub.set_xlabel(r'$\mathtt{log\;M_*}$', fontsize=25)
sub.legend(loc='upper right')
sub.set_title(''.join([
'log M* = ',
str(round(mass_bin_low[i_m], 2)), '-',
str(round(mass_bin_high[i_m], 2)), ' at Snapshot ',
str(kwarg_list[i_desc]['nsnap_ancestor'])
]),
fontsize=25)
fig_file = ''.join([
'figure/test/', 'DescendantSMF_composition', '.massevol_',
mass_evol_str, '.Mbin',
str(round(mass_bin_low[i_m], 2)), '_',
str(round(mass_bin_high[i_m], 2)), '.png'
])
fig.savefig(fig_file, bbox_inches='tight')
plt.close()
return None
def find_discordant_snps(self, progress_recorder=None):
ind1_snps = self.individual1.get_canonical_snps()
ind2_snps = self.individual2.get_canonical_snps()
if not ind1_snps or not ind2_snps:
self.delete()
return
if self.individual3:
ind3_snps = self.individual3.get_canonical_snps()
if not ind3_snps:
self.delete()
return
with tempfile.TemporaryDirectory() as tmpdir:
l = Lineage(output_dir=tmpdir, parallelize=False)
ind1_snps_file = shutil.copy(
ind1_snps.file.path,
os.path.join(tmpdir, "ind1_snps" + ind1_snps.file_ext),
)
ind2_snps_file = shutil.copy(
ind2_snps.file.path,
os.path.join(tmpdir, "ind2_snps" + ind2_snps.file_ext),
)
if self.individual3:
ind3_snps_file = shutil.copy(
ind3_snps.file.path,
os.path.join(tmpdir, "ind3_snps" + ind3_snps.file_ext),
)
ind1 = l.create_individual(self.individual1.name, ind1_snps_file)
ind2 = l.create_individual(self.individual2.name, ind2_snps_file)
if self.individual3:
ind3 = l.create_individual(self.individual3.name,
ind3_snps_file)
else:
ind3 = None
discordant_snps = l.find_discordant_snps(ind1,
ind2,
ind3,
save_output=True)
self.total_discordant_snps = len(discordant_snps)
for root, dirs, files in os.walk(tmpdir):
for file in files:
file_path = os.path.join(root, file)
if "discordant_snps" in file:
self.discordant_snps_csv.name = get_relative_user_dir_file(
self.user.uuid, uuid4())
compress_file(file_path, self.discordant_snps_csv.path)
self.discordant_snps_pickle = get_relative_user_dir_file(
self.user.uuid, uuid4(), ".pkl.gz")
discordant_snps.to_pickle(
self.discordant_snps_pickle.path)
break
self.setup_complete = True
self.save()
def DescendantHMF_composition(nsnap_descendant,
nsnap_ancestor=20,
n_step=29,
subhalo_prop={
'scatter': 0.2,
'source': 'li-march'
},
sfr_prop={
'fq': {
'name': 'wetzelsmooth'
},
'sfms': {
'name': 'linear'
}
},
evol_prop={
'sfr': {
'dutycycle': {
'name': 'notperiodic'
}
},
'mass': {
'name': 'sham'
}
}):
''' Halo Mass Function composition at z = z_final by initial halo mass at
nsnap_ancestor.
'''
# import SF inherited Lineage
sf_inherit_file = InheritSF_file(nsnap_descendant,
nsnap_ancestor=nsnap_ancestor,
abc_step=n_step,
subhalo_prop=subhalo_prop,
sfr_prop=sfr_prop,
evol_prop=evol_prop)
bloodline = Lineage(nsnap_ancestor=nsnap_ancestor,
subhalo_prop=subhalo_prop)
bloodline.Read([nsnap_descendant], filename=sf_inherit_file)
# descendant
descendant = getattr(bloodline,
'descendant_snapshot' + str(nsnap_descendant))
smf_plot = PlotSMF()
mf = SMF()
# SMF composition based on their initial mass at nsnap_ancestor
started_here = np.where(descendant.nsnap_genesis == nsnap_ancestor)
start_mass = descendant.halomass_genesis[started_here]
mass_bins = np.arange(10., 17.5, 0.5)
mass_bin_low = mass_bins[:-1]
mass_bin_high = mass_bins[1:]
for i_m in range(len(mass_bin_low)):
mbin = np.where((start_mass > mass_bin_low[i_m])
& (start_mass <= mass_bin_high[i_m]))
mass, phi = mf._smf(descendant.halo_mass[started_here[0][mbin]],
m_arr=np.arange(10., 17.1, 0.1))
try:
phi_tot += phi
smf_plot.sub.fill_between(mass,
phi_tot - phi,
phi_tot,
color=pretty_colors[i_m],
label=str(round(mass_bin_low[i_m], 2)) +
'-' + str(round(mass_bin_high[i_m], 2)))
except UnboundLocalError:
phi_tot = phi
smf_plot.sub.fill_between(mass,
np.zeros(len(phi)),
phi,
color=pretty_colors[i_m],
label=str(round(mass_bin_low[i_m], 2)) +
'-' + str(round(mass_bin_high[i_m], 2)))
#smf_plot.set_axes()
smf_plot.sub.set_yscale('log')
smf_plot.sub.set_xlim([10.0, 17.0])
smf_plot.sub.legend(loc='upper right')
mass_evol_str = '.mass_evol'
fig_file = ''.join([
'figure/test/', ''.join(
(sf_inherit_file.rsplit('/')[-1]).rsplit('.')[:-1]),
'.HMF_composition', mass_evol_str, '.png'
])
smf_plot.save_fig(fig_file)
return None
def InheritSF(nsnap_descendant,
nsnap_ancestor=20,
subhalo_prop={
'scatter': 0.0,
'source': 'li-march'
},
sfr_prop={
'fq': {
'name': 'wetzelsmooth'
},
'sfms': {
'name': 'linear',
'mslope': 0.55,
'zslope': 1.1
}
},
evol_prop={
'pq': {
'slope': 0.05,
'yint': 0.0
},
'tau': {
'name': 'line',
'fid_mass': 10.75,
'slope': -0.6,
'yint': 0.6
},
'sfr': {
'dutycycle': {
'name': 'notperiodic'
}
},
'mass': {
'name': 'sham'
}
}):
''' Evolve star formation properties of ancestor CentralGalaxyPopulation class within
the Lineage Class to descendant CentralGalaxlyPopulation object
Parameters
----------
nsnap_ancestor : int
Snapshot number of ancestor CGPop object. The ancestor object is constructed
with from subhalo catalog with subhalo_prop properties. They are also assigned
SFRs with sfr_prop properties.
subhalo_prop : dict
Dictionary that describes the subhalo properties. The key 'scatter' corresponds
to the M*-M_halo relation. The key 'soruce' describes the source SMF used for
the SHAM masses.
sfr_prop : dict
Dictionary that describes the SFR properties assigned to the ancestor CenQue object.
The key 'fq' describes the quiescent fraction used for the ancestor while the key
'sfr' describes the properties of the SFR assignment.
evol_prop : dict
Dictionary that consists of dictionaries which each describe paramter choices in
the model.
- evol_prop['pq'] dictates the quenching properties.
- evol_prop['tau'] dictates the quenching timescale.
- evol_prop['sfr'] dictates the SFR evolution.
- evol_prop['mass'] dictates the mass evolution.
'''
# make sure that snapshot = 1 is included among imported descendants
# and the first element of the list
if isinstance(nsnap_descendant, list):
raise ValueError('nsnap_descendant arg has to be an int')
# evolution properties
pq_prop = evol_prop['pq']
tau_prop = evol_prop['tau']
sfrevol_prop = evol_prop['sfr']
massevol_prop = evol_prop['mass']
# read in the lineage (< 0.05 seconds for one snapshot)
read_time = time.time()
bloodline = Lineage(nsnap_ancestor=nsnap_ancestor,
subhalo_prop=subhalo_prop)
bloodline.Read(range(nsnap_descendant, nsnap_ancestor), sfr_prop='default')
if 'subhalogrowth' in sfr_prop.keys():
sfr_prop['subhalogrowth']['nsnap_descendant'] = nsnap_descendant
bloodline.AssignSFR_ancestor(sfr_prop=sfr_prop)
print 'Lineage Read Time = ', time.time() - read_time
ancestor = bloodline.ancestor # ancestor object
t_init = ancestor.t_cosmic
z_init = ancestor.zsnap
# descendant object
descendant = getattr(bloodline,
'descendant_snapshot' + str(nsnap_descendant))
t_final = descendant.t_cosmic
z_final = descendant.zsnap
print 'Evolve until z = ', z_final
# initialize SF properties of descendant
n_descendant = len(descendant.snap_index)
descendant.sfr = np.repeat(-999., n_descendant)
descendant.ssfr = np.repeat(-999., n_descendant)
descendant.min_ssfr = np.repeat(-999., n_descendant)
descendant.tau = np.repeat(-999., n_descendant)
descendant.sfr_class = np.chararray(n_descendant, itemsize=16)
descendant.sfr_class[:] = ''
succession, will = intersection_index(
getattr(descendant, 'ancestor' + str(nsnap_ancestor)),
ancestor.snap_index)
if len(succession) != len(descendant.mass):
raise ValueError('Something wrong with the lineage')
q_ancestors = np.where(
ancestor.sfr_class[will] == 'quiescent')[0] # Q ancestors
sf_ancestors = np.where(
ancestor.sfr_class[will] == 'star-forming')[0] # SF ancestors
# Evolve queiscent ancestor galaxies
q_time = time.time()
descendant = _QuiescentEvol(ancestor,
descendant,
succession=succession,
will=will)
print 'Quiescent evolution takes ', time.time() - q_time
# Evolve Star Forming Galaxies
sf_time = time.time()
_StarformingEvol(ancestor,
descendant,
succession=succession,
will=will,
evol_prop=evol_prop,
sfr_prop=sfr_prop,
lineage=bloodline)
print 'Star Forming Evolution takes ', time.time() - sf_time
# Deal with over quenched galaxies
overquenched = np.where(descendant.min_ssfr[succession[sf_ancestors]] >
descendant.ssfr[succession[sf_ancestors]])
if len(overquenched[0]) > 0:
descendant.ssfr[succession[
sf_ancestors[overquenched]]] = descendant.min_ssfr[succession[
sf_ancestors[overquenched]]]
descendant.sfr[succession[sf_ancestors[overquenched]]] = descendant.ssfr[succession[sf_ancestors[overquenched]]] \
+ descendant.mass[succession[sf_ancestors[overquenched]]]
#descendant.tau[succession[sf_ancestors[overquenched]]] = -999.
descendant.data_columns = list(descendant.data_columns) + [
'ssfr', 'sfr', 'min_ssfr', 'sfr_class'
] #, 'tau'])
setattr(bloodline, 'descendant_snapshot' + str(nsnap_descendant),
descendant)
return bloodline
def LineageSMF(nsnap_ancestor,
descendants=None,
subhalo_prop={
'scatter': 0.0,
'source': 'li-march'
},
sfr_prop={
'fq': {
'name': 'wetzelsmooth'
},
'sfr': {
'name': 'average'
}
}):
''' Plot the SMF of lineage galaxy population (both ancestor and descendants).
Compare the lineage SMF to the central subhalo and analytic SMFs for all
subhalos. The main agreement, is between the lineage SMF and the central subhalo
SMF. If nsnap_ancestor is high, then there should be more discrepancy
between LIneage SMF and CentralSubhalos SMF because more galaxies are lost.
'''
# read in desendants from the lineage object
if descendants is None:
descendants = range(1, nsnap_ancestor)
bloodline = Lineage(nsnap_ancestor=nsnap_ancestor,
subhalo_prop=subhalo_prop)
bloodline.Read(descendants)
smf_plot = plots.PlotSMF()
# plot ancestor against the analytic SMF
ancestor = getattr(bloodline, 'ancestor')
#smf_plot.cenque(ancestor) # SMF of lineage ancestor
#smf_plot.analytic(
# ancestor.zsnap,
# source=ancestor.subhalo_prop['source'],
# line_style='-', lw=1,
# label='All Subhalos')
#smf = SMF()
#subh = CentralSubhalos()
#subh.Read(
# nsnap_ancestor,
# scatter=ancestor.subhalo_prop['scatter'],
# source=ancestor.subhalo_prop['source'])
#subh_mass, subh_phi = smf.centralsubhalos(subh)
#smf_plot.sub.plot(subh_mass, subh_phi, lw=4, ls='--', c='gray', label='Central Subhalos')
for isnap in descendants:
descendant = getattr(bloodline, 'descendant_snapshot' + str(isnap))
smf = SMF()
subh = CentralSubhalos()
subh.Read(isnap,
scatter=descendant.subhalo_prop['scatter'],
source=descendant.subhalo_prop['source'])
subh_mass, subh_phi = smf.Obj(subh)
smf_plot.sub.plot(subh_mass, subh_phi, lw=4, ls='--', c='gray')
#print (subh_phi - d_phi)/d_phi
#smf_plot.analytic(
# descendant.zsnap,
# source=descendant.subhalo_prop['source'],
# line_style='-', lw=1,
# label=None)
smf_plot.set_axes()
plt.show()
smf_plot_file = ''.join([
'figure/test/', 'LineageSMF', '.ancestor',
str(nsnap_ancestor),
bloodline._file_spec(subhalo_prop=bloodline.subhalo_prop,
sfr_prop=bloodline.sfr_prop), '.png'
])
smf_plot.save_fig(smf_plot_file)
return None
#!/usr/local/bin/python3.8
import sys
import logging, sys
logger = logging.getLogger()
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler(sys.stdout))
from lineage import Lineage
l = Lineage(output_dir='storage/app/dna/output')
var1 = sys.argv[1]
var2 = sys.argv[2]
file1 = "storage/app/dna/" + sys.argv[3]
file2 = "storage/app/dna/" + sys.argv[4]
user662 = l.create_individual(var1, file1)
user663 = l.create_individual(var2, file2)
discordant_snps = l.find_discordant_snps(user662, user663, save_output=True)
len(discordant_snps.loc[discordant_snps['chrom'] != 'MT'])
results = l.find_shared_dna([user662, user663],
cM_threshold=0.75,
snp_threshold=1100)
class TestSnps(BaseLineageTestCase):
def setUp(self):
self.l = Lineage()
self.snps_GRCh38 = SNPs("tests/input/GRCh38.csv")
self.snps = SNPs("tests/input/chromosomes.csv")
self.snps_none = SNPs(None)
self.del_output_dir_helper()
def snps_discrepant_pos(self):
return self.create_snp_df(rsid=["rs3094315"],
chrom=["1"],
pos=[1],
genotype=["AA"])
def test_assembly(self):
assert self.snps_GRCh38.assembly == "GRCh38"
def test_assembly_no_snps(self):
assert self.snps_none.assembly == ""
def test_snp_count(self):
assert self.snps.snp_count == 6
def test_snp_count_no_snps(self):
assert self.snps_none.snp_count == 0
def test_chromosomes(self):
assert self.snps.chromosomes == ["1", "2", "3", "5", "PAR", "MT"]
def test_chromosomes_no_snps(self):
assert self.snps_none.chromosomes == []
def test_chromosomes_summary(self):
assert self.snps.chromosomes_summary == "1-3, 5, PAR, MT"
def test_chromosomes_summary_no_snps(self):
assert self.snps_none.chromosomes_summary == ""
def test_build_no_snps(self):
assert self.snps_none.build is None
def test_build_detected_no_snps(self):
assert not self.snps_none.build_detected
def test_build_detected_PAR_snps(self):
if os.getenv("DOWNLOADS_ENABLED"):
snps = SNPs("tests/input/GRCh37_PAR.csv")
assert snps.build == 37
assert snps.build_detected
def test_sex_no_snps(self):
assert self.snps_none.sex == ""
def test_sex_Male_Y_chrom(self):
ind = self.simulate_snps(
self.l.create_individual("test_snps_sex_Male_Y_chrom"),
chrom="Y",
pos_start=1,
pos_max=59373566,
pos_step=10000,
)
file = ind.save_snps()
from lineage.snps import SNPs
snps = SNPs(file)
assert snps.sex == "Male"
def test_get_summary(self):
assert self.snps_GRCh38.get_summary() == {
"source": "generic",
"assembly": "GRCh38",
"build": 38,
"build_detected": True,
"snp_count": 4,
"chromosomes": "1, 3",
"sex": "",
}
def test_get_summary_no_snps(self):
assert self.snps_none.get_summary() is None
def test_is_valid_True(self):
assert self.snps_GRCh38.is_valid()
def test_is_valid_False(self):
assert not self.snps_none.is_valid()
def test__read_raw_data(self):
assert self.snps_none.snps is None
assert self.snps_none.source == ""
def test__lookup_build_with_snp_pos_None(self):
snps = SNPs()
snps._snps = self.snps_discrepant_pos()
assert snps.detect_build() is None
def test_get_assembly_None(self):
snps = SNPs()
snps._build = None
assert snps.get_assembly() is ""
def setUp(self):
self.l = Lineage()
self.del_output_dir_helper()
def find_shared_dna_genes(self, progress_recorder=None):
ind1_snps = self.individual1.get_canonical_snps()
ind2_snps = self.individual2.get_canonical_snps()
if not ind1_snps or not ind2_snps:
self.delete()
return
with tempfile.TemporaryDirectory() as tmpdir:
l = Lineage(output_dir=tmpdir, parallelize=False)
ind1_snps_file = shutil.copy(
ind1_snps.file.path,
os.path.join(tmpdir, "ind1_snps" + ind1_snps.file_ext),
)
ind2_snps_file = shutil.copy(
ind2_snps.file.path,
os.path.join(tmpdir, "ind2_snps" + ind2_snps.file_ext),
)
ind1 = l.create_individual(self.individual1.name, ind1_snps_file)
ind2 = l.create_individual(self.individual2.name, ind2_snps_file)
shared_dna_one_chrom, shared_dna_two_chrom, shared_genes_one_chrom, shared_genes_two_chrom = l.find_shared_dna(
ind1,
ind2,
cM_threshold=float(self.cM_threshold),
snp_threshold=int(self.snp_threshold),
shared_genes=True,
save_output=True,
)
self.total_shared_segments_one_chrom = len(shared_dna_one_chrom)
self.total_shared_segments_two_chrom = len(shared_dna_two_chrom)
self.total_shared_cMs_one_chrom = Decimal(
shared_dna_one_chrom["cMs"].sum())
self.total_shared_cMs_two_chrom = Decimal(
shared_dna_two_chrom["cMs"].sum())
self.total_snps_one_chrom = shared_dna_one_chrom["snps"].sum()
self.total_snps_two_chrom = shared_dna_two_chrom["snps"].sum()
self.total_chrom_one_chrom = len(
shared_dna_one_chrom["chrom"].unique())
self.total_chrom_two_chrom = len(
shared_dna_two_chrom["chrom"].unique())
self.total_shared_genes_one_chrom = len(shared_genes_one_chrom)
self.total_shared_genes_two_chrom = len(shared_genes_two_chrom)
for root, dirs, files in os.walk(tmpdir):
for file in files:
file_path = os.path.join(root, file)
if ".png" in file:
self.shared_dna_plot_png.name = get_relative_user_dir_file(
self.user.uuid, uuid4(), ".png")
shutil.move(file_path, self.shared_dna_plot_png.path)
os.chmod(self.shared_dna_plot_png.path, 0o640)
elif "shared_dna_one_chrom" in file:
self.shared_dna_one_chrom_csv = get_relative_user_dir_file(
self.user.uuid, uuid4())
compress_file(file_path,
self.shared_dna_one_chrom_csv.path)
self.shared_dna_one_chrom_pickle = get_relative_user_dir_file(
self.user.uuid, uuid4(), ".pkl.gz")
shared_dna_one_chrom.to_pickle(
self.shared_dna_one_chrom_pickle.path)
elif "shared_genes_one_chrom" in file:
self.shared_genes_one_chrom_csv = get_relative_user_dir_file(
self.user.uuid, uuid4())
compress_file(file_path,
self.shared_genes_one_chrom_csv.path)
self.shared_genes_one_chrom_pickle = get_relative_user_dir_file(
self.user.uuid, uuid4(), ".pkl.gz")
shared_genes_one_chrom.to_pickle(
self.shared_genes_one_chrom_pickle.path)
elif "shared_dna_two_chrom" in file:
self.shared_dna_two_chrom_csv = get_relative_user_dir_file(
self.user.uuid, uuid4())
compress_file(file_path,
self.shared_dna_two_chrom_csv.path)
self.shared_dna_two_chrom_pickle = get_relative_user_dir_file(
self.user.uuid, uuid4(), ".pkl.gz")
shared_dna_two_chrom.to_pickle(
self.shared_dna_two_chrom_pickle.path)
elif "shared_genes_two_chrom" in file:
self.shared_genes_two_chrom_csv = get_relative_user_dir_file(
self.user.uuid, uuid4())
compress_file(file_path,
self.shared_genes_two_chrom_csv.path)
self.shared_genes_two_chrom_pickle = get_relative_user_dir_file(
self.user.uuid, uuid4(), ".pkl.gz")
shared_genes_two_chrom.to_pickle(
self.shared_genes_two_chrom_pickle.path)
self.setup_complete = True
self.save()
# print(output.val)
# print(output.graph)
# print(output.input_node)
# print(output.used)
# dot = Digraph(comment='Merge Example')
# for n in output.graph.nodes:
# dot.node(n)
# for e in output.graph.edges:
# dot.edge(e[0], e[1])
# dot.render('graphs/pipeline_merge.gv', view=True)
lineages = []
node_counts = {}
for i in range(20):
rand_input = Lineage(np.random.random_sample())
output = pipeline(rand_input, clipper_conn)
lineages.append(output)
for n in output.graph.nodes:
if n in node_counts.keys():
node_counts[n] += 1
else:
node_counts[n] = 1
# total_counts = np.append(total_counts, counts)
# print(output.graph.adj_list)
# create a dict of counts instead of list
combined = Graph.merge_graphs([l for l in lineages])
print(node_counts)
def setUp(self):
self.l = Lineage()
self.snps_GRCh38 = SNPs("tests/input/GRCh38.csv")
self.snps = SNPs("tests/input/chromosomes.csv")
self.snps_none = SNPs(None)
self.del_output_dir_helper()
def DescendantQAplot(nsnap_descendants, **sfinh_kwargs):
''' The ultimate QAplot t rule them all. 4 panels showing all the properties.
'''
sfinherit_file = InheritSF_file(nsnap_descendants, **sfinh_kwargs)
bloodline = Lineage(nsnap_ancestor=sfinh_kwargs['nsnap_ancestor'],
subhalo_prop=sfinh_kwargs['subhalo_prop'])
if not isinstance(nsnap_descendants, list):
nsnap_descendants = [nsnap_descendants]
bloodline.Read(nsnap_descendants, filename=sfinherit_file)
for nsnap_descendant in nsnap_descendants:
descendant = getattr(bloodline,
'descendant_snapshot' + str(nsnap_descendant))
descendant.sfr_prop = sfinh_kwargs['sfr_prop']
started_here = np.where(
descendant.nsnap_genesis == sfinh_kwargs['nsnap_ancestor'])
start_mass = descendant.mass_genesis[started_here]
# Mass bins
mass_bins = np.arange(7., 12.5, 0.5)
mass_bin_low = mass_bins[:-1]
mass_bin_high = mass_bins[1:]
plt.close()
prettyplot()
fig = plt.figure(1, figsize=[25, 6])
for i_sub in range(1, 5):
sub_i = fig.add_subplot(1, 4, i_sub)
if i_sub == 1: # SMF
mf = SMF()
mass, phi = mf.Obj(descendant)
sub_i.plot(mass,
phi,
lw=4,
c=pretty_colors[descendant.nsnap],
label=r'Simulated')
censub = CentralSubhalos()
censub.Read(descendant.nsnap,
scatter=sfinh_kwargs['subhalo_prop']['scatter'],
source=sfinh_kwargs['subhalo_prop']['source'],
nsnap_ancestor=sfinh_kwargs['nsnap_ancestor'])
mass, phi = mf._smf(censub.mass)
sub_i.plot(mass,
phi,
c='k',
lw=4,
ls='--',
label='Central Subhalos')
sub_i.set_ylim([10**-5, 10**-1])
sub_i.set_xlim([7.5, 12.0])
plt.xticks([8., 9., 10., 11., 12.])
sub_i.set_yscale('log')
# x,y labels
sub_i.set_xlabel(r'Mass $\mathtt{M_*}$', fontsize=25)
sub_i.set_ylabel(r'Stellar Mass Function $\mathtt{\Phi}$',
fontsize=25)
sub_i.legend(loc='upper right', frameon=False)
elif i_sub == 2: # SFMS
# SMF composition based on their initial mass at nsnap_ancestor
for i_m in range(len(mass_bin_low)):
mbin = np.where((start_mass > mass_bin_low[i_m])
& (start_mass <= mass_bin_high[i_m]))
sub_i.scatter(descendant.mass[started_here[0][mbin]],
descendant.sfr[started_here[0][mbin]],
color=pretty_colors[i_m],
label=r"$\mathtt{M_{*,i}=}$" +
str(round(mass_bin_low[i_m], 2)) + "-" +
str(round(mass_bin_high[i_m], 2)))
qfrac = Fq()
m_arr = np.arange(9.0, 12.5, 0.5)
sub_i.plot(m_arr,
qfrac.SFRcut(
m_arr,
descendant.zsnap,
sfms_prop=(sfinh_kwargs['sfr_prop'])['sfms']),
c='k',
ls='--',
lw=4)
sub_i.set_xlim([9.0, 12.0])
sub_i.set_ylim([-5.0, 2.0])
sub_i.set_xlabel(r'$\mathtt{log\;M_*}$', fontsize=25)
sub_i.set_ylabel(r'$\mathtt{log\;SFR}$', fontsize=25)
elif i_sub == 3: #SMHM
for i_m in range(len(mass_bin_low)):
mbin = np.where((start_mass > mass_bin_low[i_m])
& (start_mass <= mass_bin_high[i_m]))
sub_i.scatter(descendant.halo_mass[started_here[0][mbin]],
descendant.mass[started_here[0][mbin]],
color=pretty_colors[i_m],
label=r"$\mathtt{M_{*,i}=}$" +
str(round(mass_bin_low[i_m], 2)) + "-" +
str(round(mass_bin_high[i_m], 2)))
stellarmass = descendant.mass[started_here]
halomass = descendant.halo_mass[started_here]
mbin = np.arange(halomass.min(), halomass.max(), 0.25)
mlow = mbin[:-1]
mhigh = mbin[1:]
muMstar = np.zeros(len(mlow))
sigMstar = np.zeros(len(mlow))
for im in range(len(mlow)):
mbin = np.where((halomass > mlow[im])
& (halomass <= mhigh[im]))
muMstar[im] = np.mean(stellarmass[mbin])
sigMstar[im] = np.std(stellarmass[mbin])
sub_i.errorbar(0.5 * (mlow + mhigh),
muMstar,
yerr=sigMstar,
color='k',
lw=3,
fmt='o',
capthick=2)
sub_i.set_ylim([9.0, 12.0])
sub_i.set_xlim([10.0, 15.0])
sub_i.set_ylabel(r'Stellar Mass $\mathtt{M_*}$', fontsize=25)
sub_i.set_xlabel(r'Halo Mass $\mathtt{M_{Halo}}$', fontsize=25)
#sub_i.legend(loc='upper left', frameon=False, scatterpoints=1)
elif i_sub == 4: # Fq
#mass, fq = descendant.Fq()
sfq = qfrac.Classify(descendant.mass,
descendant.sfr,
descendant.zsnap,
sfms_prop=descendant.sfr_prop['sfms'])
gc = GroupCat(Mrcut=18, position='central')
gc.Read()
gc_sfq = qfrac.Classify(gc.mass,
gc.sfr,
np.mean(gc.z),
sfms_prop=descendant.sfr_prop['sfms'])
#sub_i.plot(mass, fq, color=pretty_colors[descendant.nsnap], lw=3, ls='--',
# label=r'$\mathtt{z =} '+str(descendant.zsnap)+'$')
M_bin = np.array([9.7, 10.1, 10.5, 10.9, 11.3])
M_low = M_bin[:-1]
M_high = M_bin[1:]
M_mid = 0.5 * (M_low + M_high)
fq = np.zeros(len(M_low))
gc_fq = np.zeros(len(M_low))
for i_m in xrange(len(M_low)):
mlim = np.where((descendant.mass > M_low[i_m])
& (descendant.mass <= M_high[i_m]))
gc_mlim = np.where((gc.mass > M_low[i_m])
& (gc.mass <= M_high[i_m]))
ngal = np.float(len(mlim[0]))
gc_ngal = np.float(len(gc_mlim[0]))
if ngal != 0: # no galaxy in mass bin
ngal_q = np.float(
len(np.where(sfq[mlim] == 'quiescent')[0]))
fq[i_m] = ngal_q / ngal
if gc_ngal != 0:
gc_ngal_q = np.float(
len(np.where(gc_sfq[gc_mlim] == 'quiescent')[0]))
gc_fq[i_m] = gc_ngal_q / gc_ngal
sub_i.plot(M_mid,
fq,
color=pretty_colors[descendant.nsnap],
lw=3,
label=r'$\mathtt{z =} ' + str(descendant.zsnap) +
'$')
fq_model = qfrac.model(
M_bin,
descendant.zsnap,
lit=sfinh_kwargs['sfr_prop']['fq']['name'])
sub_i.plot(M_bin,
fq_model,
color='k',
lw=4,
ls='--',
label=sfinh_kwargs['sfr_prop']['fq']['name'])
sub_i.scatter(M_mid,
gc_fq,
color='k',
s=100,
lw=0,
label='Group Catalog')
sub_i.set_xlim([9.0, 12.0])
sub_i.set_ylim([0.0, 1.0])
sub_i.set_xlabel(r'Mass $\mathtt{M_*}$')
sub_i.set_ylabel(r'Quiescent Fraction $\mathtt{f_Q}$',
fontsize=20)
sub_i.legend(loc='upper left',
frameon=False,
scatterpoints=1,
markerscale=0.75)
plt.tight_layout(pad=0.4, w_pad=0.5, h_pad=1.0)
fig_name = ''.join([
'figure/test/', 'QAplot.', '.nsnap',
str(nsnap_descendant), '.'.join(
(sfinherit_file.rsplit('/')[-1]).rsplit('.')[:-1]), '.png'
])
fig.savefig(fig_name, bbox_inches='tight')
plt.close()
return None
def InheritSF(nsnap_descendant,
nsnap_ancestor=20,
subhalo_prop=None,
sfr_prop=None,
evol_prop=None,
quiet=True):
''' Evolve star formation properties of 'ancestor' CentralGalaxyPopulation class at
redshift specified by nsnap_ancestor to descendant CentralGalaxlyPopulation object
at redshift specified by nsnap_descendant. Both ancestor and descendant objects
are attributes in the Lineage class, which contains all the 'lineage' information
i.e. all the halo tracking information.
Parameters
----------
nsnap_ancestor : int
Snapshot number of ancestor CGPop object. The ancestor object is constructed
with from subhalo catalog with subhalo_prop properties. They are also assigned
SFRs with sfr_prop properties.
subhalo_prop : dict
Dictionary that describes the subhalo properties. The key 'scatter' corresponds
to the M*-M_halo relation. The key 'soruce' describes the source SMF used for
the SHAM masses.
sfr_prop : dict
Dictionary that describes the SFR properties assigned to the ancestor CenQue object.
The key 'fq' describes the quiescent fraction used for the ancestor while the key
'sfr' describes the properties of the SFR assignment.
evol_prop : dict
Dictionary that consists of dictionaries which each describe paramter choices in
the model.
- evol_prop['pq'] dictates the quenching properties.
- evol_prop['tau'] dictates the quenching timescale.
- evol_prop['sfr'] dictates the SFR evolution.
- evol_prop['mass'] dictates the mass evolution.
'''
if isinstance(nsnap_descendant, list):
d_list = True
else:
d_list = False
# read in the lineage (< 0.05 seconds for one snapshot)
#read_time = time.time()
bloodline = Lineage(nsnap_ancestor=nsnap_ancestor,
subhalo_prop=subhalo_prop,
quiet=quiet)
if not d_list:
bloodline.Read(range(nsnap_descendant, nsnap_ancestor), quiet=quiet)
else:
bloodline.Read(range(np.min(nsnap_descendant), nsnap_ancestor),
quiet=quiet)
if 'subhalogrowth' in sfr_prop.keys(
): # depending on whether SFR assign includes subhalo growth AM
sfr_prop['subhalogrowth']['nsnap_descendant'] = nsnap_descendant
bloodline.AssignSFR_ancestor(sfr_prop=sfr_prop, quiet=quiet)
#print 'Lineage Read Time = ', time.time() - read_time
ancestor = bloodline.ancestor # ancestor object
t_init = ancestor.t_cosmic
z_init = ancestor.zsnap
ancestor.tQ[np.where(ancestor.tQ != 999.)] = 0.
if not isinstance(nsnap_descendant, list):
nsnap_descendant = [nsnap_descendant]
descendants = []
sf_ancestors, q_ancestors = [], []
successions, wills = [], []
for nd in nsnap_descendant:
des = getattr(bloodline, 'descendant_snapshot' + str(nd))
des._clean_initialize() # initialize SF properties
des.sfr_prop = ancestor.sfr_prop
# match indices up with each other
succession, will = intersection_index(
getattr(des, 'ancestor' + str(nsnap_ancestor)),
ancestor.snap_index)
if len(succession) != len(des.mass):
raise ValueError('Something wrong with the lineage')
q_ancestor = np.where(
ancestor.sfr_class[will] == 'quiescent')[0] # Q ancestors
sf_ancestor = np.where(
ancestor.sfr_class[will] == 'star-forming')[0] # SF ancestors
if not quiet:
print "nsnap_descendant = ", nd, "Ancestors: Nq = ", len(
q_ancestor), ', Nsf = ', len(sf_ancestor)
descendants.append(des)
wills.append(will)
successions.append(succession)
q_ancestors.append(q_ancestor)
sf_ancestors.append(sf_ancestor)
# Evolve queiscent ancestor galaxies
if not quiet:
q_time = time.time()
descendants = _QuiescentEvol(ancestor,
descendants,
successions=successions,
wills=wills,
q_ancestors=q_ancestors)
if not quiet:
print 'Quiescent evolution takes ', time.time() - q_time
# Evolve Star Forming Galaxies
if not quiet:
sf_time = time.time()
################## PQ BASED DROPPED ##############################
# if evol_prop['type'] == 'pq_based':
# _StarformingEvol_Pq(ancestor, descendant, succession=succession, will=will,
# evol_prop=evol_prop, sfr_prop=sfr_prop,
# lineage=bloodline)
# elif evol_prop['type'] == 'simult':
################## PQ BASED DROPPED ##############################
descendants = _StarformingEvol_SimulEvo(ancestor,
descendants,
successions=successions,
wills=wills,
evol_prop=evol_prop,
sfr_prop=sfr_prop,
lineage=bloodline,
quiet=quiet)
if not quiet:
print 'Star Forming Evolution takes ', time.time() - sf_time
descendants = _Overquenching(descendants,
successions=successions,
wills=wills,
sf_ancestors=sf_ancestors)
for descendant in descendants:
descendant.data_columns = list(descendant.data_columns) + [
'ssfr', 'sfr', 'min_ssfr', 'sfr_class'
]
setattr(bloodline, 'descendant_snapshot' + str(descendant.nsnap),
descendant)
return bloodline
#setup Clipper connection
clipper_conn = ClipperConnection(
KubernetesContainerManager(useInternalIP=True))
clipper_conn.connect()
batch_size = 2
# batches = np.array([])
# times = np.array([])
for i in range(20):
# batch_size = np.random.randint(5, high=50)
print("request " + str(i))
if batch_size > 1:
input_list = [
Lineage(np.random.random_sample()) for i in range(batch_size)
]
out_lin_1 = predict(clipper_conn.get_query_addr(),
"lineage1",
input_list,
batch=True)
print(out_lin_1)
out_lin_2 = predict(clipper_conn.get_query_addr(),
"lineage2",
out_lin_1,
batch=True)
print(out_lin_2)
else:
input_lin = Lineage(np.random.random_sample())
out_lin_1 = predict(clipper_conn.get_query_addr(), "lineage1",
input_lin)
