def manage_nzbs():
client = get_nzb_client()
for transfer in Transfer.find({
'started': {'$ne': None},
'finished': None,
'type': 'binsearch',
}):
nzb_id = transfer['info'].get('nzo_id')
if not nzb_id:
continue
info = client.get_nzb(nzb_id)
if not info:
Transfer.update({'_id': transfer['_id']},
{'$set': {'finished': datetime.utcnow()}},
safe=True)
else:
info['name'] = info.get('filename', transfer['info'].get('name'))
total = get_float(info.get('mb', 0)) * 1024 ** 2
Transfer.update({'_id': transfer['_id']},
{'$set': {
'total': total,
'transferred': total - get_float(info.get('mbleft', 0)) * 1024 ** 2,
'progress': get_float(info.get('percentage', 0)),
'info': info,
}}, safe=True)
paths = Settings.get_settings('paths')
# Manage queued nzbs
for nzb in client.list_nzbs():
transfer = get_nzb_transfer(nzb['nzo_id'])
if not transfer:
now = datetime.utcnow()
Transfer.add(nzb['filename'], str(paths['default']),
type='binsearch', added=now, started=now, queued=now,
info={'nzo_id': nzb['nzo_id']})
elif transfer['finished']:
client.remove_nzb(nzb['nzo_id'])
logger.info('removed finished nzb "%s" (%s)', nzb['filename'], nzb['nzo_id'])
# Manage finished nzbs
for nzb in client.list_nzbs(history=True):
transfer = get_nzb_transfer(nzb['nzo_id'])
if nzb['status'] == 'Completed':
dst = transfer['dst'] if transfer else str(paths['default'])
elif nzb['status'] == 'Failed':
dst = str(paths['invalid'])
else:
continue
manage_nzb(client, nzb_id=nzb['nzo_id'], dst=dst)
class Manager:
def __init__(self, screen, clock, **kwargs):
# self.turns = turns()
self.screen = screen
self.status = _INTRO
self.minigames = []
self.board = Board(screen=screen, clock=clock)
self.intro = Slideshow(screen=screen, slides=((1800000, "intro.png"),))
self.minigames.append((Missiles(screen=screen, clock=clock), ["Dodge"]))
self.minigames.append((WhackAMullet(screen=screen, clock=clock), ["Kill"]))
self.minigames.append((Freeze(screen=screen, clock=clock), ["Dodge", "Kill", "Jump", "Type"]))
self.minigames.append((Platformer(clock=clock, screen=screen), ["Jump"]))
# self.minigames.append(EasyWin(screen))
self.minigames.append((ClickClack(screen), ["Type"]))
self.current = self.intro
self.catcher = None
self.wait = 7
self.currWait = 0
self.counter = 0
def draw(self, screen):
self.current.draw()
def update(self):
if self.status != _TRANSFER:
self.catcher = self.current.update(result=self.catcher)
if self.status == _MINI and self.catcher != None:
print "switching to tbg"
self.current.reset()
self.current = self.board
self.status = _TBG
self.currWait = self.wait
elif self.status == _TBG and self.catcher != None:
print "switching to transfer"
self.easywin = EasyWin(self.screen)
self.status = _TRANSFER
self.counter += 1
if self.counter == len(self.minigames):
self.counter = 0
select = self.minigames[self.counter]
# select = self.minigames[random.randint(0,len(self.minigames)-1)]
self.current = Transfer(self.screen, select[1][random.randint(0, len(select[1]) - 1)], select[0])
# self.current = self.minigames[random.randint(0,len(self.minigames)-1)]
# self.status = _MINI
self.currWait = self.wait
elif self.status == _INTRO and self.catcher != None:
self.catcher = None
self.current.reset()
self.current = self.board
self.status = _TBG
self.currWait = self.wait
else:
# clear queue
# self.currWait -= 1
result = self.current.update()
pygame.event.get()
if result != None:
print "switching to mini"
self.current = result
self.status = _MINI
def manage_torrent(client, hash, dst):
try:
torrent = client.get_torrent(hash=hash)
if not torrent:
return
if not client.check_torrent_files(torrent,
check_unfinished=settings.CHECK_UNFINISHED_TORRENTS):
invalid_dir = str(Settings.get_settings('paths')['invalid'])
if torrent.progress == 100 and not client.move_files(torrent, invalid_dir):
return
if client.remove_torrent(hash=hash, delete_data=True):
logger.info('removed invalid torrent "%s" (%s%% done)', torrent.name, int(torrent.progress))
Transfer.update({'info.hash': hash, 'finished': None},
{'$set': {'finished': datetime.utcnow()}}, safe=True)
return
if torrent.progress == 100:
destination = client.get_destination_dir(torrent, dst)
if not client.move_files(torrent, destination):
return
if client.remove_torrent(hash=hash):
logger.info('moved finished torrent "%s" to %s', torrent.name, dst)
Transfer.update({'info.hash': hash, 'finished': None},
{'$set': {'finished': datetime.utcnow()}}, safe=True)
return
torrent_settings = Settings.get_settings('torrent')
now = datetime.utcnow()
inactive_delta = torrent_settings['inactive_delta']
if inactive_delta:
date = torrent.date_active or torrent.date_added
if date < now - timedelta(hours=inactive_delta) \
and client.remove_torrent(hash=hash, delete_data=True):
logger.debug('removed inactive torrent "%s": no activity since %s', torrent.name, date)
return
added_delta = torrent_settings['added_delta']
if added_delta:
date = torrent.date_added
if date < now - timedelta(hours=added_delta) \
and client.remove_torrent(hash=hash, delete_data=True):
logger.debug('removed obsolete torrent "%s": added %s', torrent.name, date)
except TransmissionError, e:
logger.error('torrent client error: %s', str(e))
def process_transfer(self, id):
transfer = Transfer.find_one({'_id': id, 'queued': None})
if not transfer:
return
try:
info = get_torrent_client().add_torrent(transfer['src'])
Transfer.update({'_id': transfer['_id']},
{'$set': {
'queued': datetime.utcnow(),
'info': info,
}}, safe=True)
except TorrentExists, e:
Transfer.update({'_id': transfer['_id']},
{'$set': {'finished': datetime.utcnow()}},
safe=True)
logger.debug('failed to start torrent: %s', str(e))
def run():
running = {}
for res in Transfer.find({
'started': {'$ne': None},
'finished': None,
}):
add_running(running, res['type'])
settings_ = Settings.get_settings('general')
retry_delta = timedelta(seconds=settings_['retry_delta'])
for transfer in Transfer.find({
'finished': None,
'$or': [
{'added': None},
{
'added': {'$lt': datetime.utcnow() - retry_delta},
'started': None,
'tries': {'$lt': settings_['max_tries']},
},
],
}):
limit = settings.WORKERS_LIMITS.get(transfer['type'])
if limit and running.get(transfer['type'], 0) >= limit:
continue
factory = get_factory()
target = '%s.workers.add.process' % settings.PACKAGE_NAME
if factory.get(target=target,
args=(transfer['_id'], transfer['type'])):
continue
factory.add(target=target,
args=(transfer['_id'], transfer['type']),
timeout=settings.PROCESS_TIMEOUT)
now = datetime.utcnow()
Transfer.update({'_id': transfer['_id']}, {
'$set': {'added': now, 'started': now},
'$inc': {'tries': 1},
}, safe=True)
add_running(running, transfer['type'])
count_str = ' (#%s)' % transfer['tries'] if transfer['tries'] > 1 else ''
logger.info('started%s %s transfer %s to %s', count_str, transfer['type'], transfer['src'], transfer['dst'])
def process_transfer(self, id):
from mediacore.web.search.plugins.binsearch import get_nzb, BinsearchError
transfer = Transfer.find_one({'_id': id, 'queued': None})
if not transfer:
return
qs = parse_qs(urlparse(transfer['src']).query)
try:
name = clean(qs['b'][0])
except KeyError:
Transfer.update({'_id': transfer['_id']},
{'$set': {'finished': datetime.utcnow()}}, safe=True)
logger.error('failed to get nzb name from %s', transfer['src'])
return
try:
nzb_data = get_nzb(transfer['src'])
except BinsearchError, e:
Transfer.update({'_id': transfer['_id']},
{'$set': {'started': None}}, safe=True)
logger.error('failed to get nzb data from %s: %s', transfer['src'], str(e))
return
class MassFunction(object):
"""
An object containing all relevant quantities for the mass function.
The purpose of this class is to calculate many quantities associated with
the dark matter halo mass function (HMF). The class is initialized to form a
cosmology and takes in various options as to how to calculate all
further quantities.
All required outputs are provided as ``@property`` attributes for ease of
access.
Contains an update() method which can be passed arguments to update, in the
most optimal manner. All output quantities are calculated only when needed
(but stored after first calculation for quick access).
Quantities related to the transfer function can be accessed through the
``transfer`` property of this object.
Parameters
----------
M : array_like, optional, default ``np.linspace(10,15,501)``
The masses at which to perform analysis [units :math:`\log_{10}M_\odot h^{-1}`].
mf_fit : str or callable, optional, default ``"SMT"``
A string indicating which fitting function to use for :math:`f(\sigma)`
Available options:
1. ``'PS'``: Press-Schechter form from 1974
#. ``'ST'``: Sheth-Mo-Tormen empirical fit 2001 (deprecated!)
#. ``'SMT'``: Sheth-Mo-Tormen empirical fit from 2001
#. ``'Jenkins'``: Jenkins empirical fit from 2001
#. ``'Warren'``: Warren empirical fit from 2006
#. ``'Reed03'``: Reed empirical from 2003
#. ``'Reed07'``: Reed empirical from 2007
#. ``'Tinker'``: Tinker empirical from 2008
#. ``'Watson'``: Watson empirical 2012
#. ``'Watson_FoF'``: Watson Friend-of-friend fit 2012
#. ``'Crocce'``: Crocce 2010
#. ``'Courtin'``: Courtin 2011
#. ``'Angulo'``: Angulo 2012
#. ``'Angulo_Bound'``: Angulo sub-halo function 2012
#. ``'Bhattacharya'``: Bhattacharya empirical fit 2011
#. ``'Behroozi'``: Behroozi extension to Tinker for high-z 2013
Alternatively, one may define a callable function, with the signature
``func(self)``, where ``self`` is a :class:`MassFunction` object (and
has access to all its attributes). This may be passed here.
delta_wrt : str, {``"mean"``, ``"crit"``}
Defines what the overdensity of a halo is with respect to, mean density
of the universe, or critical density.
delta_h : float, optional, default ``200.0``
The overdensity for the halo definition, with respect to ``delta_wrt``
user_fit : str, optional, default ``""``
A string defining a mathematical function in terms of `x`, used as
the fitting function, where `x` is taken as :math:`\( \sigma \)`. Will only
be applicable if ``mf_fit == "user_model"``.
cut_fit : bool, optional, default ``True``
Whether to forcibly cut :math:`f(\sigma)` at bounds in literature.
If false, will use whole range of `M`.
delta_c : float, default ``1.686``
The critical overdensity for collapse, :math:`\delta_c`
kwargs : keywords
These keyword arguments are sent to the `hmf.transfer.Transfer` class.
Included are all the cosmological parameters (see the docs for details).
"""
def __init__(self, M=None, mf_fit="ST", delta_h=200.0,
delta_wrt='mean', cut_fit=True, z2=None, nz=None,
delta_c=1.686, mv_scheme="trapz", **kwargs):
"""
Initializes some parameters
"""
if M is None:
M = np.linspace(10, 15, 501)
# A list of all available kwargs (sent to Cosmology via Transfer)
self._cp = ["sigma_8", "n", "w", "cs2_lam", "t_cmb", "y_he", "N_nu",
"omegan", "H0", "h", "omegab",
"omegac", "omegav", "omegab_h2", "omegac_h2",
"force_flat", "default"]
# Set up a simple dictionary of kwargs which can be later updated
self._cpdict = {k:v for k, v in kwargs.iteritems() if k in self._cp}
# Set all given parameters.
self.mf_fit = mf_fit
self.M = M
self.delta_h = delta_h
self.delta_wrt = delta_wrt
self.cut_fit = cut_fit
self.z2 = z2
self.nz = nz
self.delta_c = delta_c
self.transfer = Transfer(**kwargs)
self.mv_scheme = mv_scheme
tools.check_kr(self.M[0], self.M[-1], self.cosmo.mean_dens,
self.transfer.lnk[0], self.transfer.lnk[-1])
def update(self, **kwargs):
"""
Update the class with the given arguments in an optimal manner.
Accepts any argument that the constructor takes.
"""
for key, val in kwargs.iteritems():
# The following takes care of everything specifically in this class
if "_MassFunction__" + key in self.__dict__:
try: doset = np.any(getattr(self, key) != val)
except ValueError: doset = not np.array_equal(getattr(self, key), val)
if doset:
setattr(self, key, val)
# We need to handle deletes in this class by parameters in Transfer here
if key is 'z':
if val != self.transfer.z:
del self.sigma
# All parameters being sent to Transfer:
the_rest = {k:v for k, v in kwargs.iteritems() if "_MassFunction__" + k not in self.__dict__}
# Some things are basically deleted when anything in Transfer is updated
if len(the_rest) > 0:
del self.delta_halo
if len(the_rest) > 1 or (len(the_rest) == 1 and 'z' not in the_rest):
del self._sigma_0
# The rest are sent to the Transfer class (stupid values weeded out there)
self.transfer.update(**the_rest)
tools.check_kr(self.M[0], self.M[-1], self.cosmo.mean_dens,
self.transfer.lnk[0], self.transfer.lnk[-1])
# --- SET PROPERTIES -------------------------------------------------------
@property
def M(self):
return self.__M
@M.setter
def M(self, val):
try:
if len(val) == 1:
raise ValueError("M must be a sequence of length > 1")
except TypeError:
raise TypeError("M must be a sequence of length > 1")
if np.any(np.abs(np.diff(val, 2)) > 1e-5) or val[1] < val[0]:
raise ValueError("M must be a linearly increasing vector! " + str(val[0]) + " " + str(val[1]))
# Delete stuff dependent on it
del self._sigma_0
self.__M = 10 ** val
@property
def delta_c(self):
return self.__delta_c
@delta_c.setter
def delta_c(self, val):
try:
val = float(val)
except ValueError:
raise ValueError("delta_c must be a number: ", val)
if val <= 0:
raise ValueError("delta_c must be > 0 (", val, ")")
if val > 10.0:
raise ValueError("delta_c must be < 10.0 (", val, ")")
self.__delta_c = val
del self.fsigma
@property
def mv_scheme(self):
return self.__mv_scheme
@mv_scheme.setter
def mv_scheme(self, val):
if val not in ['trapz', 'simps', 'romb']:
raise ValueError("mv_scheme wrong")
else:
self.__mv_scheme = val
del self._sigma_0
@property
def mf_fit(self):
return self.__mf_fit
@mf_fit.setter
def mf_fit(self, val):
# mf_fit may be a callable or a string. Try callable first.
try:
val(self)
except:
try:
val = str(val)
except:
raise ValueError("mf_fit must be a string or callable, got ", val)
if val not in Fits.mf_fits + ["Behroozi"]:
raise ValueError("mf_fit is not in the list of available fitting functions: ", val)
# Also delete stuff dependent on it
del self.fsigma
self.__mf_fit = val
@property
def delta_h(self):
return self.__delta_h
@delta_h.setter
def delta_h(self, val):
try:
val = float(val)
except ValueError:
raise ValueError("delta_halo must be a number: ", val)
if val <= 0:
raise ValueError("delta_halo must be > 0 (", val, ")")
if val > 10000:
raise ValueError("delta_halo must be < 10,000 (", val, ")")
self.__delta_h = val
# Delete stuff dependent on it
del self.delta_halo
@property
def delta_wrt(self):
return self.__delta_wrt
@delta_wrt.setter
def delta_wrt(self, val):
if val not in ['mean', 'crit']:
raise ValueError("delta_wrt must be either 'mean' or 'crit' (", val, ")")
self.__delta_wrt = val
del self.delta_halo
@property
def z2(self):
return self.__z2
@z2.setter
def z2(self, val):
if val is None:
self.__z2 = val
return
try:
val = float(val)
except ValueError:
raise ValueError("z must be a number (", val, ")")
if val <= self.transfer.z:
raise ValueError("z2 must be larger than z")
else:
self.__z2 = val
del self.dndm
@property
def nz(self):
return self.__nz
@nz.setter
def nz(self, val):
if val is None:
self.__nz = val
return
try:
val = int(val)
except ValueError:
raise ValueError("nz must be an integer")
if val < 1:
raise ValueError("nz must be >= 1")
else:
self.__nz = val
del self.dndm
@property
def cut_fit(self):
return self.__cut_fit
@cut_fit.setter
def cut_fit(self, val):
if not isinstance(val, bool):
raise ValueError("cut_fit must be a bool, " + str(val))
del self.fsigma
self.__cut_fit = val
#-------------------------------- START NON-SET PROPERTIES ----------------------------------------------
@property
def cosmo(self):
""" :class:`hmf.cosmo.Cosmology` object aliased from `self.transfer.cosmo`"""
return self.transfer.cosmo
@property
def delta_halo(self):
""" Overdensity of a halo w.r.t mean density"""
try:
return self.__delta_halo
except:
if self.delta_wrt == 'mean':
self.__delta_halo = self.delta_h
elif self.delta_wrt == 'crit':
self.__delta_halo = self.delta_h / cp.density.omega_M_z(self.transfer.z, **self.cosmo.cosmolopy_dict())
return self.__delta_halo
@delta_halo.deleter
def delta_halo(self):
try:
del self.__delta_halo
del self.fsigma
except:
pass
@property
def _sigma_0(self):
"""
The normalised mass variance at z=0 :math:`\sigma`
Notes
-----
.. math:: \sigma^2(R) = \frac{1}{2\pi^2}\int_0^\infty{k^2P(k)W^2(kR)dk}
"""
try:
return self.__sigma_0
except:
self.__sigma_0 = tools.mass_variance(self.M, self.transfer._lnP_0,
self.transfer.lnk,
self.cosmo.mean_dens,
self.mv_scheme)
return self.__sigma_0
@_sigma_0.deleter
def _sigma_0(self):
try:
del self.__sigma_0
del self._dlnsdlnm
del self.sigma
except:
pass
@property
def _dlnsdlnm(self):
"""
The value of :math:`\left|\frac{\d \ln \sigma}{\d \ln M}\right|`, ``len=len(M)``
Notes
-----
.. math:: frac{d\ln\sigma}{d\ln M} = \frac{3}{2\sigma^2\pi^2R^4}\int_0^\infty \frac{dW^2(kR)}{dM}\frac{P(k)}{k^2}dk
"""
try:
return self.__dlnsdlnm
except:
self.__dlnsdlnm = tools.dlnsdlnm(self.M, self._sigma_0, self.transfer._lnP_0,
self.transfer.lnk,
self.cosmo.mean_dens)
return self.__dlnsdlnm
@_dlnsdlnm.deleter
def _dlnsdlnm(self):
try:
del self.__dlnsdlnm
del self.dndm
del self.n_eff
except:
pass
@property
def sigma(self):
"""
The mass variance at `z`, ``len=len(M)``
"""
try:
return self.__sigma
except:
self.__sigma = self._sigma_0 * self.transfer.growth
return self.__sigma
@sigma.deleter
def sigma(self):
try:
del self.__sigma
del self.fsigma
del self.lnsigma
except:
pass
@property
def lnsigma(self):
"""
Natural log of inverse mass variance, ``len=len(M)``
"""
try:
return self.__lnsigma
except:
self.__lnsigma = np.log(1 / self.sigma)
return self.__lnsigma
@lnsigma.deleter
def lnsigma(self):
try:
del self.__lnsigma
del self.fsigma
except:
pass
@property
def n_eff(self):
"""
Effective spectral index at scale of halo radius, ``len=len(M)``
"""
try:
return self.__n_eff
except:
self.__n_eff = tools.n_eff(self._dlnsdlnm)
return self.__n_eff
@n_eff.deleter
def n_eff(self):
try:
del self.__n_eff
except:
pass
@property
def fsigma(self):
"""
The multiplicity function, :math:`f(\sigma)`, for `mf_fit`. ``len=len(M)``
"""
try:
return self.__fsigma
except:
try:
self.__fsigma = self.mf_fit(self)
except:
fits_class = Fits(self)
self.__fsigma = fits_class.nufnu()
if np.sum(np.isnan(self.__fsigma)) > 0.8 * len(self.__fsigma):
# the input mass range is almost completely outside the cut
logger.warning("The specified mass-range was almost entirely \
outside of the limits from the fit. Ignored fit range...")
self.cut_fit = False
try:
self.__fsigma = self.mf_fit(self)
except:
self.__fsigma = fits_class.nufnu()
return self.__fsigma
@fsigma.deleter
def fsigma(self):
try:
del self.__fsigma
del self.dndm
except:
pass
@property
def dndm(self):
"""
The number density of haloes, ``len=len(M)`` [units :math:`h^4 M_\odot^{-1} Mpc^{-3}`]
"""
try:
return self.__dndm
except:
if self.z2 is None: # #This is normally the case
self.__dndm = self.fsigma * self.cosmo.mean_dens * np.abs(self._dlnsdlnm) / self.M ** 2
if self.mf_fit == 'Behroozi':
a = 1 / (1 + self.transfer.z)
theta = 0.144 / (1 + np.exp(14.79 * (a - 0.213))) * (self.M / 10 ** 11.5) ** (0.5 / (1 + np.exp(6.5 * a)))
ngtm_tinker = self._ngtm()
ngtm_behroozi = 10 ** (theta + np.log10(ngtm_tinker))
dthetadM = 0.144 / (1 + np.exp(14.79 * (a - 0.213))) * \
(0.5 / (1 + np.exp(6.5 * a))) * (self.M / 10 ** 11.5) ** \
(0.5 / (1 + np.exp(6.5 * a)) - 1) / (10 ** 11.5)
self.__dndm = self.__dndm * 10 ** theta - ngtm_behroozi * np.log(10) * dthetadM
else: # #This is for a survey-volume weighted calculation
if self.nz is None:
self.nz = 10
zedges = np.linspace(self.transfer.z, self.z2, self.nz)
zcentres = (zedges[:-1] + zedges[1:]) / 2
dndm = np.zeros_like(zcentres)
vol = np.zeros_like(zedges)
vol[0] = cp.distance.comoving_volume(self.transfer.z,
**self.cosmo.cosmolopy_dict())
for i, zz in enumerate(zcentres):
self.update(z=zz)
dndm[i] = self.fsigma * self.cosmo.mean_dens * np.abs(self._dlnsdlnm) / self.M ** 2
if self.mf_fit == 'Behroozi':
a = 1 / (1 + self.transfer.z)
theta = 0.144 / (1 + np.exp(14.79 * (a - 0.213))) * (self.M / 10 ** 11.5) ** (0.5 / (1 + np.exp(6.5 * a)))
ngtm_tinker = self._ngtm()
ngtm_behroozi = 10 ** (theta + np.log10(ngtm_tinker))
dthetadM = 0.144 / (1 + np.exp(14.79 * (a - 0.213))) * (0.5 / (1 + np.exp(6.5 * a))) * (self.M / 10 ** 11.5) ** (0.5 / (1 + np.exp(6.5 * a)) - 1) / (10 ** 11.5)
dndm[i] = dndm[i] * 10 ** theta - ngtm_behroozi * np.log(10) * dthetadM
vol[i + 1] = cp.distance.comoving_volume(z=zedges[i + 1],
**self.cosmo.cosmolopy_dict())
vol = vol[1:] - vol[:-1] # Volume in shells
integrand = vol * dndm
numerator = intg.simps(integrand, x=zcentres)
denom = intg.simps(vol, zcentres)
self.__dndm = numerator / denom
return self.__dndm
@dndm.deleter
def dndm(self):
try:
del self.__dndm
del self.dndlnm
del self.dndlog10m
except:
pass
@property
def dndlnm(self):
"""
The differential mass function in terms of natural log of `M`, ``len=len(M)`` [units :math:`h^3 Mpc^{-3}`]
"""
try:
return self.__dndlnm
except:
self.__dndlnm = self.M * self.dndm
return self.__dndlnm
@dndlnm.deleter
def dndlnm(self):
try:
del self.__dndlnm
del self.ngtm
del self.nltm
del self.mgtm
del self.mltm
del self.how_big
except:
pass
@property
def dndlog10m(self):
"""
The differential mass function in terms of log of `M`, ``len=len(M)`` [units :math:`h^3 Mpc^{-3}`]
"""
try:
return self.__dndlog10m
except:
self.__dndlog10m = self.M * self.dndm * np.log(10)
return self.__dndlog10m
@dndlog10m.deleter
def dndlog10m(self):
try:
del self.__dndlog10m
except:
pass
def _upper_ngtm(self, M, mass_function, cut):
"""Calculate the mass function above given range of `M` in order to integrate"""
### WE CALCULATE THE MASS FUNCTION ABOVE THE COMPUTED RANGE ###
# mass_function is logged already (not log10 though)
m_upper = np.linspace(np.log(M[-1]), np.log(10 ** 18), 500)
if cut: # since its been cut, the best we can do is a power law
mf_func = spline(np.log(M), mass_function, k=1)
mf = mf_func(m_upper)
else:
# We try to calculate the hmf as far as we can normally
new_pert = copy.deepcopy(self)
new_pert.update(M=np.log10(np.exp(m_upper)))
mf = np.log(np.exp(m_upper) * new_pert.dndm)
if np.isnan(mf[-1]): # Then we couldn't get up all the way, so have to do linear ext.
if np.isnan(mf[1]): # Then the whole extension is nan and we have to use the original (start at 1 because 1 val won't work either)
mf_func = spline(np.log(M), mass_function, k=1)
mf = mf_func(m_upper)
else:
mfslice = mf[np.logical_not(np.isnan(mf))]
m_nan = m_upper[np.isnan(mf)]
m_true = m_upper[np.logical_not(np.isnan(mf))]
mf_func = spline(m_true, mfslice, k=1)
mf[len(mfslice):] = mf_func(m_nan)
return m_upper, mf
def _lower_ngtm(self, M, mass_function, cut):
### WE CALCULATE THE MASS FUNCTION BELOW THE COMPUTED RANGE ###
# mass_function is logged already (not log10 though)
m_lower = np.linspace(np.log(10 ** 3), np.log(M[0]), 500)
if cut: # since its been cut, the best we can do is a power law
mf_func = spline(np.log(M), mass_function, k=1)
mf = mf_func(m_lower)
else:
# We try to calculate the hmf as far as we can normally
new_pert = copy.deepcopy(self)
new_pert.update(M=np.log10(np.exp(m_lower)))
mf = np.log(np.exp(m_lower) * new_pert.dndm)
if np.isnan(mf[0]): # Then we couldn't go down all the way, so have to do linear ext.
mfslice = mf[np.logical_not(np.isnan(mf))]
m_nan = m_lower[np.isnan(mf)]
m_true = m_lower[np.logical_not(np.isnan(mf))]
mf_func = spline(m_true, mfslice, k=1)
mf[:len(mfslice)] = mf_func(m_nan)
return m_lower, mf
def _ngtm(self):
"""
Calculate n(>m).
This function is separated from the property because of the Behroozi fit
"""
# set M and mass_function within computed range
M = self.M[np.logical_not(np.isnan(self.dndlnm))]
mass_function = self.dndlnm[np.logical_not(np.isnan(self.dndlnm))]
# Calculate the mass function (and its integral) from the highest M up to 10**18
if M[-1] < 10 ** 18:
m_upper, mf = self._upper_ngtm(M, np.log(mass_function), M[-1] < self.M[-1])
int_upper = intg.simps(np.exp(mf), dx=m_upper[2] - m_upper[1], even='first')
else:
int_upper = 0
# Calculate the cumulative integral (backwards) of mass_function (Adding on the upper integral)
ngtm = np.concatenate((intg.cumtrapz(mass_function[::-1], dx=np.log(M[1]) - np.log(M[0]))[::-1], np.zeros(1))) + int_upper
# We need to set ngtm back in the original length vector with nans where they were originally
if len(ngtm) < len(self.M):
ngtm_temp = np.zeros_like(self.dndlnm)
ngtm_temp[:] = np.nan
ngtm_temp[np.logical_not(np.isnan(self.dndlnm))] = ngtm
ngtm = ngtm_temp
return ngtm
@property
def ngtm(self):
"""
The cumulative mass function above `M`, ``len=len(M)`` [units :math:`h^3 Mpc^{-3}`]
"""
try:
return self.__ngtm
except:
self.__ngtm = self._ngtm()
return self.__ngtm
@ngtm.deleter
def ngtm(self):
try:
del self.__ngtm
del self.how_big
except:
pass
@property
def mgtm(self):
"""
Mass in haloes `>M`, ``len=len(M)`` [units :math:`M_\odot h^2 Mpc^{-3}`]
"""
try:
return self.__mgtm
except:
M = self.M[np.logical_not(np.isnan(self.dndlnm))]
mass_function = self.dndlnm[np.logical_not(np.isnan(self.dndlnm))]
# Calculate the mass function (and its integral) from the highest M up to 10**18
if M[-1] < 10 ** 18:
m_upper, mf = self._upper_ngtm(M, np.log(mass_function), M[-1] < self.M[-1])
int_upper = intg.simps(np.exp(mf + m_upper) , dx=m_upper[2] - m_upper[1], even='first')
else:
int_upper = 0
# Calculate the cumulative integral (backwards) of mass_function (Adding on the upper integral)
self.__mgtm = np.concatenate((intg.cumtrapz(mass_function[::-1] * M[::-1], dx=np.log(M[1]) - np.log(M[0]))[::-1], np.zeros(1))) + int_upper
# We need to set ngtm back in the original length vector with nans where they were originally
if len(self.__mgtm) < len(self.M):
mgtm_temp = np.zeros_like(self.dndlnm)
mgtm_temp[:] = np.nan
mgtm_temp[np.logical_not(np.isnan(self.dndlnm))] = self.__mgtm
self.__mgtm = mgtm_temp
return self.__mgtm
@mgtm.deleter
def mgtm(self):
try:
del self.__mgtm
except:
pass
@property
def nltm(self):
"""
Inverse cumulative mass function, ``len=len(M)`` [units :math:`h^3 Mpc^{-3}`]
"""
try:
return self.__nltm
except:
# set M and mass_function within computed range
M = self.M[np.logical_not(np.isnan(self.dndlnm))]
mass_function = self.dndlnm[np.logical_not(np.isnan(self.dndlnm))]
# Calculate the mass function (and its integral) from 10**3 up to lowest M
if M[0] > 10 ** 3:
m_lower, mf = self._lower_ngtm(M, np.log(mass_function), M[0] > self.M[0])
int_lower = intg.simps(np.exp(mf), dx=m_lower[2] - m_lower[1], even='first')
else:
int_lower = 0
# Calculate the cumulative integral of mass_function (Adding on the lower integral)
self.__nltm = np.concatenate((np.zeros(1), intg.cumtrapz(mass_function, dx=np.log(M[1]) - np.log(M[0])))) + int_lower
# We need to set ngtm back in the original length vector with nans where they were originally
if len(self.__nltm) < len(self.M):
nltm_temp = np.zeros_like(self.dndlnm)
nltm_temp[:] = np.nan
nltm_temp[np.logical_not(np.isnan(self.dndlnm))] = self.__nltm
self.__nltm = nltm_temp
return self.__nltm
@nltm.deleter
def nltm(self):
try:
del self.__nltm
except:
pass
@property
def mltm(self):
"""
Total mass in haloes `<M`, ``len=len(M)`` [units :math:`M_\odot h^2 Mpc^{-3}`]
"""
try:
return self.__mltm
except:
# Set M within calculated range
M = self.M[np.logical_not(np.isnan(self.dndlnm))]
mass_function = self.dndlnm[np.logical_not(np.isnan(self.dndlnm))]
# Calculate the mass function (and its integral) from 10**3 up to lowest M
if M[0] > 10 ** 3:
m_lower, mf = self._lower_ngtm(M, np.log(mass_function), M[0] > self.M[0])
int_lower = intg.simps(np.exp(mf + m_lower), dx=m_lower[2] - m_lower[1], even='first')
else:
int_lower = 0
# Calculate the cumulative integral of mass_function (Adding on the lower integral)
self.__mltm = np.concatenate((np.zeros(1), intg.cumtrapz(mass_function * M, dx=np.log(M[1]) - np.log(M[0])))) + int_lower
# We need to set ngtm back in the original length vector with nans where they were originally
if len(self.__mltm) < len(self.M):
nltm_temp = np.zeros_like(self.dndlnm)
nltm_temp[:] = np.nan
nltm_temp[np.logical_not(np.isnan(self.dndlnm))] = self.__mltm
self.__mltm = nltm_temp
return self.__mltm
@property
def how_big(self):
"""
Size of simulation volume in which to expect one halo of mass M, ``len=len(M)`` [units :math:`Mpch^{-1}`]
"""
return self.ngtm ** (-1. / 3.)
@how_big.deleter
def how_big(self):
try:
del self.how_big
except:
pass
transfer = Transfer.find_one({'_id': id, 'queued': None})
if not transfer:
return
try:
data = download_torrent(transfer['src'])
except DownloadError, e:
logger.error('failed to start torrent: %s', str(e))
return
try:
info = get_torrent_client().add_torrent(data)
Transfer.update({'_id': transfer['_id']},
{'$set': {
'type': 'torrent',
'queued': datetime.utcnow(),
'info': info,
}}, safe=True)
except TorrentExists, e:
Transfer.update({'_id': transfer['_id']},
{'$set': {'finished': datetime.utcnow()}},
safe=True)
logger.debug('failed to start torrent: %s', str(e))
except (TransmissionError, TorrentError), e:
Transfer.update({'_id': transfer['_id']}, {'$set': {
'started': None,
'tries': 0,
}}, safe=True)
logger.error('failed to start torrent: %s', str(e))
def clean_aborted():
Transfer.update({'finished': None, 'is_mount': True},
{'$set': {'finished': datetime.utcnow()}}, multi=True, safe=True)
Transfer.update({'finished': None, 'queued': None},
{'$set': {'started': None}}, multi=True, safe=True)