def set_xspec(obsid, model = None,
rmffile='/Users/corcoran/Dropbox/nicer_cal/nicer_v0.06.rmf',
arffile='/Users/corcoran/Dropbox/nicer_cal/ni_xrcall_onaxis_v0.06.arf',
workdir='/Users/corcoran/research/WR140/NICER/work/',
ignore='0.0-0.45, 5.-**', showplot=False, showmodel=False
):
"""
sets the pha and model instances for a given obsid
:param obsid: NICER observation id (integer)
:param rmffile: NICER response file
:param arffile: NICER effective area file
:param workdir: NICER dataset working directory
:param ignore: energy range in keV to ignore when fitting
:param showplot: if True plot the model and spectrum vs. energy
:param showmodel: if True print the model parameters
:return:
"""
import xspec
import matplotlib as plt
obs = str(obsid)
xspec.AllData.clear()
xspec.AllModels.clear()
xspecdir = os.path.join(workdir, obs)
phaname = os.path.join(xspecdir, 'ni{obs}_0mpu7_cl.pha'.format(obs=obsid))
print "phaname = {phaname}".format(phaname=phaname)
try:
pha = xspec.Spectrum(phaname)
except:
print "Can't find {phaname}; returning".format(phaname=phaname)
return 0, 0
pha.response = rmffile
pha.response.arf = arffile
pha.ignore(ignore)
#
# Define a model
#
if not model:
model = set_model()
if showmodel:
with sys_pipes():
model.show()
if showplot:
xspec.Plot.setRebin(minSig=3, maxBins=30)
xspec.Plot.device = "/null"
xspec.Plot.xAxis = "keV"
xspec.Plot.yLog = "True"
xspec.Plot("data")
plt.figure(figsize=[10, 6])
plt.yscale('log')
plt.xscale('log')
plt.xlabel('Energy (keV)', fontsize=16)
plt.step(xspec.Plot.x(), xspec.Plot.y())
plt.step(xspec.Plot.x(), xspec.Plot.model())
return pha, model
def test_sys_pipes():
stdout = io.StringIO()
stderr = io.StringIO()
with mock.patch('sys.stdout', stdout), mock.patch('sys.stderr', stderr), sys_pipes():
printf(u"Hellø")
printf_err(u"Hi, stdérr")
assert stdout.getvalue() == u"Hellø\n"
assert stderr.getvalue() == u"Hi, stdérr\n"
def __init__(self):
if PL.bitfile_name != BITFILE:
raise ValueError("mqttsn_publish.bit must be loaded.")
self.interface_string = CFFI_INTERFACE
self.ffi = FFI()
self.dll_name = SHARED_LIB
self.dll = None
self.sensor_ptr = None
self.ffi.cdef(self.interface_string)
with sys_pipes():
self.dll = self.ffi.dlopen(self.dll_name)
def _safe_wrapper(name: str, *args, check_return: bool=True, **kwargs) -> any:
"""Wrapper to call C functions, checking if they exist and their return status.
name: C function name
check_return: Flag to treat return value as a status (non-zero is failure)
return: C function return value
"""
with sys_pipes():
if not hasattr(_lib, name):
raise RuntimeError(f"Function {name} not in library")
ret = getattr(_lib, name)(*args, **kwargs)
if check_return and ret:
raise RuntimeError(f"Function {name} call failed")
return ret
def publish_cffi(self, size, count, pl_mac_address, pl_ip_address,
server_ip_address, server_port_number,
topic_id, qos, verbose, net_iop, sensor_iop):
"""Publish data from the given temperature sensor to an MQTTSN server.
This method will use the CFFI to control the accelerator.
Parameters
----------
size : int
The size of frames to generate.
count : int
The number of publish events to complete.
pl_mac_address : int/str
The MAC Address of the PL accelerator (not the host MAC address).
pl_ip_address : int/str
The IP Address of the PL accelerator (not the host IP address).
server_ip_address : int/str
The IP Address of the MQTTSN server.
server_port_number : int
The port number of the MQTTSN server.
topic_id : int
The topic ID to publish on.
qos : int
The MQTTSN qos to use (0 means response is not required).
verbose : int
A non-zero value will get verbose debugging information.
net_iop : NetworkIOP
The network IOP object.
sensor_iop : Pmod_TMP2
The temperature sensor object.
"""
pl_ip = pl_ip_address if type(pl_ip_address) is int \
else ip_str_to_int(pl_ip_address)
pl_mac = pl_mac_address if type(pl_mac_address) is int \
else mac_str_to_int(pl_mac_address)
server_ip = server_ip_address if type(server_ip_address) is int \
else ip_str_to_int(server_ip_address)
mac_address_arg = self.ffi.cast("unsigned long long", pl_mac)
ip_address_arg = self.ffi.cast("unsigned int", pl_ip)
server_ip_arg = self.ffi.cast("unsigned int", server_ip)
with sys_pipes():
net_iop_ptr = self.map(net_iop.mmio)
sensor_iop_ptr = self.map(sensor_iop.mmio)
ol.dll.Top(size, count, mac_address_arg, ip_address_arg,
server_ip_arg, server_port_number,
topic_id, qos, verbose,
net_iop_ptr, sensor_iop_ptr)
def fit_model(model, d, warmup=5000, iter=20000):
"""Fit a Stan model.
Args:
model: The compiled Stan model
d: The ranking data
iter: The number of iterations (after warmup)
Returns:
A tuple of model fit (a Stan object) and samples from that fit (a dataframe)
"""
# The data that needs to be passed to the Stan model
groups = list(d.index.get_level_values('Group').unique())
#data_ = {
# 'n_odorants': 10, # How many odorants
# 'n_individuals': d.shape[0], # How many individuals
# 'n_groups': len(groups), # How many groups
# 'group_id': d.index.map(lambda x: groups.index(x[0])+1).values, # Integer group IDs for each individual
# 'ranks': d.iloc[:, -10:].values # The last 10 columns of the dataframe, i.e. the ranking data
#}
individuals = d.index.unique().tolist()
data_ = {
'n_odorants': 10, # How many odorants
'n_observations': d.shape[0], # How many individuals
'n_groups': len(groups), # How many groups
'n_individuals': len(individuals), # How many groups
'group_id': d.index.map(lambda x: groups.index(x[0])+1).values, # Integer group IDs for each observation
'individual_id': d.index.map(lambda x: individuals.index(x)+1).values, # Integer individual IDs for each observation
'ranks': d.iloc[:, -10:].values # The last 10 columns of the dataframe, i.e. the ranking data
}
with sys_pipes(): # Used to pipe C-level output to the browser so I can see sampling messages.
# Sample (fit) the model
# None of these parameters matter except in the sense that sampling must proceed
# slowly enough to get the answer without getting infinities.
# This is all standard fare.
fit = model.sampling(data=data_, warmup=warmup, iter=iter, chains=4, control={'adapt_delta': 0.85, 'max_treedepth': 15})
# You may see many warning messages but basically as long as nothing blows up and Rhat ~ 1 then it is OK.
# Put the results into a Pandas dataframe
samples = fit.to_dataframe()
return fit, samples
def fit_spectrum(obsid, pha, model, writexcm=True,
workdir='/Users/corcoran/research/WR140/NICER/work/',
statMethod='cstat'
):
import xspec
from wurlitzer import sys_pipes
import pylab as plt
from heasarc.utils import xspec_utils as xu
if type(pha) != int:
#
# do fit
#
xspec.Fit.statMethod = statMethod
xspec.Fit.perform()
print("Best Fit Model is\n")
with sys_pipes():
model.show()
pha.notice("0.4-10.0")
xspec.Plot.setRebin(minSig=3, maxBins=30)
xspec.Plot.device = "/null"
xspec.Plot.xAxis = "keV"
xspec.Plot.yLog = "True"
xspec.Plot("data")
plt.figure(figsize=[10, 6])
plt.yscale('log')
plt.xscale('log')
plt.xlabel('Energy (keV)', fontsize=16)
plt.title("{obs}".format(obs=obsid), fontsize=16)
plt.errorbar(xspec.Plot.x(), xspec.Plot.y(),
xerr=xspec.Plot.xErr(), yerr=xspec.Plot.yErr(), fmt='.')
plt.step(xspec.Plot.x(), xspec.Plot.model(), where="mid")
band = "0.5-10 keV"
xspec.AllModels.calcFlux(band.replace(' keV', '').replace('-',' '))
print "Flux is {0:.3e} in the {1} band".format(pha.flux[0], band)
if writexcm:
xcmfile = os.path.join(workdir, str(obsid), 'ni{obsid}_0mpu7_cl'.format(obsid=obsid))
xu.write_xcm(xcmfile, pha, model=model)
else:
print "Can't fit OBSID {obs}".format(obs=obsid)
return pha, model
def capture_cpp_log_context(verbose=False):
"""Creates a context to display or hide the c++ code logs to the user.
Tested with python, ipython, colab and jupyter notebook.
In the internal build, only impact python's print.
Args:
verbose: If true, the training logs are shown in logging and python's print.
If false, the training logs are not shown in logging nor python's print.
Returns:
A context.
"""
# Does nothing
@contextmanager
def no_op_context():
yield
# Hide the Yggdrasil training logs.
@contextmanager
def hide_cpp_logs():
# Stop displaying c++ logs.
set_yggdrasil_logging_level(0)
try:
yield
finally:
# Re-start displaying c++ logs.
set_yggdrasil_logging_level(2)
if not verbose:
# Make sure the c++ logs are not visible to the user.
return hide_cpp_logs()
def is_direct_output(stream):
"""Checks if output stream redirects to the shell/console directly."""
if stream.isatty():
return True
if isinstance(stream, io.TextIOWrapper):
return is_direct_output(stream.buffer)
if isinstance(stream, io.BufferedWriter):
return is_direct_output(stream.raw)
if isinstance(stream, io.FileIO):
return stream.fileno() in [1, 2]
return False
if ((REDIRECT_YGGDRASIL_CPP_OUTPUT_TO_PYTHON_OUTPUT == "auto"
and is_direct_output(sys.stdout))
or not REDIRECT_YGGDRASIL_CPP_OUTPUT_TO_PYTHON_OUTPUT):
# The cout and cerr of the c++ library are already visible to the user.
return no_op_context()
global REDIRECT_MESSAGE_WAS_PRINTED
if not REDIRECT_MESSAGE_WAS_PRINTED:
REDIRECT_MESSAGE_WAS_PRINTED = True
info(
"Standard output detected as not visible to the user e.g. running "
"in a notebook. Creating a training log redirection. If training get "
"stuck, try calling tfdf.keras.set_training_logs_redirection(False)."
)
# pytype: disable=import-error
# pylint: disable=g-import-not-at-top
# pylint: disable=g-importing-member
# pylint: disable=bare-except
# The cout and cerr of the c++ library are not visible to the user.
# Redirect them to python's standard output.
try:
from colabtools.googlelog import CaptureLog
return CaptureLog()
except:
try:
from wurlitzer import sys_pipes
# This can hang if the cout/cerr is visible to the user.
return sys_pipes()
except:
warning(
"Cannot redirect the training output because neither of "
"colabtools.googlelog or wurlitzer available. Run `pip install "
"wurlitzer -U` and try again.")
return no_op_context()
def _safe_wrapper(name, *args, **kwargs):
with sys_pipes():
if not hasattr(_lib, name):
raise RuntimeError(f"Function {name} not in library")
if getattr(_lib, name)(*args, **kwargs):
raise RuntimeError(f"Function {name} call failed")
def ocaml(line, cell):
with sys_pipes():
return toploop.eval(cell)
def ocaml_t(line):
type_, body = line.split(':', maxsplit=1)
with sys_pipes():
return toploop.get(type_, body)
def ocaml(line):
with sys_pipes():
return toploop.get(line)
def init_ethernet_raw(self, interface, port=1884):
"""Initialize the Ethernet raw interface. """
with sys_pipes():
self.dll.init_ethernet_raw(interface.encode('utf-8'), port)
def _parse_meta_yaml_impl(
text: str,
for_pinning=False,
platform=None,
arch=None,
recipe_dir=None,
cbc_path=None,
log_debug=False,
**kwargs: Any,
) -> "MetaYamlTypedDict":
from conda_build.config import Config
from conda_build.metadata import parse, MetaData
import conda_build.api
import conda_build.environ
from conda_build.variants import explode_variants
if (recipe_dir is not None and cbc_path is not None and arch is not None
and platform is not None):
with tempfile.TemporaryDirectory() as tmpdir:
with open(os.path.join(tmpdir, "meta.yaml"), "w") as fp:
fp.write(text)
def _run_parsing():
config = conda_build.config.get_or_merge_config(
None,
platform=platform,
arch=arch,
exclusive_config_file=cbc_path,
)
_cbc, _ = conda_build.variants.get_package_combined_spec(
tmpdir,
config=config,
)
return config, _cbc
if not log_debug:
fout = io.StringIO()
ferr = io.StringIO()
with sys_pipes(), contextlib.redirect_stdout(
fout, ), contextlib.redirect_stderr(ferr):
config, _cbc = _run_parsing()
else:
config, _cbc = _run_parsing()
cfg_as_dict = {}
for var in explode_variants(_cbc):
try:
m = MetaData(tmpdir, config=config, variant=var)
except SystemExit as e:
raise RuntimeError(str(e))
cfg_as_dict.update(conda_build.environ.get_dict(m=m))
logger.debug("jinja2 environmment:\n%s",
pprint.pformat(cfg_as_dict))
cbc = Config(
platform=platform,
arch=arch,
variant=cfg_as_dict,
**kwargs,
)
else:
_cfg = {}
_cfg.update(kwargs)
if platform is not None:
_cfg["platform"] = platform
if arch is not None:
_cfg["arch"] = arch
cbc = Config(**_cfg)
cfg_as_dict = {}
if for_pinning:
content = render_meta_yaml(text,
for_pinning=for_pinning,
**cfg_as_dict)
else:
content = render_meta_yaml(text, **cfg_as_dict)
try:
return parse(content, cbc)
except Exception:
logger.debug("template: %s", text)
logger.debug("context:\n%s", pprint.pformat(cfg_as_dict))
raise
lmbda = 1e-2
opt = cbpdn.ConvBPDN.Options({
'Verbose': True,
'MaxMainIter': 250,
'HighMemSolve': True,
'RelStopTol': 5e-3,
'AuxVarObj': False
})
"""
If GPU available, run CUDA ConvBPDN solver, otherwise run standard Python version.
"""
if cuda.device_count() > 0:
print('%s GPU found: running CUDA solver' % cuda.device_name())
tm = util.Timer()
with sys_pipes(), util.ContextTimer(tm):
X = cuda.cbpdn(D, sh, lmbda, opt)
t = tm.elapsed()
else:
print('GPU not found: running Python solver')
c = cbpdn.ConvBPDN(D, sh, lmbda, opt)
X = c.solve().squeeze()
t = c.timer.elapsed('solve')
print('Solve time: %.2f s' % t)
"""
Reconstruct the image from the sparse representation.
"""
shr = np.sum(spl.fftconv(D, X), axis=2)
imgr = sl + shr
print("Reconstruction PSNR: %.2fdB\n" % spm.psnr(img, imgr))
