def test_reinit_from_numpy_array(self):
t0 = core.G3Time('2019-01-01T12:30:00')
timestamps = np.linspace(t0.time,
t0.time + 1e7 * SEC,
3000,
dtype='int64')
vectime = core.G3VectorTime(timestamps)
revectime = core.G3VectorTime(np.asarray(vectime))
assert ((np.asarray(revectime) == timestamps).all())
def _concat_hk_stream(blocks_in):
"""Concatenates an ordered list of compatible HK blocks into a single
frame. Each block should be a valid G3TimesampleMap with the same
keys.
Returns a single G3TimesampleMap with all fields concatenated.
"""
blk = core.G3TimesampleMap()
blk.times = core.G3VectorTime(blocks_in[0].times)
fields = list(blocks_in[0].keys())
for f in fields:
f_short = f.split('.')[-1]
blk[f] = blocks_in[0][f_short]
for b in blocks_in[1:]:
blk.times.extend(b.times)
for f in fields:
f_short = f.split('.')[-1]
for _type in _SCHEMA_V1_BLOCK_TYPES:
if isinstance(blocks_in[0][f_short], _type):
break
else:
raise RuntimeError('Field "%s" is of unsupported type %s.' %
(f_short, type(blocks_in[0][f_short])))
for b in blocks_in[1:]:
blk[f].extend(b[f_short])
return blk
def test_from_numpy_array(self):
t0 = core.G3Time('2019-01-01T12:30:00')
timestamps = np.linspace(t0.time, t0.time + 1e7 * SEC, 3000)
vectime = core.G3VectorTime(timestamps)
assert (vectime[0] == t0)
assert (vectime[-1] == core.G3Time(timestamps[-1]))
assert (len(vectime) == len(timestamps))
def test_11_fallback(self):
"""Test that the code does not fail to serialize short segments or
highly random data.
"""
# Short segments
for nsamp in range(1, 20):
ts = self._get_ts(1, nsamp, sigma=0, dtype='int32')
ts.encode()
self._readback_compare(ts)
# Random time vector.
n = 200
ts = self._get_ts(1, n, sigma=0, dtype='int32')
ts.times = core.G3VectorTime((np.random.uniform(size=n * 8) *
256).astype('uint8').view(dtype='int64'))
self._readback_compare(ts)
# Random data array.
n = 200
ts = self._get_ts(1, n, sigma=0, dtype='int64')
ts.data = (np.random.uniform(size=n * 8) *
256).astype('uint8').view(dtype='int64').reshape(1, -1)
self._readback_compare(ts)
# Small n_det (note 1-10 weren't causing a problem by 11+ were...)
for n_det in range(1, 20):
ts = self._get_ts(n_det, 1, sigma=0, dtype='int64')
ts.data = (np.random.uniform(size=n_det*8) * 256).astype('uint8') \
.view(dtype='int64').reshape(-1, 1)
ts.encode()
self._readback_compare(ts)
def get_g3_time(unix_time):
"""Convert a double or numpy array of floats to G3Time or
G3VectorTime."""
src = None
if isinstance(unix_time, core.G3VectorDouble):
src = (np.array(unix_time) * core.G3Units.seconds).astype('int')
elif isinstance(unix_time, np.ndarray) and unix_time.ndim == 1:
src = (unix_time * core.G3Units.seconds).astype('int')
if src is not None:
return core.G3VectorTime([core.G3Time(t) for t in src])
return core.G3Time(int(unix_time * core.G3Units.seconds))
def get_v2_stream():
"""Generate some example HK data, in schema version 2.
Returns a list of frames constituting a valid version 2 HK stream.
"""
# Create something to help us track the aggregator session.
hksess = so3g.hk.HKSessionHelper(session_id=1234,
hkagg_version=2,
description="Test HK data.")
# Register a data provider.
prov_id = hksess.add_provider(
description='Fake data for the real world.')
# Start the stream -- write the initial session and status frames.
frames = [
hksess.session_frame(),
hksess.status_frame(),
]
# Now make a data frame.
f = hksess.data_frame(prov_id=prov_id)
# Add some data blocks.
hk = core.G3TimesampleMap()
hk.times = core.G3VectorTime([core.G3Time(i*core.G3Units.seconds) for i in [0, 1, 2, 3, 4]])
hk['speed'] = core.G3VectorDouble([1.2, 1.2, 1.2, 1.2, 1.2])
f['blocks'].append(hk)
f['block_names'].append('group0')
hk = core.G3TimesampleMap()
hk.times = core.G3VectorTime([core.G3Time(i*core.G3Units.seconds) for i in [0, 1, 2, 3, 4]])
hk['position'] = core.G3VectorInt([1, 2, 3, 4, 5])
hk['mode'] = core.G3VectorString(['going', 'going', 'going', 'going', 'gone/'])
f['blocks'].append(hk)
f['block_names'].append('group1')
frames.append(f)
return frames
def get_test_block(length, keys=['a', 'b', 'c', 'd'], offset=0, ordered=True):
type_cycle = [(core.G3VectorDouble, float), (core.G3VectorInt, int),
(core.G3VectorString, str), (core.G3VectorBool, bool)]
t0 = core.G3Time('2019-01-01T12:30:00') + offset * SEC
m = core.G3TimesampleMap()
times = np.arange(length)
if not ordered:
np.random.shuffle(times)
m.times = core.G3VectorTime(t0 + times * SEC)
for i, k in enumerate(keys):
y = (np.random.uniform(size=length) * 100).astype(int)
constructor, cast_func = type_cycle[i % len(type_cycle)]
vect = constructor(list(map(cast_func, y)))
m[k] = vect
return m
def g3_cast(data, time=False):
"""
Casts a generic datatype into a corresponding G3 type. With:
int -> G3Int
str -> G3String
float -> G3Double
and lists of type X will go to G3VectorX. If ``time`` is set to True, will
convert to G3Time or G3VectorTime with the assumption that ``data`` consists
of unix timestamps.
Args:
data (int, str, float, or list):
Generic data to be converted to a corresponding G3Type.
time (bool, optional):
If True, will assume data contains unix timestamps and try to cast
to G3Time or G3VectorTime.
Returns:
g3_data:
Corresponding G3 datatype.
"""
is_list = isinstance(data, list)
if is_list:
dtype = type(data[0])
if not all(isinstance(d, dtype) for d in data):
raise TypeError("Data list contains varying types!")
else:
dtype = type(data)
if dtype not in _g3_casts.keys():
raise TypeError("g3_cast does not support type {}. Type must"
"be one of {}".format(dtype, _g3_casts.keys()))
if is_list:
if time:
return core.G3VectorTime(
list(map(lambda t: core.G3Time(t * core.G3Units.s), data)))
else:
cast = _g3_list_casts[type(data[0])]
return cast(data)
else:
if time:
return core.G3Time(data * core.G3Units.s)
else:
cast = _g3_casts[type(data)]
return cast(data)
def get_data_frame(self, start, stop):
times = np.arange(start, stop, 1. / self.sample_rate)
nsamps = len(times)
chans = np.arange(self.nchans)
names = [f'r{ch:0>4}' for ch in chans]
count_per_phi0 = 2**16
data = np.zeros((self.nchans, nsamps), dtype=np.int32)
data += count_per_phi0 * chans[:, None]
data += (count_per_phi0 * 0.2 *
np.sin(2 * np.pi * 8 * times)).astype(int)
data += (count_per_phi0 *
np.random.normal(0, 0.03, (self.nchans, nsamps))).astype(int)
fr = core.G3Frame(core.G3FrameType.Scan)
g3times = core.G3VectorTime(times * core.G3Units.s)
fr['data'] = so3g.G3SuperTimestream(names, g3times, data)
primary_names = [
'UnixTime', 'FluxRampIncrement', 'FluxRampOffset', 'Counter0',
'Counter1', 'Counter2', 'AveragingResetBits', 'FrameCounter',
'TESRelaySetting'
]
primary_data = np.zeros((len(primary_names), nsamps), dtype=np.int64)
primary_data[0, :] = (times * 1e9).astype(int)
fr['primary'] = so3g.G3SuperTimestream(primary_names, g3times,
primary_data)
tes_bias_names = [f'bias{bg:0>2}' for bg in range(NBIASLINES)]
bias_data = np.zeros((NBIASLINES, nsamps), dtype=np.int32)
fr['tes_biases'] = so3g.G3SuperTimestream(tes_bias_names, g3times,
bias_data)
fr['timing_paradigm'] = 'Low Precision'
fr['num_samples'] = nsamps
self.tag_frame(fr)
return fr
def _get_ts(self,
nchans,
ntimes,
sigma=256,
dtype='int32',
raw=False,
seed=None):
if seed is not None:
np.random.seed(seed)
names = ['x%i' % i for i in range(nchans)]
times = core.G3VectorTime(
(1680000000 + np.arange(ntimes) * .005) * core.G3Units.s)
data = (np.random.normal(size=(len(names), len(times))) *
sigma).astype(dtype)
if raw:
return names, times, data
ts = so3g.G3SuperTimestream()
ts.names = names
ts.times = times
if dtype in FLOAT_DTYPES:
ts.quanta = np.ones(len(names))
ts.data = data
return ts
def test_copy_constructor(self):
t0 = core.G3VectorTime(np.array([100000000, 200000000]))
t1 = core.G3VectorTime(t0)
assert ((np.asarray(t0) == np.asarray(t1)).all())
def test_from_list(self):
t0 = core.G3Time('2019-01-01T12:30:00')
vectime = core.G3VectorTime([t0, t0 + 10 * SEC])
assert (vectime[0].time == t0.time)
assert (vectime[1].time == t0.time + 10 * SEC)
halfscan = 10 # degrees
for i in range(10):
# Number of samples
n = int(halfscan / v_az / dt)
# Vector of unix timestamps
t = frame_time + dt * np.arange(n)
# Vector of az and el
az = v_az * dt * np.arange(n)
if i % 2:
az = -az
el = az * 0 + 50.
# Construct a "block", which is a named G3TimesampleMap.
block = core.G3TimesampleMap()
block.times = core.G3VectorTime(
[core.G3Time(_t * core.G3Units.s) for _t in t])
block['az'] = core.G3VectorDouble(az)
block['el'] = core.G3VectorDouble(el)
# Create an output data frame template associated with this
# provider.
frame = session.data_frame(prov_id)
# Add the block and block name to the frame, and write it.
frame['block_names'].append('pointing')
frame['blocks'].append(block)
writer.Process(frame)
# For next iteration.
frame_time += n * dt
def cache_to_frames(tod,
start_frame,
n_frames,
frame_offsets,
frame_sizes,
common=None,
detector_fields=None,
flag_fields=None,
detector_map="detectors",
flag_map="flags",
units=None):
"""Gather all data from the distributed cache for a single frame.
Args:
tod (toast.TOD): instance of a TOD class.
start_frame (int): the first frame index.
n_frames (int): the number of frames.
frame_offsets (list): list of the first samples of all frames.
frame_sizes (list): list of the number of samples in each frame.
common (tuple): (cache name, G3 type, frame name) of each common
field.
detector_fields (tuple): (cache name, frame name) of each detector
field.
flag_fields (tuple): (cache name, frame name) of each flag field.
detector_map (str): the name of the frame timestream map.
flag_map (str): then name of the frame flag map.
units: G3 units of the detector data.
"""
# Local sample range
local_first = tod.local_samples[0]
nlocal = tod.local_samples[1]
# The process grid
detranks, sampranks = tod.grid_size
rankdet, ranksamp = tod.grid_ranks
# Helper function:
# For a given timestream, the gather is done across the
# process row which contains the specific detector, or across
# the first process row for common telescope data.
def gather_field(prow, fld, indx, cacheoff, ncache):
gproc = 0
gdata = None
# We are going to allreduce this later, so that every process
# knows the dimensions of the field.
allnnz = 0
if rankdet == prow:
#print(" proc {} doing gather of {}".format(tod.mpicomm.rank, fld), flush=True)
# This process is in the process row that has this field,
# participate in the gather operation.
pdata = None
# Find the data type and shape from the cache object
mtype = None
ref = tod.cache.reference(fld)
nnz = 1
if (len(ref.shape) > 1) and (ref.shape[1] > 0):
nnz = ref.shape[1]
if ref.dtype == np.dtype(np.float64):
mtype = MPI.DOUBLE
elif ref.dtype == np.dtype(np.int64):
mtype = MPI.INT64_T
elif ref.dtype == np.dtype(np.int32):
mtype = MPI.INT32_T
elif ref.dtype == np.dtype(np.uint8):
mtype = MPI.UINT8_T
else:
msg = "Cannot gather cache field {} of type {}"\
.format(fld, ref.dtype)
raise RuntimeError(msg)
#print("field {}: proc {} has nnz = {}".format(fld, tod.mpicomm.rank, nnz), flush=True)
pz = 0
if cacheoff is not None:
pdata = ref.flatten()[nnz * cacheoff:nnz * (cacheoff + ncache)]
pz = nnz * ncache
else:
pdata = np.zeros(0, dtype=ref.dtype)
psizes = tod.grid_comm_row.gather(pz, root=0)
disp = None
totsize = None
if ranksamp == 0:
#print("Gathering field {} with type {}".format(fld, mtype), flush=True)
# We are the process collecting the gathered data.
gproc = tod.mpicomm.rank
allnnz = nnz
# Compute the displacements into the receive buffer.
disp = [0]
for ps in psizes[:-1]:
last = disp[-1]
disp.append(last + ps)
totsize = np.sum(psizes)
# allocate receive buffer
gdata = np.zeros(totsize, dtype=ref.dtype)
#print("Gatherv psizes = {}, disp = {}".format(psizes, disp), flush=True)
#print("field {}: proc {} start Gatherv".format(fld, tod.mpicomm.rank), flush=True)
tod.grid_comm_row.Gatherv(pdata, [gdata, psizes, disp, mtype],
root=0)
#print("field {}: proc {} finish Gatherv".format(fld, tod.mpicomm.rank), flush=True)
del disp
del psizes
del pdata
del ref
# Now send this data to the root process of the whole communicator.
# Only one process (the first one in process row "prow") has data
# to send.
# Create a unique message tag
mtag = 10 * indx
#print(" proc {} hit allreduce of gproc".format(tod.mpicomm.rank), flush=True)
# All processes find out which one did the gather
gproc = tod.mpicomm.allreduce(gproc, MPI.SUM)
# All processes find out the field dimensions
allnnz = tod.mpicomm.allreduce(allnnz, MPI.SUM)
#print(" proc {} for field {}, gproc = {}".format(tod.mpicomm.rank, fld, gproc), flush=True)
#print("field {}: proc {}, gatherproc = {}, allnnz = {}".format(fld, tod.mpicomm.rank, gproc, allnnz), flush=True)
rdata = None
if gproc == 0:
if gdata is not None:
if allnnz == 1:
rdata = gdata
else:
rdata = gdata.reshape((-1, allnnz))
else:
# Data not yet on rank 0
if tod.mpicomm.rank == 0:
# Receive data from the first process in this row
#print(" proc {} for field {}, recv type".format(tod.mpicomm.rank, fld), flush=True)
rtype = tod.mpicomm.recv(source=gproc, tag=(mtag + 1))
#print(" proc {} for field {}, recv size".format(tod.mpicomm.rank, fld), flush=True)
rsize = tod.mpicomm.recv(source=gproc, tag=(mtag + 2))
#print(" proc {} for field {}, recv data".format(tod.mpicomm.rank, fld), flush=True)
rdata = np.zeros(rsize, dtype=np.dtype(rtype))
tod.mpicomm.Recv(rdata, source=gproc, tag=mtag)
# Reshape if needed
if allnnz > 1:
rdata = rdata.reshape((-1, allnnz))
elif (tod.mpicomm.rank == gproc):
# Send our data
#print(" proc {} for field {}, send {} samples of {}".format(tod.mpicomm.rank, fld, len(gdata), gdata.dtype.char), flush=True)
#print(" proc {} for field {}, send type with tag = {}".format(tod.mpicomm.rank, fld, mtag+1), flush=True)
tod.mpicomm.send(gdata.dtype.char, dest=0, tag=(mtag + 1))
#print(" proc {} for field {}, send size with tag = {}".format(tod.mpicomm.rank, fld, mtag+2), flush=True)
tod.mpicomm.send(len(gdata), dest=0, tag=(mtag + 2))
#print(" proc {} for field {}, send data with tag {}".format(tod.mpicomm.rank, fld, mtag), flush=True)
tod.mpicomm.Send(gdata, 0, tag=mtag)
return rdata
# For efficiency, we are going to gather the data for all frames at once.
# Then we will split those up when doing the write.
# Frame offsets relative to the memory buffers we are gathering
fdataoff = [0]
for f in frame_sizes[:-1]:
last = fdataoff[-1]
fdataoff.append(last + f)
# The list of frames- only on the root process.
fdata = None
if tod.mpicomm.rank == 0:
fdata = [c3g.G3Frame(c3g.G3FrameType.Scan) for f in range(n_frames)]
else:
fdata = [None for f in range(n_frames)]
# Compute the overlap of all frames with the local process. We want to
# to find the full sample range that this process overlaps the total set
# of frames.
cacheoff = None
ncache = 0
for f in range(n_frames):
# Compute overlap of the frame with the local samples.
fcacheoff, froff, nfr = local_frame_indices(local_first, nlocal,
frame_offsets[f],
frame_sizes[f])
#print("proc {}: frame {} has cache off {}, fr off {}, nfr {}".format(tod.mpicomm.rank, f, fcacheoff, froff, nfr), flush=True)
if fcacheoff is not None:
if cacheoff is None:
cacheoff = fcacheoff
ncache = nfr
else:
ncache += nfr
#print("proc {}: cache off now {}, ncache now {}".format(tod.mpicomm.rank, cacheoff, ncache), flush=True)
# Now gather the full sample data one field at a time. The root process
# splits up the results into frames.
# First gather common fields from the first row of the process grid.
for findx, (cachefield, g3t, framefield) in enumerate(common):
#print("proc {} entering gather_field(0, {}, {}, {}, {})".format(tod.mpicomm.rank, cachefield, findx, cacheoff, ncache), flush=True)
data = gather_field(0, cachefield, findx, cacheoff, ncache)
if tod.mpicomm.rank == 0:
#print("Casting field {} to type {}".format(field, g3t), flush=True)
if g3t == c3g.G3VectorTime:
# Special case for time values stored as int64_t, but
# wrapped in a class.
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
g3times = list()
for t in range(ndata):
g3times.append(c3g.G3Time(data[dataoff + t]))
fdata[f][framefield] = c3g.G3VectorTime(g3times)
del g3times
else:
# The bindings of G3Vector seem to only work with
# lists. This is probably horribly inefficient.
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
if len(data.shape) == 1:
fdata[f][framefield] = \
g3t(data[dataoff:dataoff+ndata].tolist())
else:
# We have a 2D quantity
fdata[f][framefield] = \
g3t(data[dataoff:dataoff+ndata,:].flatten().tolist())
del data
# Wait for everyone to catch up...
tod.mpicomm.barrier()
# For each detector field, processes which have the detector
# in their local_dets should be in the same process row.
# We do the gather over just this process row.
if (detector_fields is not None) or (flag_fields is not None):
dpats = {d: re.compile(".*{}.*".format(d)) for d in tod.local_dets}
detmaps = None
if detector_fields is not None:
if tod.mpicomm.rank == 0:
detmaps = [c3g.G3TimestreamMap() for f in range(n_frames)]
for dindx, (cachefield, framefield) in enumerate(detector_fields):
pc = -1
for det, pat in dpats.items():
if pat.match(cachefield) is not None:
#print("proc {} has field {}".format(tod.mpicomm.rank, field), flush=True)
pc = rankdet
break
# As a sanity check, verify that every process which
# has this field is in the same process row.
rowcheck = tod.mpicomm.gather(pc, root=0)
prow = 0
if tod.mpicomm.rank == 0:
rc = np.array([x for x in rowcheck if (x >= 0)],
dtype=np.int32)
#print(field, rc, flush=True)
prow = np.max(rc)
if np.min(rc) != prow:
msg = "Processes with field {} are not in the "\
"same row\n".format(cachefield)
sys.stderr.write(msg)
tod.mpicomm.abort()
# Every process finds out which process row is participating.
prow = tod.mpicomm.bcast(prow, root=0)
#print("proc {} got prow = {}".format(tod.mpicomm.rank, prow), flush=True)
# Get the data on rank 0
data = gather_field(prow, cachefield, dindx, cacheoff, ncache)
if tod.mpicomm.rank == 0:
if units is None:
# We do this conditional, since we can't use
# G3TimestreamUnits.None in python ("None" is
# interpreted as python None).
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
detmaps[f][framefield] = \
c3g.G3Timestream(data[dataoff:dataoff+ndata])
else:
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
detmaps[f][framefield] = \
c3g.G3Timestream(data[dataoff:dataoff+ndata],
units)
if tod.mpicomm.rank == 0:
for f in range(n_frames):
fdata[f][detector_map] = detmaps[f]
flagmaps = None
if flag_fields is not None:
if tod.mpicomm.rank == 0:
flagmaps = [c3g.G3MapVectorInt() for f in range(n_frames)]
for dindx, (cachefield, framefield) in enumerate(flag_fields):
pc = -1
for det, pat in dpats.items():
if pat.match(cachefield) is not None:
pc = rankdet
break
# As a sanity check, verify that every process which
# has this field is in the same process row.
rowcheck = tod.mpicomm.gather(pc, root=0)
prow = 0
if tod.mpicomm.rank == 0:
rc = np.array([x for x in rowcheck if (x >= 0)],
dtype=np.int32)
prow = np.max(rc)
if np.min(rc) != prow:
msg = "Processes with field {} are not in the "\
"same row\n".format(cachefield)
sys.stderr.write(msg)
tod.mpicomm.abort()
# Every process finds out which process row is participating.
prow = tod.mpicomm.bcast(prow, root=0)
# Get the data on rank 0
data = gather_field(prow, cachefield, dindx, cacheoff, ncache)
if tod.mpicomm.rank == 0:
# The bindings of G3Vector seem to only work with
# lists... Also there is no vectormap for unsigned
# char, so we have to use int...
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
flagmaps[f][framefield] = \
c3g.G3VectorInt(\
data[dataoff:dataoff+ndata].astype(np.int32)\
.tolist())
if tod.mpicomm.rank == 0:
for f in range(n_frames):
fdata[f][flag_map] = flagmaps[f]
return fdata
def split_field(data,
g3t,
framefield,
mapfield=None,
g3units=units,
times=None):
"""Split a gathered data buffer into frames- only on root process.
"""
if data is None:
return
if g3t == core3g.G3VectorTime:
# Special case for time values stored as int64_t, but
# wrapped in a class.
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
g3times = list()
for t in range(ndata):
g3times.append(core3g.G3Time(data[dataoff + t]))
if mapfield is None:
fdata[f][framefield] = core3g.G3VectorTime(g3times)
else:
fdata[f][framefield][mapfield] = \
core3g.G3VectorTime(g3times)
del g3times
elif g3t == so3g.IntervalsInt:
# Flag vector is written as a simple boolean.
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
# Extract flag vector (0 or 1) for this frame
frame_flags = (data[dataoff:dataoff + ndata] != 0).astype(int)
# Convert bit 0 to an IntervalsInt.
ival = so3g.IntervalsInt.from_mask(frame_flags, 1)[0]
if mapfield is None:
fdata[f][framefield] = ival
else:
fdata[f][framefield][mapfield] = ival
elif g3t == core3g.G3Timestream:
if times is None:
raise RuntimeError(
"You must provide the time stamp vector with a "
"Timestream object")
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
timeslice = times[cacheoff + dataoff:cacheoff + dataoff +
ndata]
tstart = timeslice[0] * 1e8
tstop = timeslice[-1] * 1e8
if mapfield is None:
if g3units is None:
fdata[f][framefield] = \
g3t(data[dataoff : dataoff + ndata])
else:
fdata[f][framefield] = \
g3t(data[dataoff : dataoff + ndata], g3units)
fdata[f][framefield].start = core3g.G3Time(tstart)
fdata[f][framefield].stop = core3g.G3Time(tstop)
else:
# Individual detector data. The only fields that
# we (optionally) compress.
if g3units is None:
tstream = g3t(data[dataoff:dataoff + ndata])
else:
tstream = g3t(data[dataoff:dataoff + ndata], g3units)
if compress and "compressor_gain_" + framefield in fdata[f]:
(tstream, gain,
offset) = recode_timestream(tstream, compress)
fdata[f]["compressor_gain_" +
framefield][mapfield] = gain
fdata[f]["compressor_offset_" +
framefield][mapfield] = offset
fdata[f][framefield][mapfield] = tstream
fdata[f][framefield][mapfield].start = core3g.G3Time(
tstart)
fdata[f][framefield][mapfield].stop = core3g.G3Time(tstop)
else:
# The bindings of G3Vector seem to only work with
# lists. This is probably horribly inefficient.
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
if len(data.shape) == 1:
fdata[f][framefield] = \
g3t(data[dataoff : dataoff + ndata].tolist())
else:
# We have a 2D quantity
fdata[f][framefield] = \
g3t(data[dataoff : dataoff + ndata, :].flatten()
.tolist())
return
def split_field(data, g3t, framefield, mapfield=None, g3units=units):
"""Split a gathered data buffer into frames.
"""
if tod.mpicomm.rank == 0:
if g3t == core3g.G3VectorTime:
# Special case for time values stored as int64_t, but
# wrapped in a class.
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
g3times = list()
for t in range(ndata):
g3times.append(core3g.G3Time(data[dataoff + t]))
if mapfield is None:
fdata[f][framefield] = core3g.G3VectorTime(g3times)
else:
fdata[f][framefield][mapfield] = \
core3g.G3VectorTime(g3times)
del g3times
elif g3t == so3g.IntervalsInt:
# This means that the data is actually flags
# and we should convert it into a list of intervals.
fint = flags_to_intervals(data)
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
datalast = dataoff + ndata
chunks = list()
idomain = (0, ndata - 1)
for intr in fint:
# Interval sample ranges are defined relative to the
# frame itself.
cfirst = None
clast = None
if (intr[0] < datalast) and (intr[1] >= dataoff):
# there is some overlap...
if intr[0] < dataoff:
cfirst = 0
else:
cfirst = intr[0] - dataoff
if intr[1] >= datalast:
clast = ndata - 1
else:
clast = intr[1] - dataoff
chunks.append([cfirst, clast])
if mapfield is None:
if len(chunks) == 0:
fdata[f][framefield] = \
so3g.IntervalsInt()
else:
fdata[f][framefield] = \
so3g.IntervalsInt.from_array(
np.array(chunks, dtype=np.int64))
fdata[f][framefield].domain = idomain
else:
if len(chunks) == 0:
fdata[f][framefield][mapfield] = \
so3g.IntervalsInt()
else:
fdata[f][framefield][mapfield] = \
so3g.IntervalsInt.from_array(
np.array(chunks, dtype=np.int64))
fdata[f][framefield][mapfield].domain = idomain
del fint
elif g3t == core3g.G3Timestream:
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
if mapfield is None:
if g3units is None:
fdata[f][framefield] = \
g3t(data[dataoff:dataoff+ndata])
else:
fdata[f][framefield] = \
g3t(data[dataoff:dataoff+ndata], g3units)
else:
if g3units is None:
fdata[f][framefield][mapfield] = \
g3t(data[dataoff:dataoff+ndata])
else:
fdata[f][framefield][mapfield] = \
g3t(data[dataoff:dataoff+ndata], g3units)
else:
# The bindings of G3Vector seem to only work with
# lists. This is probably horribly inefficient.
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
if len(data.shape) == 1:
fdata[f][framefield] = \
g3t(data[dataoff:dataoff+ndata].tolist())
else:
# We have a 2D quantity
fdata[f][framefield] = \
g3t(data[dataoff:dataoff+ndata, :].flatten()
.tolist())
return
