def main():
# Open data files and cast as andata objects.
data_list = [andata.CorrData(h5py.File(p, "r")) for p in paths]
# Define a dataset filter that takes the first 64 frequenies.
def dset_filter(dataset):
# Must have this attribute.
if "freq" in dataset.attrs["axis"]:
# Must be first axis.
if dataset.attrs["axis"][0] != "freq":
raise RuntimeError("Expected 'freq' to be zeroth axis.")
dataset = dataset[FREQ_SEL]
return dataset
for ii, d in enumerate(data_list):
out_f = h5py.File(OUT_FILENAMES[ii], "w")
tdata = andata.concatenate(
[d],
start=STARTS[ii],
stop=STOPS[ii],
out_group=out_f,
dataset_filter=dset_filter,
)
# Adjust the frequency index map.
freq = out_f["index_map/freq"][FREQ_SEL]
del out_f["index_map/freq"]
out_f.create_dataset("index_map/freq", data=freq)
# Adjust the attributes. XXX others?
out_f.attrs["n_freq"] = [len(freq)]
out_f.close()
def _process_current_day(self):
# Combine the current set of files into a timestream
day = str(self._current_day)
# Calculate the length of data in the current day
start = datetime.utcfromtimestamp(self._timestream_list[0].time[0])
end = datetime.utcfromtimestamp(self._timestream_list[-1].time[-1])
tdelta = end - start
day_length = tdelta.days * 24.0 + tdelta.seconds / 3600.0
# If the amount of data for this day is too small, then just skip
if day_length < self.min_span:
return None
self.log.debug("Constructing %s [%i files]", day,
len(self._timestream_list))
# Construct the combined timestream
ts = andata.concatenate(self._timestream_list)
# Add attributes for the date and a tag for labelling saved files
ts.attrs["tag"] = day
ts.attrs["date"] = day
return ts
def test_start_end_inds(data_list):
right_answer = andata.AnData.from_acq_h5(acq_fname_list, start=3, stop=26)
merged_data = andata.concatenate(data_list, start=3, stop=26)
assert np.allclose(merged_data.vis, right_answer.vis)
assert len(merged_data.index_map["time"]) == len(
right_answer.index_map["time"])
def test_works(data_list):
right_answer = andata.AnData.from_acq_h5(acq_fname_list)
merged_data = andata.concatenate(data_list)
assert np.allclose(merged_data.vis, right_answer.vis)
assert len(merged_data.index_map["time"]) == len(
right_answer.index_map["time"])