def test_dataset_1d_frequencies(nt_dataset_pinchoff, tmp_path):
ds = Dataset(1, db_name="temp.db", db_folder=str(tmp_path))
ds.compute_power_spectrum()
assert len(ds.power_spectrum) == 2
assert len(ds.power_spectrum) == len(ds.data)
with pytest.raises(KeyError):
ds.power_spectrum['freq_y']
ds_vx = ds.data['dc_current'][default_coord_names['voltage'][0]].values
ds_sig = ds.data['dc_current'].values
xv = np.unique(ds_vx)
signal = ds_sig.copy()
signal = sg.detrend(signal, axis=0)
frequencies_res = fp.fft(signal)
frequencies_res = np.abs(fp.fftshift(frequencies_res))**2
fx = fp.fftshift(fp.fftfreq(frequencies_res.shape[0], d=xv[1] - xv[0]))
coord_name = default_coord_names['frequency'][0]
ds_fx = ds.power_spectrum['dc_current'][coord_name].values
ds_freq = ds.power_spectrum['dc_current'].values
assert np.allclose(ds_fx, fx)
assert np.allclose(ds_freq, frequencies_res)
def test_dataset_2d_frequencies(nt_dataset_doubledot, tmp_path):
ds = Dataset(1, db_name="temp.db", db_folder=str(tmp_path))
ds.compute_power_spectrum()
assert len(ds.power_spectrum) == 2
ds_vx = ds.data['dc_current'][default_coord_names['voltage'][0]].values
ds_vy = ds.data['dc_current'][default_coord_names['voltage'][1]].values
ds_curr = ds.data['dc_current'].values.copy()
xv = np.unique(ds_vx)
yv = np.unique(ds_vy)
ds_curr = sg.detrend(ds_curr, axis=0)
ds_curr = sg.detrend(ds_curr, axis=1)
frequencies_res = fp.fft2(ds_curr)
frequencies_res = np.abs(fp.fftshift(frequencies_res))**2
fx_1d = fp.fftshift(fp.fftfreq(frequencies_res.shape[0], d=xv[1] - xv[0]))
fy_1d = fp.fftshift(fp.fftfreq(frequencies_res.shape[1], d=yv[1] - yv[0]))
# fx, fy = np.meshgrid(fx_1d, fy_1d, indexing="ij")
# frequencies_res = np.abs(frequencies_res)
coord_name = default_coord_names['frequency'][0]
ds_fx = ds.power_spectrum['dc_current'][coord_name].values
coord_name = default_coord_names['frequency'][1]
ds_fy = ds.power_spectrum['dc_current'][coord_name].values
ds_freq = ds.power_spectrum['dc_current'].values
assert np.allclose(ds_fx, fx_1d)
assert np.allclose(ds_fy, fy_1d)
assert np.allclose(ds_freq, frequencies_res)
def test_1D_prepare_filtered_data(nt_dataset_pinchoff, tmp_path):
pf = Dataset(1, "temp.db", db_folder=str(tmp_path))
pf.prepare_filtered_data()
assert len(pf.filtered_data) == len(pf.data)
assert pf.filtered_data.dc_current.shape == pf.data.dc_current.shape
rtol = 1e-05
assert not np.allclose(pf.filtered_data.dc_sensor.values,
pf.data.dc_sensor.values, rtol=rtol)
def test_dataset_normalisation(nt_dataset_pinchoff, tmp_path):
ds = Dataset(1, db_name="temp.db", db_folder=str(tmp_path))
v_x = np.linspace(-0.1, 0, 100)
sig = 0.6 * (1 + np.tanh(1000 * v_x + 50))
norm_sig = ds._normalize_data(sig, "dc_current")
manually_normalized = sig / 1.2
assert np.allclose(manually_normalized, norm_sig)
assert np.max(norm_sig) <= 1.0
assert np.min(norm_sig) >= 0.0
def test_dataset_2ddata_loading(nt_dataset_doubledot, tmp_path):
ds = Dataset(1, db_name="temp.db", db_folder=str(tmp_path))
assert ds.exp_id == 1
assert ds.dimensions['dc_current'] == 2
assert ds.dimensions['dc_sensor'] == 2
assert len(ds.raw_data) == 2
assert len(ds.data) == 2
ds_vx = ds.raw_data['current']['v_x'].values
ds_vy = ds.raw_data['current']['v_y'].values
ds_sig = ds.raw_data['current'].values
ds_sens = ds.raw_data['sensor'].values
xv, yv, ddot, sensor = generate_doubledot_data()
x = np.unique(xv)
y = np.unique(yv)
assert np.allclose(ds_vx, x)
assert np.allclose(ds_vy, y)
assert np.allclose(ds_sig, ddot.T)
assert np.allclose(ds_sens, sensor.T)
assert ds.get_plot_label('dc_current', 0) == "voltage x [V]"
assert ds.get_plot_label('dc_sensor', 0) == "voltage x [V]"
assert ds.get_plot_label('dc_current', 1) == "voltage y [V]"
assert ds.get_plot_label('dc_sensor', 1) == "voltage y [V]"
assert ds.get_plot_label('dc_current', 2) == "dc current [A]"
assert ds.get_plot_label('dc_sensor', 2) == "dc sensor [A]"
def test_dataset_property_getters(nt_dataset_pinchoff, tmp_path):
ds = Dataset(1, db_name="temp.db", db_folder=str(tmp_path))
assert ds._normalization_constants == ds.normalization_constants
assert ds.features == {
"dc_current": {
"amplitude": 0.6,
"slope": 1000,
"low_signal": 0,
"high_signal": 1,
"residuals": 0.5,
"offset": 50,
"transition_signal": 0.5,
"low_voltage": -0.06,
"high_voltage": -0.03,
"transition_voltage": -0.05,
},
"dc_sensor": {
"amplitude": 0.5,
"slope": 800,
"low_signal": 0,
"high_signal": 1,
"residuals": 0.5,
"offset": 50,
"transition_signal": 0.5,
"low_voltage": -0.06,
"high_voltage": -0.03,
"transition_voltage": -0.05,
}
}
assert ds._snapshot == ds.snapshot
qc_ds = load_by_id(1)
assert ds.snapshot == json.loads(qc_ds.get_metadata("snapshot"))
nt_metadata = json.loads(qc_ds.get_metadata(nt.meta_tag))
assert ds.nt_metadata == nt_metadata
with pytest.raises(AttributeError):
ds.snapshot = {}
with pytest.raises(AttributeError):
ds.normalization_constants = {}
with pytest.raises(AttributeError):
ds.features = {}
qc_ds = load_by_id(1)
assert ds.features == nt_metadata["features"]
def __init__(
self,
qc_run_id: int,
db_name: str,
db_folder: Optional[str] = None,
) -> None:
if db_folder is None:
db_folder = nt.config["db_folder"]
Dataset.__init__(
self,
qc_run_id,
db_name,
db_folder=db_folder,
)
self._features: Dict[str, Any] = {}
self._next_actions: Dict[str, List[str]] = {}
def test_dataset_defaults_for_missing_metadata(
nt_dataset_doubledot_partial_metadata, tmp_path):
ds = Dataset(1, db_name="temp.db", db_folder=str(tmp_path))
assert ds.normalization_constants["dc_current"] == [0, 1.4]
assert ds.normalization_constants["rf"] == [0, 1]
assert ds.normalization_constants["dc_sensor"] == [0, 1]
assert ds.readout_methods == {
"dc_current": "current",
"dc_sensor": "sensor"
}
assert ds.device_name == "noname_device"
assert ds.quality is None
assert ds.label == ["doubledot"]
def predict(
self,
dataid: int,
db_name: str,
db_folder: Optional[str] = None,
) -> np.ndarray:
""""""
if db_folder is None:
db_folder = nt.config["db_folder"]
DATA_TYPE_MAPPING = dict(nt.config["core"]["data_types"])
df = Dataset(dataid, db_name)
condensed_data_all = prep_data(df, self.category)
predictions = []
for condensed_data in condensed_data_all:
relevant_data = np.empty((1, 0))
for data_type in self.data_types:
to_append = condensed_data[DATA_TYPE_MAPPING[data_type]]
if data_type == "features":
to_append[to_append == nt.config["core"]
["fill_value"]] = np.nan
to_append = to_append[:,
np.isfinite(to_append).any(axis=0)]
try:
to_append = to_append[:, self.feature_indexes]
except IndexError as ie:
logger.warning(
"Some data in {} ".format(dataid) +
"does not have the " +
"feature requested. Make sure all data " +
"has been " + "fitted with appropriate fit " +
"classes.")
relevant_data = np.append(relevant_data, to_append, axis=1)
_, relevant_data = self.prep_data(test_data=relevant_data)
predictions.append(self.clf.predict(relevant_data))
return predictions
def summarize(self):
"""
generates list of files to send
fig saved with
self.dir = os.path.join(nt.config['db_folder'], 'tuning_results',
self.device_name)
filename = str(self.ds.guid)
"""
self.smalltalk += "\n Data IDs in this bundle: \n"
self._files = {}
inv_dict = {}
# sort IDs to make sure pdfs are printed in same oder as they were
# taken
for k, v in self.stage_summaries.items():
for qc_id in flatten_list(v):
inv_dict[qc_id] = k
sorted_ids = list(flatten_list(self.stage_summaries.values()))
sorted_ids.sort(key=int)
# for stage, value in self.stage_summaries.items():
for qc_run_id in sorted_ids:
# stage = inv_dict[qc_run_id]
# if stage[0:7] == 'failed_':
# stage = stage[7:]
# try:
# s = self.comments[qc_run_id]
# except KeyError:
# s = ''
# self.comments[qc_run_id] = 'Classified as poor result.\n' + s
ds = Dataset(qc_run_id, self.db_name)
device_name = ds.device_name
f_folder = os.path.join(self.db_folder, "tuning_results",
device_name)
# for qc_run_id in flatten_list(value):
self.smalltalk += str(qc_run_id) + ", "
# filename = stage + '_fit_ds'
# filename += str(qc_run_id) + '.png'
filename = os.path.join(f_folder, str(ds.ds.guid) + ".png")
self._files[str(qc_run_id)] = filename
def test_dataset_attributes_after_init(nt_dataset_doubledot, tmp_path):
"""
Ensure that all attributes are populated after initialisation.
Also tests property getters
"""
attributes = [
"db_name",
"db_folder",
"qc_run_id",
"snapshot",
"normalization_constants",
"device_name",
"readout_methods",
"quality",
"label",
"raw_data",
"data",
"power_spectrum",
"filtered_data",
"dimensions",
"exp_id",
]
ds = Dataset(1, db_name="temp.db", db_folder=str(tmp_path))
for attr in attributes:
assert hasattr(ds, attr)
getattr(ds, attr)
assert ds.device_name == "test_device"
assert ds._normalization_constants["dc_current"] == [0, 2]
assert ds._normalization_constants["rf"] == [0, 1]
assert ds._normalization_constants["dc_sensor"] == [-0.32, 3]
assert ds.readout_methods == {
"dc_current": "current",
"dc_sensor": "sensor"
}
assert ds.qc_run_id == 1
assert ds.db_name == "temp.db"
assert ds.quality == 1
assert set(ds.label) == {"doubledot"}
def test_dataset_1ddata_loading(nt_dataset_pinchoff, tmp_path):
ds = Dataset(1, db_name="temp.db", db_folder=str(tmp_path))
assert ds.exp_id == 1
assert ds.dimensions['dc_current'] == 1
assert ds.dimensions['dc_sensor'] == 1
assert len(ds.raw_data) == 2
assert len(ds.data) == 2
vx = np.linspace(-0.1, 0, 120)
ds_vx = ds.raw_data['current']['voltage'].values
assert np.allclose(ds_vx, vx)
ds_sig = ds.raw_data['current'].values
sig = 0.6 * (1 + np.tanh(1000 * vx + 50))
assert np.allclose(ds_sig, sig)
assert ds.get_plot_label('dc_current', 0) == "voltage x [V]"
assert ds.get_plot_label('dc_sensor', 0) == "voltage x [V]"
assert ds.get_plot_label('dc_current', 1) == "dc current [A]"
assert ds.get_plot_label('dc_sensor', 1) == "dc sensor [A]"
with pytest.raises(AssertionError):
ds.get_plot_label('dc_sensor', 2)
def export_data(
category: str,
db_names: List[str],
stages: List[str],
skip_ids: Optional[Dict[str, List[int]]] = None,
quality: Optional[int] = None,
filename: Optional[str] = None,
db_folder: Optional[str] = None,
) -> None:
""""""
assert isinstance(db_names, list)
assert isinstance(stages, list)
if db_folder is None:
db_folder = nt.config["db_folder"]
if category in ["pinchoff", "clmboscs"]:
dim = 1
elif category in [
"dotregime",
"singledot",
"doubledot",
"clmbdiam",
"outerbarriers",
]:
dim = 2
else:
logger.error("Trying to export data of" +
" a category: {}/".format(category) +
" Please update utils/export_data.py and tell me" +
" the dimensions of the data ")
shape = tuple(nt.config["core"]["standard_shapes"][str(dim)])
condensed_data_all = np.empty(
(len(nt.config["core"]["data_types"]), 0, np.prod(shape)))
relevant_ids: Dict[str, List[int]] = {}
for db_name in db_names:
relevant_ids[db_name] = []
nt.set_database(db_name, db_folder)
for stage in stages:
try:
if quality is None:
relevant_ids[db_name] += nt.get_dataIDs(
db_name, stage, db_folder=db_folder)
else:
relevant_ids[db_name] += nt.get_dataIDs(
db_name, stage, quality=quality, db_folder=db_folder)
except Exception as e:
logger.error("""Unable to load relevant ids
in {}""".format(db_name))
logger.error(e)
break
labels_exp = []
for db_name, dataids in relevant_ids.items():
nt.set_database(db_name, db_folder)
skip_us = []
if skip_ids is not None:
try:
skip_us = skip_ids[db_name]
except KeyError:
logger.warning("No data IDs to skip in {}.".format(db_name))
for d_id in dataids:
if d_id not in skip_us:
df = Dataset(d_id, db_name, db_folder=db_folder)
condensed_data = prep_data(df, category)
condensed_data_all = np.append(condensed_data_all,
condensed_data[0],
axis=1)
new_label = export_label(df.label, df.quality, category)
labels_exp.append(new_label)
n = list(condensed_data_all.shape)
n[-1] += 1
data_w_labels = np.zeros(n)
data_w_labels[:, :, -1] = labels_exp
data_w_labels[:, :, :-1] = condensed_data_all
if filename is None:
filename = "_".join(stages)
path = os.path.join(db_folder, filename)
np.save(path, data_w_labels)
def plot_dataset(
qc_run_id: int,
db_name: str,
save_figures: bool = True,
db_folder: Optional[str] = None,
plot_filtered_data: bool = False,
plot_params: Optional[plot_params_type] = None,
ax: Optional[matplotlib.axes.Axes] = None,
colorbar: Optional[matplotlib.colorbar.Colorbar] = None,
filename: Optional[str] = None,
file_location: Optional[str] = None,
) -> AxesTuple:
"""
If to be saved and no file location specified, the figure will be saved at
os.path.join(nt.config['db_folder'], 'tuning_results', dataset.device_name)
in both eps and png
"""
if plot_params is None:
plot_params = default_plot_params
matplotlib.rcParams.update(plot_params)
if db_folder is None:
_, db_folder = nt.get_database()
dataset = Dataset(qc_run_id, db_name, db_folder=db_folder)
if plot_filtered_data:
data = dataset.filtered_data
else:
data = dataset.data
if ax is None:
fig_size = copy.deepcopy(plot_params["figure.figsize"])
fig_size[1] *= len(dataset.data) * 0.8 # type: ignore
fig, ax = plt.subplots(
len(dataset.data),
1,
squeeze=False,
figsize=fig_size,
)
colorbars: List[matplotlib.colorbar.Colorbar] = []
fig_title = dataset.guid
for r_i, read_meth in enumerate(dataset.readout_methods):
c_name = default_coord_names['voltage'][0]
voltage_x = data[read_meth][c_name].values
signal = data[read_meth].values.T
if dataset.dimensions[read_meth] == 1:
colorbar = None
ax[r_i, 0].plot(
voltage_x,
signal,
zorder=6,
)
ax[r_i, 0].set_xlabel(dataset.get_plot_label(read_meth, 0))
ax[r_i, 0].set_ylabel(dataset.get_plot_label(read_meth, 1))
ax[r_i, 0].set_title(str(fig_title))
divider = make_axes_locatable(ax[r_i, 0])
cbar_ax = divider.append_axes("right", size="5%", pad=0.06)
cbar_ax.set_facecolor("none")
for caxis in ["top", "bottom", "left", "right"]:
cbar_ax.spines[caxis].set_linewidth(0)
cbar_ax.set_xticks([])
cbar_ax.set_yticks([])
colorbars.append(colorbars)
ax[r_i, 0].figure.tight_layout()
elif dataset.dimensions[read_meth] == 2:
c_name = default_coord_names['voltage'][1]
voltage_y = data[read_meth][c_name].values
colormesh = ax[r_i, 0].pcolormesh(
voltage_x,
voltage_y,
signal,
shading="auto",
)
if colorbar is not None:
colorbars.append(ax[r_i, 0].figure.colorbar(colormesh,
ax=ax[r_i, 0],
cax=colorbar.ax))
else:
# colorbar = fig.colorbar(colormesh, ax=ax[r_i, 0])
divider = make_axes_locatable(ax[r_i, 0])
cbar_ax = divider.append_axes("right", size="5%", pad=0.06)
colorbars.append(
fig.colorbar(
colormesh,
ax=ax[r_i, 0],
cax=cbar_ax,
))
colorbars[-1].set_label(
dataset.get_plot_label(read_meth, 2),
rotation=-270,
)
ax[r_i, 0].set_xlabel(dataset.get_plot_label(read_meth, 0))
ax[r_i, 0].set_ylabel(dataset.get_plot_label(read_meth, 1))
ax[r_i, 0].set_title(str(fig_title))
ax[r_i, 0].figure.tight_layout()
else:
raise NotImplementedError
if save_figures:
if file_location is None:
file_location = os.path.join(nt.config["db_folder"],
"tuning_results", dataset.device_name)
if not os.path.exists(file_location):
os.makedirs(file_location)
if filename is None:
filename = "dataset_" + str(dataset.guid)
else:
filename = os.path.splitext(filename)[0]
path = os.path.join(file_location, filename + ".png")
plt.savefig(path, format="png", dpi=600, bbox_inches="tight")
return ax, colorbars