def sys_data_sampler(self, sys_data, Ndots_per_image):
u, images = sys_data.u, sys_data.y
#images has shape (Ns, C, H, W) for (Ns, H, W)
if len(images.shape) == 4:
Ns, C, W, H = images.shape
elif len(images.shape) == 3:
Ns, W, H = images.shape
C = None
else:
assert False, 'check images.shape'
sampleselector = torch.broadcast_to(
torch.arange(Ns)[:, None], (Ns, Ndots_per_image))
h = np.random.randint(low=0, high=H, size=(Ns, Ndots_per_image))
w = np.random.randint(low=0, high=W, size=(Ns, Ndots_per_image))
images_shots = images[
sampleselector, :, h, w] if C != None else images[
sampleselector, h,
w] #what shape does this have? I hope it has (Ns, Nshots, C)
sys_data = System_data(u=u, y=images_shots, x=images)
sys_data.h, sys_data.w = h, w
return sys_data
def sun_spot_data(dir_placement=None, force_download=False, split_data=True):
url = 'http://www.sidc.be/silso/DATA/SN_y_tot_V2.0.txt'
download_size = None
save_dir = cashed_download(url,
'sun_spot_data',
dir_placement=dir_placement,
download_size=download_size,
force_download=force_download,
zipped=False)
with open(os.path.join(save_dir, 'SN_y_tot_V2.0.txt'), 'r') as f:
data = f.read()[:-2]
fixed_name = os.path.join(save_dir, 'SN_y_tot_V2.0_fix.txt')
with open(fixed_name, 'w') as f:
f.write(data)
data = np.loadtxt(fixed_name)
yEst = data[:, 1]
datasets = System_data(u=None, y=yEst)
return datasets.train_test_split(
split_fraction=0.4) if split_data else datasets #is already splitted
def WienerHammerstein_Process_Noise(dir_placement=None,
force_download=False,
split_data=True):
'''Warning this is a quite a bit of data'''
# url = 'http://www.nonlinearbenchmark.org/FILES/BENCHMARKS/WIENERHAMMERSTEINPROCESS/WienerHammersteinFiles.zip'
url = 'https://data.4tu.nl/ndownloader/files/24671987'
download_size = 423134764
save_dir = cashed_download(url,
'WienHammer',
zip_name='WienerHammersteinFiles.zip',
dir_placement=dir_placement,
download_size=download_size,
force_download=force_download)
save_dir = os.path.join(save_dir,
'WienerHammersteinFiles') #matfiles location
matfiles = [
os.path.join(save_dir, a).replace('\\', '/')
for a in os.listdir(save_dir) if a.split('.')[-1] == 'mat'
]
dataset = []
dataset_test = []
#file = 'WH_CombinedZeroMultisineSinesweep.mat' #'WH_Triangle_meas.mat'
for file in matfiles:
out = loadmat(os.path.join(save_dir, file))
r, u, y, fs = out['dataMeas'][0, 0]
fs = fs[0, 0]
data = [System_data(u=ui, y=yi) for ui, yi in zip(u.T, y.T)]
if split_data and 'Test' in file:
dataset_test.extend(data)
else:
dataset.extend(data)
dataset = System_data_list(dataset)
return (dataset, System_data_list(dataset_test)
) if split_data else dataset #brackets required if before ,
def ballbeam(dir_placement=None, force_download=False, split_data=True):
'''This file describes the data in the ballbeam.dat file.
1. Contributed by:
Peter Van Overschee
K.U.Leuven - ESAT - SISTA
K. Mercierlaan 94
3001 Heverlee
[email protected]
2. Process/Description:
Data of a the ball and beam practicum at ESAT-SISTA.
3. Sampling time
0.1 sec.
4. Number of samples:
1000 samples
5. Inputs:
a. angle of the beam
6. Outputs:
a. position of the ball
7. References:
a. Van Overschee P., "Subspace identification : Theory,
Implementation, Application" , Ph.D. Thesis, K.U.Leuven, February
1995, pp. 200-206
8. Known properties/peculiarities
9. Some MATLAB-code to retrieve the data
!gunzip ballbeam.dat.Z
load ballbeam.dat
U=ballbeam(:,1);
Y=ballbeam(:,2);
'''
url = 'ftp://ftp.esat.kuleuven.ac.be/pub/SISTA/data/mechanical/ballbeam.dat.gz'
data = daisydata_download(url,
dir_placement=dir_placement,
force_download=force_download)
data = System_data(u=data[:, 0], y=data[:, 1])
return data.train_test_split() if split_data else data
def apply_experiment(self, sys_data, save_state=False): #can put this in apply controller
'''Does an experiment with for given a system data (fixed u)
Parameters
----------
sys_data : System_data or System_data_list (or list or tuple)
The experiment which should be applied
Notes
-----
This will initialize the state using self.init_state if sys_data.y (and u)
is not None and skip the appropriate number of steps associated with it.
If either is missing than self.reset_state() is used to initialize the state.
Afterwards this state is advanced using sys_data.u and the output is saved at each step.
Lastly, the number of skipped/copied steps in init_state is saved as sys_data.cheat_n such
that it can be accounted for later.
'''
if isinstance(sys_data,(tuple,list,System_data_list)):
return System_data_list([self.apply_experiment(sd, save_state=save_state) for sd in sys_data])
#check if sys_data.x holds the
#u = (Ns)
#x = (Ns, C, H, W) or (Ns, H, W)
#y = (Ns, Np, C), (Ns, Np, C)
#h = (Ns, Np)
#w = (Ns, Np)
if not (hasattr(sys_data,'h') and hasattr(sys_data,'w')):
return super(SS_encoder_shotgun_MLP, self).apply_experiment(sys_data, save_state=save_state)
h, w = sys_data.h, sys_data.w
Y = []
sys_data.x, sys_data.y = sys_data.y, sys_data.x #move image to y
sys_data_norm = self.norm.transform(sys_data) #transform image if needed
sys_data.x, sys_data.y = sys_data.y, sys_data.x #move image back to x
sys_data_norm.x, sys_data_norm.y = sys_data_norm.y, sys_data_norm.x #move image back to x
sys_data_norm.h, sys_data_norm.w = h, w #set h and w on the normed version
U = sys_data_norm.u #get the input
Images = sys_data_norm.x #get the images
assert sys_data_norm.y is not None, 'not implemented' #if y is not None than init state
obs, k0 = self.init_state(sys_data_norm) #normed obs in the shape of y in the last step.
Y.extend(sys_data_norm.y[:k0]) #h(x_{k0-1})
if save_state:
X = [self.get_state()]*(k0+1)
for k in range(k0,len(U)):
Y.append(obs)
if k < len(U)-1: #skip last step
obs = self.step(U[k], h=h[k+1], w=w[k+1])
if save_state:
X.append(self.get_state())
#how the norm? the Y need to be transformed from (Ns, Np, C) with the norm
#norm.y0 has shape (C, W, H) or (C, 1, 1) or similar
Y = np.array(Y) #(Ns, Np, C)
# if self.norm.y0 is 1:
# return System_data(u=sys_data.u, y=Y, x=np.array(X) if save_state else None,normed=False,cheat_n=k0)
#has the shape of a constant or (1, 1) or (C, 1, 1) the possiblity of (C, H, W) I will exclude for now.
from copy import deepcopy
norm_sampler = deepcopy(self.norm)
if isinstance(self.norm.y0,(int,float)):
pass
elif self.norm.y0.shape==(1,1):
norm_sampler.y0 = norm_sampler.y0[0,0]
norm_sampler.ystd = norm_sampler.ystd[0,0] #ystd to a float
elif self.norm.y0.shape==(sys_data.x.shape[0],1,1):
norm_sampler.y0 = norm_sampler.y0[:,0,0]
norm_sampler.ystd = norm_sampler.ystd[:,0,0] #ystd to (C,) such that it can divide #(Ns, Np, C)
else:
raise NotImplementedError(f'norm of {self.norm} is not yet implemented for sampled simulations')
sys_data_sim = norm_sampler.inverse_transform(System_data(u=np.array(U),y=np.array(Y),x=np.array(X) if save_state else None,normed=True,cheat_n=k0))
sys_data_sim.h, sys_data_sim.w = sys_data.h, sys_data.w
return sys_data_sim
def winding(dir_placement=None, force_download=False, split_data=True):
'''Contributed by:
Favoreel
KULeuven
Departement Electrotechniek ESAT/SISTA
Kardinaal Mercierlaan 94
B-3001 Leuven
Belgium
[email protected]
Description:
The process is a test setup of an industrial winding process.
The main part of the plant is composed of a plastic web that
is unwinded from first reel (unwinding reel), goes over the
traction reel and is finally rewinded on the the rewinding reel.
Reel 1 and 3 are coupled with a DC-motor that is controlled with
input setpoint currents I1* and I3*. The angular speed of
each reel (S1, S2 and S3) and the tensions in the web between
reel 1 and 2 (T1) and between reel 2 and 3 (T3) are measured
by dynamo tachometers and tension meters.
We thank Th. Bastogne from the University of Nancy for
providing us with these data.
We are grateful to Thierry Bastogne of the Universite Henri Point Care, who
provided us with these data.
Sampling: 0.1 Sec
Number: 2500
Inputs: u1: The angular speed of reel 1 (S1)
u2: The angular speed of reel 2 (S2)
u3: The angular speed of reel 3 (S3)
u4: The setpoint current at motor 1 (I1*)
u5: The setpoint current at motor 2 (I3*)
Outputs: y1: Tension in the web between reel 1 and 2 (T1)
y2: Tension in the web between reel 2 and 3 (T3)
References:
- Bastogne T., Identification des systemes multivariables par
les methodes des sous-espaces. Application a un systeme
d'entrainement de bande. PhD thesis. These de doctorat
de l'Universite Henri Poincare, Nancy 1.
- Bastogne T., Noura H., Richard A., Hittinger J.M.,
Application of subspace methods to the identification of a
winding process. In: Proc. of the 4th European Control
Conference, Vol. 5, Brussels.
Properties:
Columns:
Column 1: input u1
Column 2: input u2
Column 3: input u3
Column 4: input u4
Column 5: input u5
Column 6: output y1
Column 7: output y2
Category:
Industrial test setup
'''
url = 'ftp://ftp.esat.kuleuven.ac.be/pub/SISTA/data/process_industry/winding.dat.gz'
data = daisydata_download(url,
dir_placement=dir_placement,
force_download=force_download)
data = System_data(u=data[:, 0:5], y=data[:, 5:7])
return data.train_test_split() if split_data else data
def exchanger(dir_placement=None, force_download=False, split_data=True):
'''This file describes the data in exchanger.dat
1. Contributed by:
Sergio Bittanti
Politecnico di Milano
Dipartimento di Elettronica e Informazione,
Politecnico di Milano,
Piazza Leonardo da Vinci 32, 20133 MILANO (Italy)
[email protected]
2. Process/Description:
The process is a liquid-satured steam heat exchanger, where water is
heated by pressurized saturated steam through a copper tube. The
output variable is the outlet liquid temperature. The input variables
are the liquid flow rate, the steam temperature, and the inlet liquid
temperature. In this experiment the steam temperature and the inlet
liquid temperature are kept constant to their nominal values.
3. Sampling time:
1 s
4. Number of samples:
4000
5. Inputs:
q: liquid flow rate
6. Outputs:
th: outlet liquid temperature
7. References:
S. Bittanti and L. Piroddi, "Nonlinear identification and control of a
heat exchanger: a neural network approach", Journal of the Franklin
Institute, 1996. L. Piroddi, Neural Networks for Nonlinear Predictive
Control. Ph.D. Thesis, Politecnico di Milano (in Italian), 1995.
8. Known properties/peculiarities:
The heat exchanger process is a significant benchmark for nonlinear
control design purposes, since it is characterized by a non minimum
phase behaviour. In the references cited above the control problem of
regulating the output temperature of the liquid-satured steam heat
exchanger by acting on the liquid flow rate is addressed, and both
direct and inverse identifications of the data are performed.
Columns:
Column 1: time-steps
Column 2: input q
Column 3: output th
'''
url = 'ftp://ftp.esat.kuleuven.ac.be/pub/SISTA/data/process_industry/exchanger.dat.gz'
data = daisydata_download(url,
dir_placement=dir_placement,
force_download=force_download)
data = System_data(u=data[:, 1], y=data[:, 2])
return data.train_test_split() if split_data else data
def destill(dir_placement=None,
force_download=False,
split_data=True,
noise=10):
'''This file describes the data in the destill.dat file.
1. Contributed by:
Peter Van Overschee
K.U.Leuven - ESAT - SISTA
K. Mercierlaan 94
3001 Heverlee
[email protected]
2. Process/Description:
Data of a simulation (not real !) related to the identification
of an ethane-ethylene destillationcolumn. The series consists of 4
series:
U_dest, Y_dest: without noise (original series)
U_dest_n10, Y_dest_n10: 10 percent additive white noise
U_dest_n20, Y_dest_n20: 20 percent additive white noise
U_dest_n30, Y_dest_n30: 30 percent additive white noise
3. Sampling time
15 min.
4. Number of samples:
90 samples
5. Inputs:
a. ratio between the reboiler duty and the feed flow
b. ratio between the reflux rate and the feed flow
c. ratio between the distillate and the feed flow
d. input ethane composition
e. top pressure
6. Outputs:
a. top ethane composition
b. bottom ethylene composition
c. top-bottom differential pressure.
7. References:
R.P. Guidorzi, M.P. Losito, T. Muratori, The range error test in the
structural identification of linear multivariable systems,
IEEE transactions on automatic control, Vol AC-27, pp 1044-1054, oct.
1982.
8. Known properties/peculiarities
9. Some MATLAB-code to retrieve the data
!gunzip destill.dat.Z
load destill.dat
U=destill(:,1:20);
Y=destill(:,21:32);
U_dest=U(:,1:5);
U_dest_n10=U(:,6:10);
U_dest_n20=U(:,11:15);
U_dest_n30=U(:,16:20);
Y_dest=Y(:,1:3);
Y_dest_n10=Y(:,4:6);
Y_dest_n20=Y(:,7:9);
Y_dest_n30=Y(:,10:12);
'''
url = 'ftp://ftp.esat.kuleuven.ac.be/pub/SISTA/data/process_industry/destill.dat.gz'
destill = daisydata_download(url,
dir_placement=dir_placement,
force_download=force_download)
U = destill[:, :20]
Y = destill[:, 20:]
U_dest = U[:, :5]
U_dest_n10 = U[:, 5:10]
U_dest_n20 = U[:, 10:15]
U_dest_n30 = U[:, 15:20]
Y_dest = Y[:, 0:3]
Y_dest_n10 = Y[:, 3:6]
Y_dest_n20 = Y[:, 6:9]
Y_dest_n30 = Y[:, 9:12]
if noise == 0:
data = System_data(u=U_dest, y=Y_dest)
elif noise == 10:
data = System_data(u=U_dest_n10, y=Y_dest_n10)
elif noise == 20:
data = System_data(u=U_dest_n20, y=Y_dest_n20)
elif noise == 30:
data = System_data(u=U_dest_n30, y=Y_dest_n30)
return data.train_test_split() if split_data else data
def erie(dir_placement=None, force_download=False, split_data=True, noise=10):
'''This file describes the data in the erie.dat file.
1. Contributed by:
Peter Van Overschee
K.U.Leuven - ESAT - SISTA
K. Mercierlaan 94
3001 Heverlee
[email protected]
2. Process/Description:
Data of a simulation (not real !) related to the related to the
identification of the western basin of Lake Erie. The series consists
of 4 series:
U_erie, Y_erie: without noise (original series)
U_erie_n10, Y_erie_n10: 10 percent additive white noise
U_erie_n20, Y_erie_n20: 20 percent additive white noise
U_erie_n30, Y_erie_n30: 30 percent additive white noise
3. Sampling time
1 month
4. Number of samples:
57 samples
5. Inputs:
a. water temperature
b. water conductivity
c. water alkalinity
d. NO3
e. total hardness
6. Outputs:
a. dissolved oxigen
b. algae
7. References:
R.P. Guidorzi, M.P. Losito, T. Muratori, On the last eigenvalue
test in the structural identification of linear multivariable
systems, Proceedings of the V European meeting on cybernetics and
systems research, Vienna, april 1980.
8. Known properties/peculiarities
The considered period runs from march 1968 till november 1972.
9. Some MATLAB-code to retrieve the data
!guzip erie.dat.Z
load erie.dat
U=erie(:,1:20);
Y=erie(:,21:28);
U_erie=U(:,1:5);
U_erie_n10=U(:,6:10);
U_erie_n20=U(:,11:15);
U_erie_n30=U(:,16:20);
Y_erie=Y(:,1:2);
Y_erie_n10=Y(:,3:4);
Y_erie_n20=Y(:,5:6);
Y_erie_n30=Y(:,7:8);
'''
url = 'ftp://ftp.esat.kuleuven.ac.be/pub/SISTA/data/environmental/erie.dat.gz'
erie = daisydata_download(url,
dir_placement=dir_placement,
force_download=force_download)
U = erie[:, :20]
Y = erie[:, 20:]
U_erie = U[:, :5]
U_erie_n10 = U[:, 5:10]
U_erie_n20 = U[:, 10:15]
U_erie_n30 = U[:, 15:20]
Y_erie = Y[:, 0:2] #two outputs
Y_erie_n10 = Y[:, 2:4]
Y_erie_n20 = Y[:, 4:6]
Y_erie_n30 = Y[:, 6:8]
if noise == 0:
data = System_data(u=U_erie, y=Y_erie)
elif noise == 10:
data = System_data(u=U_erie_n10, y=Y_erie_n10)
elif noise == 20:
data = System_data(u=U_erie_n20, y=Y_erie_n20)
elif noise == 30:
data = System_data(u=U_erie_n30, y=Y_erie_n30)
return data.train_test_split() if split_data else data