def BoucWen(dir_placement=None, force_download=False, split_data=True):
#todo: dot p file integration as system for training data
#generate more data
# url = 'http://www.nonlinearbenchmark.org/FILES/BENCHMARKS/BOUCWEN/BoucWenFiles.zip'
url = 'https://data.4tu.nl/ndownloader/files/24703124'
download_size = 5284363
save_dir = cashed_download(url,
'BoucWen',
zip_name='BoucWenFiles.zip',
dir_placement=dir_placement,
download_size=download_size,
force_download=force_download)
save_dir = os.path.join(
save_dir,
'BoucWenFiles/Test signals/Validation signals') #matfiles location
datafiles = []
out = loadmat(os.path.join(save_dir, 'uval_multisine.mat'))
u_multisine = out['uval_multisine'][0]
out = loadmat(os.path.join(save_dir, 'yval_multisine.mat'))
y_multisine = out['yval_multisine'][0]
datafiles.append(System_data(u=u_multisine, y=y_multisine))
out = loadmat(os.path.join(save_dir, 'uval_sinesweep.mat'))
u_sinesweep = out['uval_sinesweep'][0]
out = loadmat(os.path.join(save_dir, 'yval_sinesweep.mat'))
y_sinesweep = out['yval_sinesweep'][0]
datafiles.append(System_data(u=u_sinesweep, y=y_sinesweep))
datafiles = System_data_list(datafiles)
if not split_data:
return datafiles
else:
return datafiles.train_test_split()
def EMPS(dir_placement=None,
vir_as_u=True,
force_download=False,
split_data=True):
'''The Electro-Mechanical Positioning System is a standard configuration of a drive system for prismatic joint of robots or machine tools. The main source of nonlinearity is caused by friction effects that are present in the setup. Due to the presence of a pure integrator in the system, the measurements are obtained in a closed-loop setting.
The provided data is described in this link. The provided Electro-Mechanical Positioning System datasets are available for download here. This zip-file contains the system description and available data sets .mat file format.
Please refer to the Electro-Mechanical Positioning System as:
A. Janot, M. Gautier and M. Brunot, Data Set and Reference Models of EMPS, 2019 Workshop on Nonlinear System Identification Benchmarks, Eindhoven, The Netherlands, April 10-12, 2019.
Special thanks to Alexandre Janot for making this dataset available.'''
#q_cur current measured position
#q_ref target/reference potion
#non-linear due to singed friction force Fc ~ sing(dq/dt)
#t time
#vir applied the vector of motor force expressed in the load side i.e. in N;
# url = 'http://www.nonlinearbenchmark.org/FILES/BENCHMARKS/EMPS/EMPS.zip'
url = 'https://drive.google.com/file/d/1zwoXYa9-3f8NQ0ohzmjpF7UxbNgRTHkS/view'
download_size = 1949929
save_dir = cashed_download(url,
'EMPS',
zip_name='EMPS.zip',
dir_placement=dir_placement,
download_size=download_size,
force_download=force_download)
matfile = loadmat(os.path.join(save_dir, 'DATA_EMPS.mat'))
q_cur, q_ref, t, vir = [
matfile[a][:, 0] for a in ['qm', 'qg', 't', 'vir']
] #qg is reference, either, q_ref is input or vir is input
out_data = System_data(u=vir, y=q_cur) if vir_as_u else System_data(
u=q_ref, y=q_cur)
return out_data.train_test_split() if split_data else out_data
def ETT_data_get(name,
dir_placement=None,
force_download=False,
split_data=True,
include_time_in_u=False,
full_return=False):
url = name
file_name = url.split('/')[-1]
download_size = None
save_dir = cashed_download(url,'beihang', dir_placement=dir_placement,\
download_size=download_size,force_download=force_download,zipped=False)
file_loc = os.path.join(save_dir, file_name)
ETT, target, loads, times, time = load_cor(file_loc)
if full_return:
return ETT, target, loads, times, time
u = loads
if include_time_in_u:
u = np.concatenate([u, times], axis=1)
y = target
sys_data = System_data(u=u, y=y, dt=15 / 60 / 24)
return sys_data.train_test_split(split_fraction=4 /
20) if split_data else sys_data
def CD_player_arm(
dir_placement=None,
force_download=False,
split_data=True,
data_set=0): #todo: check this function if column are correct
'''
This data set has two data sets which can be selected by setting the data_set argument to either 0 or 1
Contributed by:
Favoreel
KULeuven
Departement Electrotechniek ESAT/SISTA
Kardinaal Mercierlaan 94
B-3001 Leuven
Belgium
[email protected]
Description:
Data from the mechanical construction of a CD player arm.
The inputs are the forces of the mechanical actuators
while the outputs are related to the tracking accuracy of the arm.
The data was measured in closed loop, and then through a two-step
procedure converted to open loop equivalent data
The inputs are highly colored.
Sampling:
Number:
2048
Inputs:
u: forces of the mechanical actuators
Outputs:
y: tracking accuracy of the arm
References:
We are grateful to R. de Callafon of the
Mechanical Engineering Systems and Control group of Delft, who
provided us with these data.
- Van Den Hof P., Schrama R.J.P., An Indirect Method for Transfer
Function Estimation From Closed Loop Data. Automatica, Vol. 29,
no. 6, pp. 1523-1527, 1993.
Properties:
Columns:
Column 1: input u1
Column 2: input u2
Column 1: output y1
Column 2: output y2
Category:
mechanical systems
'''
url = 'ftp://ftp.esat.kuleuven.ac.be/pub/SISTA/data/mechanical/CD_player_arm.dat.gz'
data = daisydata_download(url,
dir_placement=dir_placement,
force_download=force_download)
data0 = System_data(u=data[:, 0], y=data[:, 2])
data1 = System_data(u=data[:, 1], y=data[:, 3])
data_sets = System_data_list([data0, data1])
return data_sets.train_test_split() if split_data else data_sets
def flexible_structure(dir_placement=None,
force_download=False,
split_data=True):
'''Contributed by:
Maher ABDELGHANI
IRISA-INRIA
Campus de Beaulieu
35042 Rennes cedex
FRANCE
[email protected]
Description:
Experiment on a Steel Subframe Flexible
structure performed at LMS-International,
Leuven-Belgium.
-Structure suspended with flexible rubber bands.
-2 shakers at 2 locations were used for force input signals.
- 28 accelerometers around the structure were used for measurements.
- The 30 channels were simulataneously measured using the LMS-CadaX
Data Acquisition Module.
Sampling:
1/1024 (s)
Number:
8523 samples/channel
Inputs:
2 inputs:
u1= White noise Force
u2=White noise force.
Outputs:
28 outputs:
Accelerations
References:
1. M.Abdelghani, M.Basseville, A.Benvensite,"In-Operation Damage
Monitoring and Diagnosis of Vibrating Structures, with Application to
Offshore Structures and Rotating Machinery", IMAC-XV Feb.3-6 1997,
Fl. USA.
2. M.Abdelghani, C.T.Chou, M. Verhaegen, "Using Subspace Methods for the
Identification and Modal Analysis of Structures", IMAC-XV,
Feb.3-6 1997, Fl.USA.
Properties:
Frequency Range: 10-512 Hz.
Columns:
colomn1= input1 (u1)
colomn2=input2 (u2)
colomns3--30: outputs1--28
Category:
Mechanical Structure
'''
url = 'ftp://ftp.esat.kuleuven.ac.be/pub/SISTA/data/mechanical/flexible_structure.dat.gz'
data = daisydata_download(url,
dir_placement=dir_placement,
force_download=force_download)
data = System_data(u=data[:, 0], y=data[:, 1])
return data.train_test_split() if split_data else data
def CED(dir_placement=None, force_download=False, split_data=True):
'''The coupled electric drives consists of two electric motors that drive a pulley using a flexible belt.
The pulley is held by a spring, resulting in a lightly damped dynamic mode. The electric drives can
be individually controlled allowing the tension and the speed of the belt to be simultaneously controlled.
The drive control is symmetric around zero, hence both clockwise and counter clockwise movement is possible.
The focus is only on the speed control system. The angular speed of the pulley is measured as an output with
a pulse counter and this sensor is insensitive to the sign of the velocity. The available data sets are short,
which constitute a challenge when performing identification.
The provided data is part of a technical note available online through this link.
The provided Coupled Electric Drives datasets are available for download here.
This zip-file contains the system description and available data sets, both in
the .csv and .mat file format.
Please refer to the Coupled Electric Drives dataset as:
T. Wigren and M. Schoukens, Coupled Electric Drives Data Set and Reference Models,
Technical Report, Department of Information Technology, Uppsala University,
Department of Information Technology, Uppsala University, 2017.
Previously published results on the Coupled Electric Drives benchmark are listed in
the history section of this webpage.
Special thanks to Torbjön Wigren for making this dataset available.
NOTE: We are re-evaluating the continuous-time models reported in the technical note.
For now, the discrete-time model reported in eq. (9) of the technical note can be used
in combination of PRBS dataset with amplitude 1.'''
#http://www.it.uu.se/research/publications/reports/2017-024/2017-024-nc.pdf
url = 'http://www.it.uu.se/research/publications/reports/2017-024/CoupledElectricDrivesDataSetAndReferenceModels.zip'
download_size = 278528
save_dir = cashed_download(url,
'CED',
dir_placement=dir_placement,
download_size=download_size,
force_download=force_download)
# there is something not right with this data set.
# datasets = []
# d = 'DATAPRBS.MAT'
# # for d in [,'DATAUNIF.MAT']:
# matfile = loadmat(os.path.join(save_dir,d))
# u1,u2,u3,z1,z2,z3 = [matfile[a][:,0] for a in ['u1','u2','u3','z1','z2','z3']]
# datasets.extend([System_data(u=u,y=y) for (u,y) in [(u1,z1),(u2,z2),(u3,z3)]])
d = 'DATAUNIF.MAT'
matfile = loadmat(os.path.join(save_dir, d))
u11, u12, z11, z12 = [
matfile[a][:, 0] for a in ['u11', 'u12', 'z11', 'z12']
]
datasets = System_data_list(
[System_data(u=u11, y=z11),
System_data(u=u12, y=z12)])
return datasets if not split_data else datasets.sdl
def steamgen(dir_placement=None, force_download=False, split_data=True):
'''Contributed by:
Jairo Espinosa
ESAT-SISTA KULEUVEN
Kardinaal Mercierlaan 94
B-3001 Heverlee Belgium
[email protected]
Description:
The data comes from a model of a Steam Generator at Abbott Power Plant in Champaign IL.
The model is described in the paper of Pellegrineti [1].
Sampling:
3 sec
Number:
9600
Inputs:
u1: Fuel scaled 0-1
u2: Air scaled 0-1
u3: Reference level inches
u4: Disturbance defined by the load level
Outputs:
y1: Drum pressure PSI
y2: Excess Oxygen in exhaust gases %
y3: Level of water in the drum
y4: Steam Flow Kg./s
References:
[1] G. Pellegrinetti and J. Benstman, Nonlinear Control Oriented Boiler Modeling -A Benchamrk Problem for Controller Design, IEEE Tran. Control Systems Tech. Vol.4No.1 Jan.1996
[2] J. Espinosa and J. Vandewalle Predictive Control Using Fuzzy Models Applied to a Steam Generating Unit, Submitted to FLINS 98 3rd. International Workshop on Fuzzy Logic Systems and Intelligent Technologies for Nuclear Science and Industry
Properties:
To make possible the open loop identification the wter level was
stabilized by appliying to the water flow input a feedforward action proportional to the steam flow
with value 0.0403 and a PI action with values Kp=0.258 Ti=1.1026e-4 the reference of this controller
is the input u3.
Columns:
Column 1: time-steps
Column 2: input fuel
Column 3: input air
Column 4: input level ref.
Column 5: input disturbance
Column 6: output drum pressure
Column 7: output excess oxygen
Column 8: output water level
Column 9: output steam flow
Category:
Process industry systems
Where:
ftp://ftp.esat.kuleuven.ac.be/pub/SISTA/espinosa/datasets/powplant.dat
'''
url = 'ftp://ftp.esat.kuleuven.ac.be/pub/SISTA/data/process_industry/steamgen.dat.gz'
data = daisydata_download(url,
dir_placement=dir_placement,
force_download=force_download)
data = System_data(u=data[:, 1:5], y=data[:, 5:])
return data.train_test_split() if split_data else data
def internet_traffic(dir_placement=None,
force_download=False,
split_data=True):
'''Contributed by:
Katrien De C**k
K.U.Leuven, ESAT-SISTA
Kardinaal Mercierlaan 94
3001 Heverlee
[email protected]
Description: one hour of internet traffic between the Lawrence Berkeley Laboratory and the rest of the world
Sampling:
Number: 99999
Inputs:
Output: number of packets per time unit
References:
Katrien De C**k and Bart De Moor, Identification of the first order parameters of a circulant modulated Poisson process. Accepted for publication in the proceedings of the International Conference on Telecommunication (ICT '98)
V. Paxson and S. Floyd, Wide-area traffic: The failure of Poisson modeling, IEEE/ACM Transactions on Networking, 1995
Properties:
Columns:
Column 1: time-steps
Column 2: output y
Category: Time series
'''
url = 'ftp://ftp.esat.kuleuven.ac.be/pub/SISTA/data/timeseries/internet_traffic.dat.gz'
dir_name = 'DaISy_data'
save_dir = cashed_download(url,
dir_name,
dir_placement=dir_placement,
download_size=None,
force_download=force_download)
file = os.path.join(save_dir, url.split('/')[-1][:-3])
with open(file) as f:
splitted = '\n'.join(f.read().split('\n')[:-5]) #weird shit
with tempfile.TemporaryFile() as fp:
fp.write(bytes(splitted, 'utf-8'))
fp.seek(0)
data = np.loadtxt(fp)
data = System_data(u=None, y=data[:, 1])
return data.train_test_split() if split_data else data
def Cascaded_Tanks(dir_placement=None,force_download=False,split_data=True):
# url = 'http://www.nonlinearbenchmark.org/FILES/BENCHMARKS/CASCADEDTANKS/CascadedTanksFiles.zip'
url = 'https://data.4tu.nl/ndownloader/files/24686327'
download_size = 7520592
save_dir = cashed_download(url,'Cascaded_Tanks',zip_name='CascadedTanksFiles.zip',dir_placement=dir_placement,download_size=download_size,force_download=force_download)
save_dir = os.path.join(save_dir,'CascadedTanksFiles')
out = loadmat(os.path.join(save_dir,'dataBenchmark.mat'))
uEst, uVal, yEst, yVal, Ts = out['uEst'][:,0],out['uVal'][:,0],out['yEst'][:,0],out['yVal'][:,0],out['Ts'][0,0]
datasets = [System_data(u=uEst,y=yEst),System_data(u=uVal,y=yVal)]
datasets = System_data_list(datasets)
return datasets if not split_data else (datasets.sdl[0], datasets.sdl[1])
def ParWHF(dir_placement=None, force_download=False, split_data=True):
'''Parallel Wienner-Hammerstein'''
# url = 'http://www.nonlinearbenchmark.org/FILES/BENCHMARKS/PARWH/ParWHFiles.zip'
url = 'https://data.4tu.nl/ndownloader/files/24666227'
download_size = 58203304
save_dir = cashed_download(url,
'ParWHF',
zip_name='ParWHFiles.zip',
dir_placement=dir_placement,
download_size=download_size,
force_download=force_download)
save_dir = os.path.join(save_dir, 'ParWHFiles') #matfiles location
out = loadmat(os.path.join(save_dir, 'ParWHData.mat'))
# print(out.keys())
# print(out['amp'][0]) #5 values
# print(out['fs'][0,0])
# print(out['lines'][0]) #range 2:4096
# print('uEst',out['uEst'].shape) #(16384, 2, 20, 5), (N samplees, P periods, M Phase changes, nAmp changes)
# print('uVal',out['uVal'].shape) #(16384, 2, 1, 5)
# print('uValArr',out['uValArr'].shape) #(16384, 2)
# print('yEst',out['yEst'].shape) #(16384, 2, 20, 5)
# print('yVal',out['yVal'].shape) #(16384, 2, 1, 5)
# print('yValArr',out['yValArr'].shape) #(16384, 2)
datafiles = []
datafiles_test = []
#todo split train, validation and test
uEst = out['uEst'].reshape((16384, -1))
yEst = out['yEst'].reshape((16384, -1))
datafiles.extend(
[System_data(u=ui, y=yi) for ui, yi in zip(uEst.T, yEst.T)])
uVal = out['uVal'].reshape((16384, -1))
yVal = out['yVal'].reshape((16384, -1))
data = [System_data(u=ui, y=yi) for ui, yi in zip(uVal.T, yVal.T)]
datafiles_test.extend(data) if split_data else datafiles.extend(data)
uValArr = out['uValArr'].reshape((16384, -1))
yValArr = out['yValArr'].reshape((16384, -1))
data = [System_data(u=ui, y=yi) for ui, yi in zip(uValArr.T, yValArr.T)]
datafiles_test.extend(data) if split_data else datafiles.extend(data)
datafiles = System_data_list(datafiles)
if split_data:
datafiles_test = System_data_list(datafiles_test)
return (datafiles, datafiles_test)
else:
return datafiles
def powerplant(dir_placement=None, force_download=False, split_data=True):
'''This file describes the data in the powerplant.dat file.
1. Contributed by:
Peter Van Overschee
K.U.Leuven - ESAT - SISTA
K. Mercierlaan 94
3001 Heverlee
[email protected]
2. Process/Description:
data of a power plant (Pont-sur-Sambre (France)) of 120 MW
3. Sampling time
1228.8 sec
4. Number of samples:
200 samples
5. Inputs:
1. gas flow
2. turbine valves opening
3. super heater spray flow
4. gas dampers
5. air flow
6. Outputs:
1. steam pressure
2. main stem temperature
3. reheat steam temperature
7. References:
a. R.P. Guidorzi, P. Rossi, Identification of a power plant from normal
operating records. Automatic control theory and applications (Canada,
Vol 2, pp 63-67, sept 1974.
b. Moonen M., De Moor B., Vandenberghe L., Vandewalle J., On- and
off-line identification of linear state-space models, International
Journal of Control, Vol. 49, Jan. 1989, pp.219-232
8. Known properties/peculiarities
9. Some MATLAB-code to retrieve the data
!gunzip powerplant.dat.Z
load powerplant.dat
U=powerplant(:,1:5);
Y=powerplant(:,6:8);
Yr=powerplant(:,9:11);
'''
url = 'ftp://ftp.esat.kuleuven.ac.be/pub/SISTA/data/process_industry/powerplant.dat.gz'
powerplant = daisydata_download(url,
dir_placement=dir_placement,
force_download=force_download)
U = powerplant[:, 0:5]
Y = powerplant[:, 5:8]
Yr = powerplant[:, 8:11]
data = System_data(u=U, y=Y)
return data.train_test_split() if split_data else data
def evaporator(dir_placement=None, force_download=False, split_data=True):
'''Contributed by:
Favoreel
KULeuven
Departement Electrotechniek ESAT/SISTA
Kardinaal Mercierlaan 94
B-3001 Leuven
Belgium
[email protected]
Description:
A four-stage evaporator to reduce the water content of a product,
for example milk. The 3 inputs are feed flow, vapor flow to the
first evaporator stage and cooling water flow. The three outputs
are the dry matter content, the flow and the temperature of the
outcoming product.
Sampling:
Number:
6305
Inputs:
u1: feed flow to the first evaporator stage
u2: vapor flow to the first evaporator stage
u3: cooling water flow
Outputs:
y1: dry matter content
y2: flow of the outcoming product
y3: temperature of the outcoming product
References:
- Zhu Y., Van Overschee P., De Moor B., Ljung L., Comparison of
three classes of identification methods. Proc. of SYSID '94,
Vol. 1, 4-6 July, Copenhagen, Denmark, pp.~175-180, 1994.
Properties:
Columns:
Column 1: input u1
Column 2: input u2
Column 3: input u3
Column 4: output y1
Column 5: output y2
Column 6: output y3
Category:
Thermic systems
Where:
'''
url = 'ftp://ftp.esat.kuleuven.ac.be/pub/SISTA/data/process_industry/evaporator.dat.gz'
data = daisydata_download(url,
dir_placement=dir_placement,
force_download=force_download)
data = System_data(u=data[:, 1:4], y=data[:, 4:7])
return data.train_test_split() if split_data else data
def pHdata(dir_placement=None, force_download=False, split_data=True):
'''Contributed by:
Jairo Espinosa
K.U.Leuven ESAT-SISTA
K.Mercierlaan 94
B3001 Heverlee
[email protected]
Description:
Simulation data of a pH neutralization process in a constant volume
stirring tank.
Volume of the tank 1100 liters
Concentration of the acid solution (HAC) 0.0032 Mol/l
Concentration of the base solution (NaOH) 0,05 Mol/l
Sampling:
10 sec
Number:
2001
Inputs:
u1: Acid solution flow in liters
u2: Base solution flow in liters
Outputs:
y: pH of the solution in the tank
References:
T.J. Mc Avoy, E.Hsu and S.Lowenthal, Dynamics of pH in controlled
stirred tank reactor, Ind.Eng.Chem.Process Des.Develop.11(1972)
71-78
Properties:
Highly non-linear system.
Columns:
Column 1: time-steps
Column 2: input u1
Column 3: input u2
Column 4: output y
Category:
Process industry systems
'''
url = 'ftp://ftp.esat.kuleuven.ac.be/pub/SISTA/data/process_industry/pHdata.dat.gz'
data = daisydata_download(url,
dir_placement=dir_placement,
force_download=force_download)
data = System_data(u=data[:, 1:3], y=data[:, 3])
return data.train_test_split() if split_data else data
def F16(dir_placement=None,yn=0,force_download=False,split_data=True):
'''The F-16 Ground Vibration Test benchmark features a high order system with clearance and friction nonlinearities at the mounting interface of the payloads.
The experimental data made available to the Workshop participants were acquired on a full-scale F-16 aircraft on the occasion of the Siemens LMS Ground Vibration Testing Master Class, held in September 2014 at the Saffraanberg military basis, Sint-Truiden, Belgium.
During the test campaign, two dummy payloads were mounted at the wing tips to simulate the mass and inertia properties of real devices typically equipping an F-16 in flight. The aircraft structure was instrumented with accelerometers. One shaker was attached underneath the right wing to apply input signals. The dominant source of nonlinearity in the structural dynamics was expected to originate from the mounting interfaces of the two payloads. These interfaces consist of T-shaped connecting elements on the payload side, slid through a rail attached to the wing side. A preliminary investigation showed that the back connection of the right-wing-to-payload interface was the predominant source of nonlinear distortions in the aircraft dynamics, and is therefore the focus of this benchmark study.
A detailed formulation of the identification problem can be found here. All the provided files and information on the F-16 aircraft benchmark system are available for download here. This zip-file contains a detailed system description, the estimation and test data sets, and some pictures of the setup. The data is available in the .csv and .mat file format.
Please refer to the F16 benchmark as:
J.P. Noël and M. Schoukens, F-16 aircraft benchmark based on ground vibration test data, 2017 Workshop on Nonlinear System Identification Benchmarks, pp. 19-23, Brussels, Belgium, April 24-26, 2017.
Previously published results on the F-16 Ground Vibration Test benchmark are listed in the history section of this webpage.
Special thanks to Bart Peeters (Siemens Industry Software) for his help in creating this benchmark.'''
#todo this is still broken for some mat files
# assert False, 'this is still broken for some files where y has many more dimensions than expected'
# url = 'http://www.nonlinearbenchmark.org/FILES/BENCHMARKS/F16/F16GVT_Files.zip'
url = 'https://data.4tu.nl/ndownloader/files/24675560'
download_size=148455295
save_dir = cashed_download(url,'F16',zip_name='F16GVT_Files.zip',dir_placement=dir_placement,download_size=download_size,force_download=force_download)
save_dir = os.path.join(save_dir,'F16GVT_Files/BenchmarkData') #matfiles location
matfiles = [os.path.join(save_dir,a).replace('\\','/') for a in os.listdir(save_dir) if a.split('.')[-1]=='mat']
datasets = []
for file in sorted(matfiles):
out = loadmat(file)
Force, Voltage, (y1,y2,y3),Fs = out['Force'][0], out['Voltage'][0], out['Acceleration'], out['Fs'][0,0]
#u = Force
#y = one of the ys, multi objective regression?
name = file.split('/')[-1]
if 'SpecialOddMSine' not in name:
datasets.append(System_data(u=Force,y=[y1,y2,y3][yn]))
datasets = System_data_list(datasets)
return datasets if not split_data else datasets.train_test_split()
def sys_data_sampler(self, sys_data, Ndots_per_image):
u, images = sys_data.u, sys_data.y
#images has shape (Ns, C, H, W) for (Ns, H, W)
if len(images.shape)==4:
Ns, C, W, H = images.shape
elif len(images.shape)==3:
Ns, W, H = images.shape
C = None
else:
assert False, 'check images.shape'
sampleselector = torch.broadcast_to(torch.arange(Ns)[:,None],(Ns,Ndots_per_image))
h = np.random.randint(low=0, high=H, size=(Ns,Ndots_per_image))
w = np.random.randint(low=0, high=W, size=(Ns,Ndots_per_image))
images_shots = images[sampleselector,:,h,w] if C!=None else images[sampleselector,h,w] #what shape does this have? I hope it has (Ns, Nshots, C)
sys_data = System_data(u=u, y=images_shots, x=images)
sys_data.h, sys_data.w = h, w
return sys_data
def glassfurnace(dir_placement=None, force_download=False, split_data=True):
'''This file describes the data in the glassfurnace.dat file.
1. Contributed by:
Peter Van Overschee
K.U.Leuven - ESAT - SISTA
K. Mercierlaan 94
3001 Heverlee
[email protected]
2. Process/Description:
Data of a glassfurnace (Philips)
3. Sampling time
4. Number of samples:
1247 samples
5. Inputs:
a. heating input
b. cooling input
c. heating input
6. Outputs:
a. 6 outputs from temperature sensors in a cross section of the
furnace
7. References:
a. Van Overschee P., De Moor B., N4SID : Subspace Algorithms for
the Identification of Combined Deterministic-Stochastic Systems,
Automatica, Special Issue on Statistical Signal Processing and Control,
Vol. 30, No. 1, 1994, pp. 75-93
b. Van Overschee P., "Subspace identification : Theory,
Implementation, Application" , Ph.D. Thesis, K.U.Leuven, February 1995.
8. Known properties/peculiarities
9. Some MATLAB-code to retrieve the data
!gunzip glassfurnace.dat.Z
load glassfurnace.dat
T=glassfurnace(:,1);
U=glassfurnace(:,2:4);
Y=glassfurnace(:,5:10);
'''
url = 'ftp://ftp.esat.kuleuven.ac.be/pub/SISTA/data/process_industry/glassfurnace.dat.gz'
glassfurnace = daisydata_download(url,
dir_placement=dir_placement,
force_download=force_download)
U = glassfurnace[:, 1:4]
Y = glassfurnace[:, 4:10]
data = System_data(u=U, y=Y)
return data.train_test_split() if split_data else data
def robot_arm(dir_placement=None, force_download=False, split_data=True):
'''Contributed by:
Favoreel
KULeuven
Departement Electrotechniek ESAT/SISTA
Kardinaal Mercierlaan 94
B-3001 Leuven
Belgium
[email protected]
Description:
Data from a flexible robot arm. The arm is installed on an electrical
motor. We have modeled the transfer function from the measured reaction
torque of the structure on the ground to the acceleration of the
flexible arm. The applied input is a periodic sine sweep.
Sampling:
Number:
1024
Inputs:
u: reaction torque of the structure
Outputs:
y: accelaration of the flexible arm
References:
We are grateful to Hendrik Van Brussel and Jan Swevers of the laboratory
of Production Manufacturing and Automation of the Katholieke
Universiteit Leuven, who provided us with these data, which were
obtained in the framework of the Belgian Programme on
Interuniversity Attraction Poles (IUAP-nr.50) initiated by the
Belgian State - Prime Minister's Office - Science Policy
Programming.
Properties:
Columns:
Column 1: input u
Column 2: output y
Category:
mechanical systems
Where:
'''
url = 'ftp://ftp.esat.kuleuven.ac.be/pub/SISTA/data/mechanical/robot_arm.dat.gz'
data = daisydata_download(url,
dir_placement=dir_placement,
force_download=force_download)
data = System_data(u=data[:, 0], y=data[:, 1])
return data.train_test_split() if split_data else data
def dryer2(dir_placement=None, force_download=False, split_data=True):
'''
This file describes the data in the dryer.dat file.
1. Contributed by:
Jan Maciejowski
Cambridge University, Engineering Department
Trumpington Street, Cambridge
CB2 1PZ, England.
[email protected]
2. Process/Description:
Data from an industrial dryer (by Cambridge Control Ltd)
3. Sampling time:
10 sec
4. Number of samples:
867 samples
5. Inputs:
a. fuel flow rate
b. hot gas exhaust fan speed
c. rate of flow of raw material
6. Outputs:
a. dry bulb temperature
b. wet bulb temperature
c. moisture content of raw material
7. References:
a. Maciejowski J.M., Parameter estimation of multivariable
systems using balanced realizations, in:
Bittanti,S. (ed), Identification,
Adaptation, and Learning, Springer (NATO ASI Series), 1996.
b. Chou C.T., Maciejowski J.M., System Identification Using
Balanced Parametrizations, IEEE Transactions on Automatic Control,
vol. 42, no. 7, July 1997, pp. 956-974.
8. Known properties/peculiarities:
9. Some MATLAB-code to retrieve the data
!gunzip dryer.dat.Z
load dryer.dat
U=dryer(:,2:4);
Y=dryer(:,5:7);
'''
url = 'ftp://ftp.esat.kuleuven.ac.be/pub/SISTA/data/process_industry/dryer2.dat.gz'
data = daisydata_download(url,
dir_placement=dir_placement,
force_download=force_download)
data = System_data(u=data[:, 1:4], y=data[:, 4:7])
return data.train_test_split() if split_data else data
def cstr(dir_placement=None, force_download=False, split_data=True):
'''Contributed by:
Jairo ESPINOSA
ESAT-SISTA KULEUVEN
Kardinaal Mercierlaan 94
B-3001 Heverlee Belgium
[email protected]
Description:
The Process is a model of a Continuous
Stirring Tank Reactor, where the reaction
is exothermic and the concentration is
controlled by regulating the coolant
flow.
Sampling:
0.1 min
Number:
7500
Inputs:
q: Coolant Flow l/min
Outputs:
Ca: Concentration mol/l
T: Temperature Kelvin degrees
References:
J.D. Morningred, B.E.Paden, D.E. Seborg and D.A. Mellichamp "An adaptive nonlinear predictive controller" in. Proc. of the A.C.C. vol.2 1990 pp.1614-1619
G.Lightbody and G.W.Irwin. Nonlinear Control Structures Based on Embedded Neural System Models, IEEE Tran. on Neural Networks Vol.8 No.3 pp.553-567
J.Espinosa and J. Vandewalle, Predictive Control Using Fuzzy Models, Submitted to the 3rd. On-Line World Conference on Soft Computing in Engineering Design and Manufacturing.
Properties:
Columns:
Column 1: time-steps
Column 2: input q
Column 3: output Ca
Column 4: output T
Category:
Process Industry Systems
Where:
ftp://ftp.esat.kuleuven.ac.be/pub/SISTA/espinosa/datasets/cstr.dat
'''
url = 'ftp://ftp.esat.kuleuven.ac.be/pub/SISTA/data/process_industry/cstr.dat.gz'
data = daisydata_download(url,
dir_placement=dir_placement,
force_download=force_download)
data = System_data(u=data[:, 1], y=data[:, 2:4])
return data.train_test_split() if split_data else data
def thermic_res_wall(dir_placement=None,
force_download=False,
split_data=True):
'''Contributed by:
Favoreel
KULeuven
Departement Electrotechniek ESAT/SISTA
Kardinaal Mercierlaan 94
B-3001 Leuven
Belgium
[email protected]
Description:
Heat flow density through a two layer wall (brick and insulation
layer). The inputs are the internal and external temperature of
the wall. The output is the heat flow density through the wall.
Sampling:
Number:
1680
Inputs:
u1: internal wall temperature
u2: external wall temperature
Outputs:
y: heat flow density through the wall
References:
- System Identification Competition, Benchmark tests for estimation
methods of thermal characteristics of buildings and building
components. Organization: J. Bloem, Joint Research Centre,
Ispra, Italy, 1994.
Properties:
Columns:
Column 1: input u1
Column 2: input u2
Column 3: output y
Category:
thermic systems
Where:
'''
url = 'ftp://ftp.esat.kuleuven.ac.be/pub/SISTA/data/thermic/thermic_res_wall.dat.gz'
data = daisydata_download(url,
dir_placement=dir_placement,
force_download=force_download)
data = System_data(u=data[:, 0:2], y=data[:, 2])
return data.train_test_split() if split_data else data
def WienerHammerBenchMark(dir_placement=None,force_download=False, split_data=True):
url = 'http://www.ee.kth.se/~hjalmars/ifac_tc11_benchmarks/2009_wienerhammerstein/WienerHammerBenchMark.mat'
download_size=1707601
save_dir = cashed_download(url,'WienerHammerBenchMark',dir_placement=dir_placement,download_size=download_size,force_download=force_download,zipped=False)
out = loadmat(os.path.join(save_dir,'WienerHammerBenchMark.mat'))
u,y,fs = out['uBenchMark'][:,0],out['yBenchMark'][:,0],out['fs'][0,0]
out = System_data(u=u[5200:184000],y=y[5200:184000]) #fine only were u is active
# out = System_data(u=u,y=y)
return (out[:100000],out[100000:]) if split_data else out
def foetal_ecg(dir_placement=None, force_download=False, split_data=True):
'''Contributed by:
Lieven De Lathauwer
[email protected]
Description:
cutaneous potential recordings of a pregnant woman (8 channels)
Sampling:
10 sec
Number:
2500 x 8
Inputs:
Outputs:
1-5: abdominal
6,7,8: thoracic
References:
Dirk Callaerts,
"Signal Separation Methods based on Singular Value Decomposition
and their Application to the Real-Time Extraction of the
Fetal Electrocardiogram from Cutaneous Recordings",
Ph.D. Thesis, K.U.Leuven - E.E. Dept., Dec. 1989.
L. De Lathauwer, B. De Moor, J. Vandewalle, ``Fetal
Electrocardiogram Extraction by Blind Source Subspace Separation'',
IEEE Trans. Biomedical Engineering, Vol. 47, No. 5, May 2000,
Special Topic Section on Advances in Statistical Signal Processing
for Biomedicine, pp. 567-572.
Properties:
Columns:
Column 1: time-steps
Column 2-9: observations
Category:
4
'''
url = 'ftp://ftp.esat.kuleuven.ac.be/pub/SISTA/data/biomedical/foetal_ecg.dat.gz'
data = daisydata_download(url,
dir_placement=dir_placement,
force_download=force_download)
data = System_data(u=None, y=data[:, 1:9])
return data.train_test_split() if split_data else data
def flutter(dir_placement=None, force_download=False, split_data=True):
'''Contributed by:
Favoreel
KULeuven
Departement Electrotechniek ESAT/SISTA
Kardinaal Mercierlaan 94
B-3001 Leuven
Belgium
[email protected]
Description:
Wing flutter data. Due to industrial secrecy agreements we are
not allowed to reveal more details. Important to know is that
the input is highly colored.
Sampling:
Number:
1024
Inputs:
u:
Outputs:
y:
References:
Feron E., Brenner M., Paduano J. and Turevskiy A.. "Time-frequency
analysis for transfer function estimation and application to flutter
clearance", in AIAA J. on Guidance, Control & Dynamics, vol. 21,
no. 3, pp. 375-382, May-June, 1998.
Properties:
Columns:
Column 1: input u
Column 2: output y
Category:
mechanical systems
Where:
'''
url = 'ftp://ftp.esat.kuleuven.ac.be/pub/SISTA/data/mechanical/flutter.dat.gz'
data = daisydata_download(url,
dir_placement=dir_placement,
force_download=force_download)
data = System_data(u=data[:, 0], y=data[:, 1])
return data.train_test_split() if split_data else data
def dryer(dir_placement=None, force_download=False, split_data=True):
'''Contributed by:
Favoreel
KULeuven
Departement Electrotechniek ESAT/SISTA
Kardinaal Mercierlaan 94
B-3001 Leuven
Belgium
[email protected]
Description:
Laboratory setup acting like a hair dryer. Air is fanned through a tube
and heated at the inlet. The air temperature is measured by a
thermocouple at the output. The input is the voltage over the heating
device (a mesh of resistor wires).
Sampling:
Number:
1000
Inputs:
u: voltage of the heating device
Outputs:
y: output air temperature
References:
- Ljung L. System identification - Theory for the
User. Prentice Hall, Englewood Cliffs, NJ, 1987.
- Ljung. L. System Identification Toolbox. For Use
with Matlab. The Mathworks Inc., Mass., U.S.A., 1991.
Properties:
Columns:
Column 1: input u
Column 2: output y
Category:
mechanical systems
Where:
'''
url = 'ftp://ftp.esat.kuleuven.ac.be/pub/SISTA/data/mechanical/dryer.dat.gz'
data = daisydata_download(url,
dir_placement=dir_placement,
force_download=force_download)
data = System_data(u=data[:, 0], y=data[:, 1])
return data.train_test_split() if split_data else data
def Industrial_robot(dir_placement=None,force_download=False, split_data=True):
'''An identification benchmark dataset for a full robot movement with a KUKA KR300 R2500
ultra SE industrial robot is presented. It is a robot with a nominal payload capacity of
300 kg, a weight of 1120 kg, and a reach of 2500mm. It exhibits 12 states accounting for
position and velocity for each of the 6 joints. The robot encounters backlash in all
joints, pose-dependent inertia, pose-dependent gravitational loads, pose-dependent
hydraulic forces, pose- and velocity-dependent centripetal and Coriolis forces as well
as nonlinear friction, which is temperature-dependent and therefore potentially
time-varying. Prepared datasets for black-box identification of the forward or the
inverse robot dynamics are provided. Additional to the data for the black-box modeling,
we supply high-frequency raw data and videos of each experiment. A baseline and figures
of merit are defined to make results comparable across different identification methods.
A detailed formulation of the identification problem can be found here. All the provided
files and information on the industrial robot dataset can be found here.
https://kluedo.ub.uni-kl.de/frontdoor/index/index/docId/6731
https://fdm-fallback.uni-kl.de/TUK/FB/MV/WSKL/0001/
https://fdm-fallback.uni-kl.de/TUK/FB/MV/WSKL/0001/Robot_Identification_Benchmark_Without_Raw_Data.rar
Special thanks to Jonas Weigand and co-authors for creating and sharing this benchmark!'''
url = 'https://fdm-fallback.uni-kl.de/TUK/FB/MV/WSKL/0001/Robot_Identification_Benchmark_Without_Raw_Data.rar'
download_size=12717003
save_dir = cashed_download(url, 'Industrial_robot', zip_name='Robot_Identification_Benchmark_Without_Raw_Data.rar',\
dir_placement=dir_placement, download_size=download_size, force_download=force_download)
# save_dir = os.path.join(save_dir,'forward_identification_without_raw_data') #matfiles location
out = loadmat(os.path.join(save_dir,'forward_identification_without_raw_data.mat'))
K = 606
trains = [System_data(y=out['y_train'][:,n*K:(n+1)*K].T,u = out['u_train'][:,n*K:(n+1)*K].T) \
for n in range(out['y_train'].shape[1]//K)]
train = System_data_list(trains)
tests = [System_data(y=out['y_test'][:,n*K:(n+1)*K].T,u = out['u_test'][:,n*K:(n+1)*K].T) \
for n in range(out['y_test'].shape[1]//K)]
test = System_data_list(tests)
return train, test
def Silverbox(dir_placement=None,force_download=False, split_data=True):
'''The Silverbox system can be seen as an electronic implementation of the Duffing oscillator. It is build as a
2nd order linear time-invariant system with a 3rd degree polynomial static nonlinearity around it in feedback.
This type of dynamics are, for instance, often encountered in mechanical systems.
The provided data is part of a previously published ECC paper available online. A technical note describing the
Silverbox benchmark can be found here. All the provided data (.mat file format) on the Silverbox system is available
for download here. This .zip file contains the Silverbox dataset as specified in the benchmark document (V1 is the
input record, while V2 is the measured output), extended with .csv version of the same data and an extra data record
containing a Schroeder phase multisine measurement.
Please refer to the Silverbox benchmark as:
T. Wigren and J. Schoukens. Three free data sets for development and benchmarking in nonlinear system identification.
2013 European Control Conference (ECC), pp.2933-2938 July 17-19, 2013, Zurich, Switzerland.
Previously published results on the Silverbox benchmark are listed in the history section of this webpage.
Special thanks to Johan Schoukens for creating this benchmark, and to Torbjörn Wigren for hosting this benchmark.
'''
# url = 'http://www.nonlinearbenchmark.org/FILES/BENCHMARKS/SILVERBOX/SilverboxFiles.zip' #old
url = 'https://drive.google.com/file/d/17iS-6oBUUgrmiAcrZoG9S5sOaljZnDSy/view'
download_size=5793999
save_dir = cashed_download(url, 'Silverbox', zip_name='SilverboxFiles.zip',\
dir_placement=dir_placement, download_size=download_size, force_download=force_download)
save_dir = os.path.join(save_dir,'SilverboxFiles') #matfiles location
out = loadmat(os.path.join(save_dir,'Schroeder80mV.mat'))
u,y = out['V1'][0], out['V2'][0]
data1 = System_data(u=u,y=y)
out = loadmat(os.path.join(save_dir,'SNLS80mV.mat')) #train test
u,y = out['V1'][0], out['V2'][0]
data2 = System_data(u=u,y=y)
if split_data:
data_out = System_data(u=data2.u[40650:127400],y=data2.y[40650:127400])
return data_out.train_test_split()
return System_data_list([data1, data2])
def sun_spot_data(dir_placement=None, force_download=False, split_data=True):
url = 'http://www.sidc.be/silso/DATA/SN_y_tot_V2.0.txt'
download_size = None
save_dir = cashed_download(url,
'sun_spot_data',
dir_placement=dir_placement,
download_size=download_size,
force_download=force_download,
zipped=False)
with open(os.path.join(save_dir, 'SN_y_tot_V2.0.txt'), 'r') as f:
data = f.read()[:-2]
fixed_name = os.path.join(save_dir, 'SN_y_tot_V2.0_fix.txt')
with open(fixed_name, 'w') as f:
f.write(data)
data = np.loadtxt(fixed_name)
yEst = data[:, 1]
datasets = System_data(u=None, y=yEst)
return datasets.train_test_split(
split_fraction=0.4) if split_data else datasets #is already splitted
def ballbeam(dir_placement=None, force_download=False, split_data=True):
'''This file describes the data in the ballbeam.dat file.
1. Contributed by:
Peter Van Overschee
K.U.Leuven - ESAT - SISTA
K. Mercierlaan 94
3001 Heverlee
[email protected]
2. Process/Description:
Data of a the ball and beam practicum at ESAT-SISTA.
3. Sampling time
0.1 sec.
4. Number of samples:
1000 samples
5. Inputs:
a. angle of the beam
6. Outputs:
a. position of the ball
7. References:
a. Van Overschee P., "Subspace identification : Theory,
Implementation, Application" , Ph.D. Thesis, K.U.Leuven, February
1995, pp. 200-206
8. Known properties/peculiarities
9. Some MATLAB-code to retrieve the data
!gunzip ballbeam.dat.Z
load ballbeam.dat
U=ballbeam(:,1);
Y=ballbeam(:,2);
'''
url = 'ftp://ftp.esat.kuleuven.ac.be/pub/SISTA/data/mechanical/ballbeam.dat.gz'
data = daisydata_download(url,
dir_placement=dir_placement,
force_download=force_download)
data = System_data(u=data[:, 0], y=data[:, 1])
return data.train_test_split() if split_data else data
def WienerHammerstein_Process_Noise(dir_placement=None,
force_download=False,
split_data=True):
'''Warning this is a quite a bit of data'''
# url = 'http://www.nonlinearbenchmark.org/FILES/BENCHMARKS/WIENERHAMMERSTEINPROCESS/WienerHammersteinFiles.zip'
url = 'https://data.4tu.nl/ndownloader/files/24671987'
download_size = 423134764
save_dir = cashed_download(url,
'WienHammer',
zip_name='WienerHammersteinFiles.zip',
dir_placement=dir_placement,
download_size=download_size,
force_download=force_download)
save_dir = os.path.join(save_dir,
'WienerHammersteinFiles') #matfiles location
matfiles = [
os.path.join(save_dir, a).replace('\\', '/')
for a in os.listdir(save_dir) if a.split('.')[-1] == 'mat'
]
dataset = []
dataset_test = []
#file = 'WH_CombinedZeroMultisineSinesweep.mat' #'WH_Triangle_meas.mat'
for file in matfiles:
out = loadmat(os.path.join(save_dir, file))
r, u, y, fs = out['dataMeas'][0, 0]
fs = fs[0, 0]
data = [System_data(u=ui, y=yi) for ui, yi in zip(u.T, y.T)]
if split_data and 'Test' in file:
dataset_test.extend(data)
else:
dataset.extend(data)
dataset = System_data_list(dataset)
return (dataset, System_data_list(dataset_test)
) if split_data else dataset #brackets required if before ,
def apply_experiment(self, sys_data, save_state=False): #can put this in apply controller
'''Does an experiment with for given a system data (fixed u)
Parameters
----------
sys_data : System_data or System_data_list (or list or tuple)
The experiment which should be applied
Notes
-----
This will initialize the state using self.init_state if sys_data.y (and u)
is not None and skip the appropriate number of steps associated with it.
If either is missing than self.reset_state() is used to initialize the state.
Afterwards this state is advanced using sys_data.u and the output is saved at each step.
Lastly, the number of skipped/copied steps in init_state is saved as sys_data.cheat_n such
that it can be accounted for later.
'''
if isinstance(sys_data,(tuple,list,System_data_list)):
return System_data_list([self.apply_experiment(sd, save_state=save_state) for sd in sys_data])
#check if sys_data.x holds the
#u = (Ns)
#x = (Ns, C, H, W) or (Ns, H, W)
#y = (Ns, Np, C), (Ns, Np, C)
#h = (Ns, Np)
#w = (Ns, Np)
if not (hasattr(sys_data,'h') and hasattr(sys_data,'w')):
return super(SS_encoder_shotgun_MLP, self).apply_experiment(sys_data, save_state=save_state)
h, w = sys_data.h, sys_data.w
Y = []
sys_data.x, sys_data.y = sys_data.y, sys_data.x #move image to y
sys_data_norm = self.norm.transform(sys_data) #transform image if needed
sys_data.x, sys_data.y = sys_data.y, sys_data.x #move image back to x
sys_data_norm.x, sys_data_norm.y = sys_data_norm.y, sys_data_norm.x #move image back to x
sys_data_norm.h, sys_data_norm.w = h, w #set h and w on the normed version
U = sys_data_norm.u #get the input
Images = sys_data_norm.x #get the images
assert sys_data_norm.y is not None, 'not implemented' #if y is not None than init state
obs, k0 = self.init_state(sys_data_norm) #normed obs in the shape of y in the last step.
Y.extend(sys_data_norm.y[:k0]) #h(x_{k0-1})
if save_state:
X = [self.get_state()]*(k0+1)
for k in range(k0,len(U)):
Y.append(obs)
if k < len(U)-1: #skip last step
obs = self.step(U[k], h=h[k+1], w=w[k+1])
if save_state:
X.append(self.get_state())
#how the norm? the Y need to be transformed from (Ns, Np, C) with the norm
#norm.y0 has shape (C, W, H) or (C, 1, 1) or similar
Y = np.array(Y) #(Ns, Np, C)
# if self.norm.y0 is 1:
# return System_data(u=sys_data.u, y=Y, x=np.array(X) if save_state else None,normed=False,cheat_n=k0)
#has the shape of a constant or (1, 1) or (C, 1, 1) the possiblity of (C, H, W) I will exclude for now.
from copy import deepcopy
norm_sampler = deepcopy(self.norm)
if isinstance(self.norm.y0,(int,float)):
pass
elif self.norm.y0.shape==(1,1):
norm_sampler.y0 = norm_sampler.y0[0,0]
norm_sampler.ystd = norm_sampler.ystd[0,0] #ystd to a float
elif self.norm.y0.shape==(sys_data.x.shape[0],1,1):
norm_sampler.y0 = norm_sampler.y0[:,0,0]
norm_sampler.ystd = norm_sampler.ystd[:,0,0] #ystd to (C,) such that it can divide #(Ns, Np, C)
else:
raise NotImplementedError(f'norm of {self.norm} is not yet implemented for sampled simulations')
sys_data_sim = norm_sampler.inverse_transform(System_data(u=np.array(U),y=np.array(Y),x=np.array(X) if save_state else None,normed=True,cheat_n=k0))
sys_data_sim.h, sys_data_sim.w = sys_data.h, sys_data.w
return sys_data_sim
