def convert(path):
contents = opusFC.listContents(path)
opusd = opusdictionary.OpusDict()
opath = path + "test.nxs"
name_dict = get_name_dict()
with h5py.File(opath, 'w') as f:
nxEntry = f.create_group("entry1")
nxEntry.attrs["NX_class"] = "NXentry"
for db in contents:
data = opusFC.getOpusData(path, db)
if isinstance(data, opusFC.ImageDataReturn):
add_image_data(nxEntry, data)
if isinstance(data, opusFC.MultiRegionDataReturn):
add_multi_data(nxEntry, data)
if isinstance(data, opusFC.ImageTRCDataReturn):
add_image_traces(nxEntry, data)
#nxColl = nxEntry.create_group("parameters")
#nxColl.attrs["NX_class"] = "NXcollection"
#for k,v in data.parameters.items():
# name = k
# if (k in opusd):
# name = opusd[k]
# nxColl.create_dataset(name,data=v)
images = opusFC.getVisImages(path)
count = 1
for im in images:
nxe = f.create_group("microscope" + str(count))
nxe.attrs["NX_class"] = "NXentry"
add_visible_image(nxe, im)
count = count + 1
print("Created file " + opath)
def parameters():
fl = file_list()
count = 0
for f in fl:
try:
dbs = opusFC.isOpusFile(f)
#control structure for the loop
if dbs == True:
conte = opusFC.listContents(f)
data = opusFC.getOpusData(f, conte[0])
count += 1
print(count)
#print conte, f
data = opusFC.getOpusData(f, conte[0])
print data.parameters, f
SNM.append(data.parameters['SNM'])
INS.append(data.parameters['INS'])
DAT.append(data.parameters['DAT'])
TIM.append(data.parameters['TIM'])
DUR.append(data.parameters['DUR'])
CNM.append(data.parameters['CNM'])
RES.append(data.parameters['RES'])
ZFF.append(data.parameters['ZFF'])
NPT.append(data.parameters['NPT'])
LWN.append(data.parameters['LWN'])
FXV.append(data.parameters['FXV'])
LXV.append(data.parameters['LXV'])
minY.append(data.minY)
maxY.append(data.maxY)
continue
else:
print('identified as non-Opus', f)
except ValueError:
print('Cant process a non-Opus file', f)
continue
LXV = []
FXV = []
minY = []
maxY = []
dbs=[]
# loop of all files
count = 0
for f in file_list:
try:
dbs = opusFC.listContents(f)
count += 1
print(count,f)
print(dbs)
for i in dbs:
data = opusFC.getOpusData(f, i)
print(data.parameters)
except ValueError:
print('Cant process a non-Opus file', f)
continue
def read(self):
import opusFC
if self.sheet:
db = self.sheet
else:
db = self.sheets[0]
db = tuple(db.split(" "))
dim = db[1]
try:
data = opusFC.getOpusData(self.filename, db)
except Exception:
raise IOError("Couldn't load spectrum from " + self.filename)
attrs, clses, metas = [], [], []
attrs = [
ContinuousVariable.make(repr(data.x[i]))
for i in range(data.x.shape[0])
]
y_data = None
meta_data = None
if type(data) == opusFC.MultiRegionDataReturn:
y_data = []
meta_data = []
metas.extend([
ContinuousVariable.make('map_x'),
ContinuousVariable.make('map_y'),
StringVariable.make('map_region'),
TimeVariable.make('start_time')
])
for region in data.regions:
y_data.append(region.spectra)
mapX = region.mapX
mapY = region.mapY
map_region = np.full_like(mapX, region.title, dtype=object)
start_time = region.start_time
meta_region = np.column_stack(
(mapX, mapY, map_region, start_time))
meta_data.append(meta_region.astype(object))
y_data = np.vstack(y_data)
meta_data = np.vstack(meta_data)
elif type(data) == opusFC.MultiRegionTRCDataReturn:
y_data = []
meta_data = []
metas.extend([
ContinuousVariable.make('map_x'),
ContinuousVariable.make('map_y'),
StringVariable.make('map_region')
])
attrs = [
ContinuousVariable.make(repr(data.labels[i]))
for i in range(len(data.labels))
]
for region in data.regions:
y_data.append(region.spectra)
mapX = region.mapX
mapY = region.mapY
map_region = np.full_like(mapX, region.title, dtype=object)
meta_region = np.column_stack((mapX, mapY, map_region))
meta_data.append(meta_region.astype(object))
y_data = np.vstack(y_data)
meta_data = np.vstack(meta_data)
elif type(data) == opusFC.ImageDataReturn:
metas.extend([
ContinuousVariable.make('map_x'),
ContinuousVariable.make('map_y')
])
data_3D = data.spectra
for i in np.ndindex(data_3D.shape[:1]):
map_y = np.full_like(data.mapX, data.mapY[i])
coord = np.column_stack((data.mapX, map_y))
if y_data is None:
y_data = data_3D[i]
meta_data = coord.astype(object)
else:
y_data = np.vstack((y_data, data_3D[i]))
meta_data = np.vstack((meta_data, coord))
elif type(data) == opusFC.ImageTRCDataReturn:
metas.extend([
ContinuousVariable.make('map_x'),
ContinuousVariable.make('map_y')
])
attrs = [
ContinuousVariable.make(repr(data.labels[i]))
for i in range(len(data.labels))
]
data_3D = data.traces
for i in np.ndindex(data_3D.shape[:1]):
map_y = np.full_like(data.mapX, data.mapY[i])
coord = np.column_stack((data.mapX, map_y))
if y_data is None:
y_data = data_3D[i]
meta_data = coord.astype(object)
else:
y_data = np.vstack((y_data, data_3D[i]))
meta_data = np.vstack((meta_data, coord))
elif type(data) == opusFC.TimeResolvedTRCDataReturn:
y_data = data.traces
elif type(data) == opusFC.TimeResolvedDataReturn:
metas.extend([ContinuousVariable.make('z')])
y_data = data.spectra
meta_data = data.z
elif type(data) == opusFC.SingleDataReturn:
y_data = data.y[None, :]
else:
raise ValueError(
"Empty or unsupported opusFC DataReturn object: " + type(data))
import_params = ['SRT', 'SNM']
for param_key in import_params:
try:
param = data.parameters[param_key]
except KeyError:
pass # TODO should notify user?
else:
try:
param_name = opusFC.paramDict[param_key]
except KeyError:
param_name = param_key
if param_key == 'SRT':
var = TimeVariable.make(param_name)
elif type(param) is float:
var = ContinuousVariable.make(param_name)
elif type(param) is str:
var = StringVariable.make(param_name)
else:
raise ValueError #Found a type to handle
metas.extend([var])
params = np.full((y_data.shape[0], ), param,
np.array(param).dtype)
if meta_data is not None:
# NB dtype default will be np.array(fill_value).dtype in future
meta_data = np.column_stack(
(meta_data, params.astype(object)))
else:
meta_data = params
domain = Orange.data.Domain(attrs, clses, metas)
meta_data = np.atleast_2d(meta_data)
table = Orange.data.Table.from_numpy(domain,
y_data.astype(float, order='C'),
metas=meta_data)
return table
cwd = os.getcwd()
file_list = os.listdir(cwd)
# Loop through files in file_list
L = []
#del file_list[0] # remove .DS_Store
file_list = glob.glob(cwd + "/*.[0-9]")
# .SNM
SNM = []
for f in file_list:
dbs = opusFC.listContents(f)
data = opusFC.getOpusData(f, dbs[0])
SNM.append(data.parameters['SNM'])
# .INS
INS = []
for f in file_list:
data = opusFC.getOpusData(f, dbs[0])
INS.append(data.parameters['INS'])
# .DAT
DAT = []
for f in file_list:
data = opusFC.getOpusData(f, dbs[0])
DAT.append(data.parameters['DAT'])
# .TIM
def read(self):
import opusFC
if self.sheet:
db = self.sheet
else:
db = self.sheets[0]
db = tuple(db.split(" "))
dim = db[1]
try:
data = opusFC.getOpusData(self.filename, db)
except Exception:
raise IOError("Couldn't load spectrum from " + self.filename)
attrs, clses, metas = [], [], []
attrs = [ContinuousVariable.make(repr(data.x[i]))
for i in range(data.x.shape[0])]
y_data = None
meta_data = None
if dim == '3D':
metas.extend([ContinuousVariable.make('map_x'),
ContinuousVariable.make('map_y')])
if db[0] == 'TRC':
attrs = [ContinuousVariable.make(repr(data.labels[i]))
for i in range(len(data.labels))]
data_3D = data.traces
else:
data_3D = data.spectra
for i in np.ndindex(data_3D.shape[:1]):
map_y = np.full_like(data.mapX, data.mapY[i])
coord = np.column_stack((data.mapX, map_y))
if y_data is None:
y_data = data_3D[i]
meta_data = coord.astype(object)
else:
y_data = np.vstack((y_data, data_3D[i]))
meta_data = np.vstack((meta_data, coord))
elif dim == '2D':
y_data = data.y[None,:]
try:
stime = data.parameters['SRT']
except KeyError:
pass # TODO notify user?
else:
metas.extend([TimeVariable.make(opusFC.paramDict['SRT'])])
if meta_data is not None:
dates = np.full(meta_data[:,0].shape, stime, np.array(stime).dtype)
meta_data = np.column_stack((meta_data, dates.astype(object)))
else:
meta_data = np.array([stime])[None,:]
import_params = ['SNM']
for param_key in import_params:
try:
param = data.parameters[param_key]
except Exception:
pass # TODO should notify user?
else:
try:
param_name = opusFC.paramDict[param_key]
except KeyError:
param_name = param_key
if type(param) is float:
var = ContinuousVariable.make(param_name)
elif type(param) is str:
var = StringVariable.make(param_name)
else:
raise ValueError #Found a type to handle
metas.extend([var])
if meta_data is not None:
# NB dtype default will be np.array(fill_value).dtype in future
params = np.full(meta_data[:,0].shape, param, np.array(param).dtype)
meta_data = np.column_stack((meta_data, params.astype(object)))
else:
meta_data = np.array([param])[None,:]
domain = Orange.data.Domain(attrs, clses, metas)
table = Orange.data.Table.from_numpy(domain,
y_data.astype(float, order='C'),
metas=meta_data)
return table
def export_spectra(in_folder=None,
out_folder=DATA_KSSL,
nb_decimals=4,
max_wavenumber=4000,
valid_name=['XN', 'XS'],
nb_chunks=1):
"""Exports KSSL MIRS spectra into a series of .csv files
Parameters
----------
in_folder: string, optional
Specify the path of the folder containing the KSSL MIRS spectra
out_folder: string, optional
Specify the path of the folder that will contain exported files
nb_decimals: int, optional
Specify floating point precision (to save memory)
max_wavenumber: int, optional
Specify the max wavenumber to be considered in spectra
valid_name: list of str, optional
Specify valid spectra file names
nb_chunks: int, optional
Specify tne number of chunks/files to be created
Returns
-------
None
"""
in_folder = Path(in_folder)
out_folder = Path(out_folder)
if not in_folder.exists():
raise IOError('in_folder not found.')
if not out_folder.exists():
out_folder.mkdir(parents=True)
columns = None
valid_files = [
f for f in in_folder.rglob('*.0')
if re.search(r'X.', f.name)[0] in valid_name
]
for (l_bound, u_bound) in list(chunk(len(valid_files), nb_chunks)):
columns = None
rows_list = []
for i, f in enumerate(tqdm(valid_files[l_bound:u_bound])):
dbs = opusFC.listContents(f)
if dbs:
data = opusFC.getOpusData(str(f), dbs[0])
row = [f.name] + list(data.y[data.x <= max_wavenumber])
rows_list.append(row)
if columns is None:
columns = list(
(data.x[data.x <= max_wavenumber]).astype(int))
df = pd.DataFrame(rows_list, columns=['id'] + list(columns))
df = df.round(nb_decimals)
df.to_csv(out_folder /
'spectra_{}_{}.csv'.format(l_bound, u_bound - 1),
index=False)
def read(self):
try:
import opusFC
except ImportError:
raise RuntimeError(self._OPUS_WARNING)
if self.sheet:
db = self.sheet
else:
db = self.sheets[0]
db = tuple(db.split(" "))
dim = db[1]
try:
data = opusFC.getOpusData(self.filename, db)
except Exception:
raise IOError("Couldn't load spectrum from " + self.filename)
attrs, clses, metas = [], [], []
attrs = [ContinuousVariable.make(repr(data.x[i]))
for i in range(data.x.shape[0])]
y_data = None
meta_data = None
if type(data) == opusFC.MultiRegionDataReturn:
y_data = []
meta_data = []
metas.extend([ContinuousVariable.make('map_x'),
ContinuousVariable.make('map_y'),
StringVariable.make('map_region'),
TimeVariable.make('start_time')])
for region in data.regions:
y_data.append(region.spectra)
mapX = region.mapX
mapY = region.mapY
map_region = np.full_like(mapX, region.title, dtype=object)
start_time = region.start_time
meta_region = np.column_stack((mapX, mapY,
map_region, start_time))
meta_data.append(meta_region.astype(object))
y_data = np.vstack(y_data)
meta_data = np.vstack(meta_data)
elif type(data) == opusFC.MultiRegionTRCDataReturn:
y_data = []
meta_data = []
metas.extend([ContinuousVariable.make('map_x'),
ContinuousVariable.make('map_y'),
StringVariable.make('map_region')])
attrs = [ContinuousVariable.make(repr(data.labels[i]))
for i in range(len(data.labels))]
for region in data.regions:
y_data.append(region.spectra)
mapX = region.mapX
mapY = region.mapY
map_region = np.full_like(mapX, region.title, dtype=object)
meta_region = np.column_stack((mapX, mapY, map_region))
meta_data.append(meta_region.astype(object))
y_data = np.vstack(y_data)
meta_data = np.vstack(meta_data)
elif type(data) == opusFC.ImageDataReturn:
metas.extend([ContinuousVariable.make('map_x'),
ContinuousVariable.make('map_y')])
data_3D = data.spectra
for i in np.ndindex(data_3D.shape[:1]):
map_y = np.full_like(data.mapX, data.mapY[i])
coord = np.column_stack((data.mapX, map_y))
if y_data is None:
y_data = data_3D[i]
meta_data = coord.astype(object)
else:
y_data = np.vstack((y_data, data_3D[i]))
meta_data = np.vstack((meta_data, coord))
elif type(data) == opusFC.ImageTRCDataReturn:
metas.extend([ContinuousVariable.make('map_x'),
ContinuousVariable.make('map_y')])
attrs = [ContinuousVariable.make(repr(data.labels[i]))
for i in range(len(data.labels))]
data_3D = data.traces
for i in np.ndindex(data_3D.shape[:1]):
map_y = np.full_like(data.mapX, data.mapY[i])
coord = np.column_stack((data.mapX, map_y))
if y_data is None:
y_data = data_3D[i]
meta_data = coord.astype(object)
else:
y_data = np.vstack((y_data, data_3D[i]))
meta_data = np.vstack((meta_data, coord))
elif type(data) == opusFC.TimeResolvedTRCDataReturn:
y_data = data.traces
elif type(data) == opusFC.TimeResolvedDataReturn:
metas.extend([ContinuousVariable.make('z')])
y_data = data.spectra
meta_data = data.z
elif type(data) == opusFC.SingleDataReturn:
y_data = data.y[None, :]
else:
raise ValueError("Empty or unsupported opusFC DataReturn object: " + type(data))
import_params = ['SRT', 'SNM']
for param_key in import_params:
try:
param = data.parameters[param_key]
except KeyError:
pass # TODO should notify user?
else:
try:
param_name = opusFC.paramDict[param_key]
except KeyError:
param_name = param_key
if param_key == 'SRT':
var = TimeVariable.make(param_name)
elif type(param) is float:
var = ContinuousVariable.make(param_name)
elif type(param) is str:
var = StringVariable.make(param_name)
else:
raise ValueError #Found a type to handle
metas.extend([var])
params = np.full((y_data.shape[0],), param, np.array(param).dtype)
if meta_data is not None:
# NB dtype default will be np.array(fill_value).dtype in future
meta_data = np.column_stack((meta_data, params.astype(object)))
else:
meta_data = params
domain = Orange.data.Domain(attrs, clses, metas)
meta_data = np.atleast_2d(meta_data)
table = Orange.data.Table.from_numpy(domain,
y_data.astype(float, order='C'),
metas=meta_data)
return table
