def tableConvert(damaskTalbe,
hdf5table=None, mode='new', groupname=None):
"""
DESCRIPTION
-----------
tableConvert(PATH_TO_DAMASK_ASCII_TALBE,
hdf5table=MY_HDF5_NAME)
convert a single ASCII table with DAMASK convention into HDF5 format, which can be
either stored in a new HDF5 file or appended to an existing one.
PARAMETERS
----------
damaskTalbe: str
path to the damask ASCII table for conversion
hdf5table: str
path to the new hdf5 table, default is the same as damaskTalbe with
hdf5 as new extension
mode: str = ['append', 'new']
data storage mode
"""
# set HDF5 path
if hdf5table is None:
hdf5table = damaskTalbe.replace(".txt", ".hdf5")
# set writing mode
mode = mode.lower()
if mode == 'new':
wmode = 'w'
elif mode == 'append':
wmode = 'r+'
else:
raise ValueError("unknown mode, use new/append")
# set up HDF5 table
mytable = h5py.file(hdf5table, wmode)
def tableConvert(damaskTalbe, hdf5table=None, mode="new", groupname=None):
"""
DESCRIPTION
-----------
tableConvert(PATH_TO_DAMASK_ASCII_TALBE,
hdf5table=MY_HDF5_NAME)
convert a single ASCII table with DAMASK convention into HDF5 format, which can be
either stored in a new HDF5 file or appended to an existing one.
PARAMETERS
----------
damaskTalbe: str
path to the damask ASCII table for conversion
hdf5table: str
path to the new hdf5 table, default is the same as damaskTalbe with
hdf5 as new extension
mode: str = ['append', 'new']
data storage mode
"""
# set HDF5 path
if hdf5table is None:
hdf5table = damaskTalbe.replace(".txt", ".hdf5")
# set writing mode
mode = mode.lower()
if mode == "new":
wmode = "w"
elif mode == "append":
wmode = "r+"
else:
raise ValueError("unknown mode, use new/append")
# set up HDF5 table
mytable = h5py.file(hdf5table, wmode)
def get_value(self, bit_value, iter_of_file, clef): """permet de reccuperer une valeur grace a une adresse""" #recreation du chemin PATH=self.__DICTIONARY_DIRECTORY_PATH__+'/_'+str(bit_value)+'/_'+str(bit_value)+"_"+str(iter_of_file)+__extension_fich_dict__ db = h5py.file(PATH, 'r') #get from shelve bizarre valeur=db[clef] db.close() return(valeur)
def __creat_dict__(self, bit_value, stat="w"):
"""creer un dictionnaire sur le disque pour les futurs valeurs. c est ici qu est formaté le nom des dossiers et des fichiers. basé sur shelve molule"""
max_iter=nbr_de_dict_existant()
if stat=="c" and max_iter!=-1: #si mode creation + fichier deja existants
raise Warning("des archives existent deja dans", __DICTIONARY_DIRECTORY_PATH__+'/_'+str(bit_value))
db= h5py.file(self.__DICTIONARY_DIRECTORY_PATH__+'/_'+str(bit_value)+'/_'+str(bit_value)+"_"+str(iter)+__extension_fich_dict__, 'a')
db['init'] = h0 #inserer les val en hexadecimal?
bd.close() #sync et ferme le dico
return(max_iter)
def proc_RX(WF_file,
shots,
rxData=None,
sigmas=np.arange(0, 5, 0.25),
deltas=np.arange(-1, 1.5, 0.5),
TX=None,
countPeaks=True,
WF_library=None,
TX_file=None,
scat_file=None,
catalogBuffer=None):
"""
Routine to process the wavefoms in an ATM waveform file. Can be run inside
the loop of fit_ATM_scat, or run on a few waveforms at a time for plot generation
"""
if TX is None and TX_file is not None:
with h5py.file(TX_file) as h5f:
TX = waveform(np.array(h5f['/TX/t']), np.array(h5f(['/TX/p'])))
TX.t -= TX.nSigmaMean()[0]
TX.tc = 0
# make the library of templates
if WF_library is None:
WF_library = dict()
WF_library.update({0.: TX})
if scat_file is not None:
WF_library.update(make_rx_scat_catalog(TX, h5_file=scat_file))
if rxData is None:
# make the return waveform structure
D = read_ATM_file(WF_file, shot0=shots[0], nShots=shots[-1] - shots[0])
rxData = D['RX']
rxData.t -= np.nanmean(rxData.t)
else:
D = dict()
if countPeaks:
rxData.nPeaks = rxData.count_peaks(W=4)
else:
rxData.nPeaks = np.ones(rxData.size)
threshold_mask = rxData.p >= 255
rxData.subBG(t50_minus=3.)
rxData.tc = [ii.nSigmaMean()[0] for ii in rxData]
#rxData.tc=rxData.t50()
rxData.p[threshold_mask] = np.NaN
# fit the data. Catalogbuffer contains waveform templates that have already been tried
D_out, catalogBuffer=fit_catalog(rxData, WF_library, sigmas, deltas, return_data_est=True, \
return_catalog=True, catalog=catalogBuffer)
return D_out, rxData, D, catalogBuffer
def fileOpener(file, mode='r'):
""" A file opener helper function with some error handling.
This can open files through a file object, a h5py file, or just the filename.
"""
# Were we handed a file object or just a file name string?
if type(file) is file:
filename, mode = file.name(), file.mode()
file.close()
h5f = h5.file(filename, mode)
elif type(file) is h5._hl.files.File:
h5f = file
elif type(file) is str:
filename = file
h5f = h5.File(filename, mode)
else:
raise FileError
return h5f
def proc_WFs(WF_file, TX_file, shots, scat_file=None):
h5f = h5py.file(TX_file)
TX = {'t': np.array(h5f['/TX/t']), 'p': np.array(h5f(['/TX/p']))}
h5f.close()
# make the library of templates
WF_library = dict()
WF_library.update({0.: TX})
WF_library.update(make_rx_scat_catalog(TX, h5_file=scat_file))
catalogBuffer = None
result = dict()
for shot0 in shots:
D = read_ATM_file(WF_file, shot0=shot0, nShots=1)
# make the return waveform structure
rxData = D['RX'][0:D['RX'].size]
rxData.t = rxData.t - rxData.t.mean()
nP = np.ones(rxData.size)
sigmas = np.arange(0, 5, 0.125)
rxData.nPeaks = nP
threshold_mask = rxData.p >= 255
rxData.subBG()
rxData.tc = [ii.nSigmaMean()[0] for ii in rxData]
rxData.p[threshold_mask] = np.NaN
# choose a set of delta t values
deltas = np.arange(-1, 1.5, 0.5)
# fit the data. Catalogbuffer contains waveform templates that have already been tried
D_out, catalogBuffer=fit_catalog(rxData, WF_library, sigmas, deltas, return_data_est=True, \
return_catalog=True, catalog=catalogBuffer)
for ind, shot in enumerate(D_out['shot']):
if shot in shots:
result[shot] = dict()
for key in D_out:
result[shot][key] = D_out[key][ind]
return result
"""pour inserer une nouvelle valeur dans les dict (apres verification de non-existance)""" #verification de la place disponible #redonne l'adresse si trouve de la place ( variable writable) sinon false writable='' for files in range(__nbr_de_dict_existant__(self, bit_value)+1): path =self.__DICTIONARY_DIRECTORY_PATH__+'/_'+str(bit_value)+'/_'+str(bit_value)+"_"+str(iter)+__extension_fich_dict__ if __check_size__(path) < self.__DICTIONARY_MAX_SIZE__ : writable=path break if writable='': #if no place where found bit_value=valeur[__byte_number_for_inital_division__] #n premiers bits pour classement new_val_of_dict = __creat_dict__(self, bit_value, stat="w") #creation d un nouveau writable=self.__DICTIONARY_DIRECTORY_PATH__+'/_'+str(bit_value)+'/_'+str(bit_value)+"_"+str(new_val_of_dict)+__extension_fich_dict__ #maintenant qu'on est sur d avoir de la place dispo db = h5py.file(writable, 'r') creationdunenouvelleClé #### clef = ?? db[clef] = value_to_write db.close() return(bit_value, iter_of_file, clef) def __check_size__(self, file_path): """a pour but de determiner la taille du fichier archive""" size= os.path.getsize(file_path) return(size) def get_value(self, bit_value, iter_of_file, clef): """permet de reccuperer une valeur grace a une adresse""" #recreation du chemin PATH=self.__DICTIONARY_DIRECTORY_PATH__+'/_'+str(bit_value)+'/_'+str(bit_value)+"_"+str(iter_of_file)+__extension_fich_dict__
data[i+29] = (features, 1)
for i in range(1,31):
(rate, x_data[i]) = get_sig("ModernOTData/off" + i + ".wav")
(features, f_names) = get_st_features(x_data[i], rate)
data[i+59] = (features, 2)
for i in range(1,31):
(rate, x_data[i]) = get_sig("ModernOTData/slack" + i + ".wav")
(features, f_names) = get_st_features(x_data[i], rate)
data[i+89] = (features, 3)
for i in range(1,31):
(rate, x_data[i]) = get_sig("ModernOTData/light" + i + ".wav")
(features, f_names) = get_st_features(x_data[i], rate)
data[i+119] = (features, 4)
#shuffle and split data
np.random.shuffle(data)
train = data[:int(len(data)*0.8)]
test = data[int(len(data)*0.8):]
trainX = train[:, 0]
trainY = train[:, 1]
testX = test[:, 0]
testY = test[:, 1]
#save data as an hdf5 file
h5f = h5py.file('data.h5', 'w')
h5f.create_dataset('testx', data=testX)
h5f.create_dataset('testy', data=testY)
h5f.create_dataset('trainx', data=trainX)
h5f.create_dataset('trainy', data=trainY)
def load_dataset():
train_dataset = h5py.file('datasets/train_catvnoncat.h5', 'r')
train_set_x_orig = np.array()