def main(): start = '2021-03-02' end = '2021-03-05' dates = (start, end) BTC = dr.DataReader('BTCUSDT', 'binance', dates, tick='1d') TSLA = dr.DataReader('TSLA', 'yahoo', dates) print('BTC:', BTC.Dates, '\n', 'TSLA:', TSLA.Dates)
def plot_whole_signal_and_tasks_times(path, ident): """Plots signal, which was acquired wirelessly with GNU Radio, along with colored task timeframes.""" dataread = datareader.DataReader(path, ident) # initialize path to data data = dataread.read_grc_data() # read from files data = dataread.unwrap_grc_data() # unwrap phase. returns time and y values task_timestamps = dataread.get_data_task_timestamps() relax_timestamps = dataread.get_relax_timestamps() plt.figure(2) plt.clf() plt.plot(data[0], data[1]) plt.xlabel('time (s)', fontsize=12) plt.ylabel('distance', fontsize=12) loc = plticker.MultipleLocator(base=50.0) for (start, stop) in task_timestamps: plt.axvspan(start, stop, alpha=0.4, color='r') for (start, stop) in relax_timestamps: plt.axvspan(start, stop, alpha=0.4, color='b') tasks = ['HP\nhigh', 'HP\nlow', 'HP\nmedium', 'FA\nhigh', 'FA\nmedium', 'FA\nlow', 'GC\nmedium', 'GC\nhigh', 'GC\nlow', 'NC\nlow', 'NC\nhigh', 'NC\nmedium', 'SX\nlow', 'SX\nmedium', 'SX\nhigh', 'PT\nmedium', 'PT\nlow', 'PT\nhigh'] task_i = 0 for (start, stop) in task_timestamps: plt.text((start + stop) / 2, 0, tasks[task_i], horizontalalignment='center', fontsize=12, clip_on=True) task_i += 1 plt.show()
def do_validation(m_valid, sess, valid_path): # dataset reader setting # valid_dr = dr.DataReader(valid_path["valid_input_path"], valid_path["valid_output_path"], valid_path["norm_path"], dist_num=config.dist_num, is_shuffle=False, is_val=True) valid_cost_list = [] while True: valid_inputs, valid_labels, valid_inphase, valid_outphase = valid_dr.whole_batch(valid_dr.num_samples) feed_dict = {m_valid.inputs: valid_inputs, m_valid.labels: valid_labels, m_valid.keep_prob: 1.0} valid_cost = sess.run(m_valid.cost, feed_dict=feed_dict) valid_cost_list.append(np.expand_dims(valid_cost, axis=1)) if valid_dr.file_change_checker(): valid_dr.file_change_initialize() if valid_dr.eof_checker(): valid_dr.reader_initialize() print('Valid data reader was initialized!') # initialize eof flag & num_file & start index break valid_cost_list = np.concatenate(valid_cost_list, axis=0) total_avg_valid_cost = np.asscalar(np.mean(valid_cost_list)) return total_avg_valid_cost
def do_validation_G(m_valid, sess, valid_path): # dataset reader setting # valid_dr = dr.DataReader(valid_path["valid_input_path"], valid_path["valid_output_path"], valid_path["norm_path"], dist_num=config.dist_num) avg_valid_accuracy = 0. avg_valid_cost = 0. itr_sum = 0. accuracy_list = [0 for i in range(valid_dr._file_len)] cost_list = [0 for i in range(valid_dr._file_len)] itr_file = 0 while True: valid_inputs, valid_labels = valid_dr.next_batch(config.batch_size) if valid_dr.file_change_checker(): # print(itr_file) accuracy_list[itr_file] = avg_valid_accuracy / itr_sum cost_list[itr_file] = avg_valid_cost / itr_sum avg_valid_cost = 0. avg_valid_accuracy = 0. itr_sum = 0 itr_file += 1 valid_dr.file_change_initialize() if valid_dr.eof_checker(): valid_dr.reader_initialize() print('Valid data reader was initialized!' ) # initialize eof flag & num_file & start index break feed_dict = { m_valid.inputs: valid_inputs, m_valid.labels: valid_labels } # valid_cost, valid_softpred, valid_raw_labels\ # = sess.run([m_valid.cost, m_valid.softpred, m_valid.raw_labels], feed_dict=feed_dict) # # fpr, tpr, thresholds = metrics.roc_curve(valid_raw_labels, valid_softpred, pos_label=1) # valid_auc = metrics.auc(fpr, tpr) valid_cost, valid_accuracy = sess.run( [m_valid.C1_loss, m_valid.d1_loss], feed_dict=feed_dict) avg_valid_accuracy += valid_accuracy avg_valid_cost += valid_cost itr_sum += 1 total_avg_valid_accuracy = np.asscalar(np.mean(np.asarray(accuracy_list))) total_avg_valid_cost = np.asscalar(np.mean(np.asarray(cost_list))) return total_avg_valid_accuracy, total_avg_valid_cost
def get_data(self, symbol): coin_data = dr.DataReader('{}USDT'.format(symbol), 'binance', self.Dates, tick='1d') data = (coin_data.Dates, coin_data.Closes) return data
def __init__(self, parent=None, filename='default.csv'): super(ProcessorThread, self).__init__() self.file = filename self.rowBuffer = [] self.changeZ = False self.running = True self.reader = datareader.DataReader(parent=self, filename=self.file) self.reader.lineRead.connect(self.newLineRead)
def enhance(self, wav_dir): noisy_speech = utils.read_raw(wav_dir) temp_dir = './temp/temp.npy' np.save(temp_dir, noisy_speech) test_dr = dr.DataReader(temp_dir, '', self.norm_path, dist_num=config.dist_num, is_training=False, is_shuffle=False) mean, std = test_dr.norm_process(self.norm_path + '/norm_noisy.mat') while True: test_inputs, test_labels, test_inphase, test_outphase = test_dr.whole_batch( test_dr.num_samples) if config.mode != 'lstm' and config.mode != 'fcn': feed_dict = { self.node_inputs: test_inputs, self.node_labels: test_labels, self.node_keep_prob: 1.0 } else: feed_dict = { self.node_inputs: test_inputs, self.node_labels: test_labels } pred = self.sess.run(self.node_prediction, feed_dict=feed_dict) if test_dr.file_change_checker(): print(wav_dir) lpsd = np.expand_dims(np.reshape(pred, [-1, config.freq_size]), axis=2) lpsd = np.squeeze((lpsd * std * config.global_std) + mean) recon_speech = utils.get_recon(np.transpose(lpsd, (1, 0)), np.transpose( test_inphase, (1, 0)), win_size=config.win_size, win_step=config.win_step, fs=config.fs) test_dr.reader_initialize() break file_dir = self.save_dir + '/' + os.path.basename(wav_dir).replace( 'noisy', 'enhanced').replace('raw', 'wav') librosa.output.write_wav(file_dir, recon_speech, int(config.fs), norm=True) return recon_speech
def speech_enhance(wav_dir, graph_name): noisy_speech = utils.read_raw(wav_dir) temp_dir = './temp/temp.npy' np.save(temp_dir, noisy_speech) graph = gt.load_graph(graph_name) norm_path = os.path.abspath('./data/train/norm') test_dr = dr.DataReader(temp_dir, '', norm_path, dist_num=config.dist_num, is_training=False, is_shuffle=False) node_inputs = graph.get_tensor_by_name('prefix/model_1/inputs:0') node_labels = graph.get_tensor_by_name('prefix/model_1/labels:0') node_keep_prob = graph.get_tensor_by_name('prefix/model_1/keep_prob:0') node_prediction = graph.get_tensor_by_name('prefix/model_1/pred:0') pred = [] lab = [] sess_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) sess_config.gpu_options.allow_growth = True while True: test_inputs, test_labels = test_dr.next_batch(config.test_batch_size) feed_dict = {node_inputs: test_inputs, node_labels: test_labels, node_keep_prob: 1.0} with tf.Session(graph=graph, config=sess_config) as sess: pred_temp, lab_temp = sess.run([node_prediction, node_labels], feed_dict=feed_dict) pred.append(pred_temp) lab.append(lab_temp) # print(test_dr.file_change_checker()) if test_dr.file_change_checker(): print(wav_dir) phase = test_dr.phase[0] lpsd = np.expand_dims(np.reshape(np.concatenate(pred, axis=0), [-1, config.freq_size])[0:phase.shape[0], :], axis=2) mean, std = test_dr.norm_process(norm_path + '/norm_noisy.mat') lpsd = np.squeeze((lpsd * std) + mean) # denorm recon_speech = utils.get_recon(np.transpose(lpsd, (1, 0)), np.transpose(phase, (1, 0)), win_size=config.win_size, win_step=config.win_step, fs=config.fs) # plt.plot(recon_speech) # plt.show() # lab = np.reshape(np.asarray(lab), [-1, 1]) test_dr.reader_initialize() break return recon_speech
def __init__(self, symbols, period, start, end, username, password, cache=True): self.symbols = symbols self.start = start self.end = end self.period = period self.cache = cache self.datareader = datareader.DataReader(username, password, cache)
def main(): start = '2020-02-05' end = '2021-03-05' dates = (start, end) BTC = dr.DataReader('BTCUSDT', 'binance', dates) BTCVariations = vr.Variations(BTC.Dates, BTC.Closes, normalized=True) plt.plot(BTCVariations.Dates, BTCVariations.Variations) plt.xlim(BTCVariations.Dates[0], BTCVariations.Dates[-1]) plt.ylim(min(BTCVariations.Variations), max(BTCVariations.Variations)) plt.show() plt.plot(BTCVariations.ShiftedDates, BTCVariations.ShiftedCloses) plt.xlim(BTCVariations.ShiftedDates[0], BTCVariations.ShiftedDates[-1]) plt.show()
def read_data_from_prepared_datapath(self): """ Function is called in the end of process :return: """ reader = datareader.DataReader() self.datap = reader.Get3DData(self.datapath, dataplus_format=True) _set_label_text(self.text_dcm_dir, _make_text_short(self.datapath), self.datapath) _set_label_text(self.text_dcm_data, self.get_data_info()) if self.after_function is not None: self.after_function(self) self.__show_message('Data read finished')
def newFile(self, newfile): global DEP_DATA DEP_DATA = [] self.rowBuffer = [] if self.reader: self.reader.end() self.reader = datareader.DataReader(parent=self, filename=newfile) self.reader.lineRead.connect(self.newLineRead) self.reader.start() # re-initialize DATA_DICT column numbers used for data processing try: self.tcolnum = getCol('Src%d Motor Tilt Position' %int(filename_handler.FILE_INFO['Source'])) except IndexError: self.srcError.emit(int(filename_handler.FILE_INFO['Source'])) self.zcolnum = getCol('Platen Zshift Motor 1 Position') self.anglecolnum = getCol('Platen Motor Position')
def show(data3d_a_path, sliver_seg_path, ourSegmentation): reader = datareader.DataReader() #data3d_a_path = os.path.join(path_to_script, data3d_a_path) datap_a = reader.Get3DData(data3d_a_path, dataplus_format=True) if 'orig_shape' in datap_a.keys(): # pklz data3d_a = qmisc.uncrop(datap_a['data3d'], datap_a['crinfo'], datap_a['orig_shape']) else: #dicom data3d_a = datap_a['data3d'] if sliver_seg_path is not None: sliver_seg_path = os.path.join(path_to_script, sliver_seg_path) sliver_datap = reader.Get3DData(sliver_seg_path, dataplus_format=True) if 'segmentation' in sliver_datap.keys(): sliver_seg = sliver_datap['segmentation'] sliver_seg = qmisc.uncrop(sliver_datap['segmentation'], sliver_datap['crinfo'], data3d_a.shape) else: sliver_seg = sliver_datap['data3d'] pyed = sed3.sed3(data3d_a, contour=sliver_seg) print("Sliver07 segmentation") pyed.show() if ourSegmentation != None: ourSegmentation = os.path.join(path_to_script, ourSegmentation) datap_our = reader.Get3DData(ourSegmentation, dataplus_format=True) #data_our = misc.obj_from_file(ourSegmentation, 'pickle') #data3d_our = data_our['segmentation'] our_seg = qmisc.uncrop(datap_our['segmentation'], datap_our['crinfo'], data3d_a.shape) if ourSegmentation != None: pyed = sed3.sed3(data3d_a, contour=our_seg) print("Our segmentation") pyed.show() if (ourSegmentation is not None) and (sliver_seg_path is not None): diff = (our_seg.astype(np.int8) - sliver_seg) diff[diff == -1] = 2 #import ipdb; ipdb.set_trace() # BREAKPOINT pyed = sed3.sed3(data3d_a, contour=our_seg, seeds=diff) print("Sliver07 and our segmentation differences") pyed.show()
def get_engagement_increase_vs_decrease_timeframes(path, ident, seconds): """Returns raw data from either engagement 'increase' or 'decrease' time frames and their class (0 or 1). TODO: join functions""" dataread = datareader.DataReader(path, ident) # initialize path to data data = dataread.read_grc_data() # read from files samp_rate = int(round(len(data[1]) / max(data[0]))) cog_res = dataread.read_cognitive_load_study( str(ident) + '-primary-extract.txt') tasks_data = np.empty((0, seconds * samp_rate)) tasks_y = np.empty((0, 1)) busy_n = dataread.get_data_task_timestamps(return_indexes=True) relax_n = dataread.get_relax_timestamps(return_indexes=True) for i in cog_res['task_number']: task_num_table = i - 225 # 0 - 17 ### engagement increase / decrease if task_num_table == 0: continue mid = int( (relax_n[task_num_table][0] + relax_n[task_num_table][1]) / 2) length = int(samp_rate * 30) for j in range(10): new_end = int(mid - j * samp_rate) new_start2 = int(mid + j * samp_rate) dataextract_decrease = dataextractor.DataExtractor( data[0][new_end - length:new_end], data[1][new_end - length:new_end], samp_rate) dataextract_increase = dataextractor.DataExtractor( data[0][new_start2:new_start2 + length], data[1][new_start2:new_start2 + length], samp_rate) try: tasks_data = np.vstack((tasks_data, dataextract_increase.y)) tasks_y = np.vstack((tasks_y, 1)) tasks_data = np.vstack((tasks_data, dataextract_decrease.y)) tasks_y = np.vstack((tasks_y, 0)) except ValueError: print(ident) # ignore short windows return tasks_data, tasks_y
def read_data_orig_and_seg(inputdata, i): """ Loads data_orig and data_seg from yaml file """ reader = datareader.DataReader() data3d_a_path = os.path.join(inputdata['basedir'], inputdata['data'][i]['sliverseg']) data3d_a, metadata_a = reader.Get3DData(data3d_a_path) data3d_b_path = os.path.join(inputdata['basedir'], inputdata['data'][i]['sliverorig']) data3d_b, metadata_b = reader.Get3DData(data3d_b_path) #import pdb; pdb.set_trace() data3d_seg = (data3d_a > 0).astype(np.int8) data3d_orig = data3d_b return data3d_orig, data3d_seg
def get_task_complexities_timeframes(path, ident, seconds): """Returns raw data along with task complexity class. TODO: join functions. Add parameter to choose different task types and complexities""" dataread = datareader.DataReader(path, ident) # initialize path to data data = dataread.read_grc_data() # read from files samp_rate = int(round(len(data[1]) / max(data[0]))) cog_res = dataread.read_cognitive_load_study( str(ident) + '-primary-extract.txt') tasks_data = np.empty((0, seconds * samp_rate)) tasks_y = np.empty((0, 1)) busy_n = dataread.get_data_task_timestamps(return_indexes=True) relax_n = dataread.get_relax_timestamps(return_indexes=True) for i in cog_res['task_number']: task_num_table = i - 225 # 0 - 17 ### task complexity classification if cog_res['task_complexity'][task_num_table] == 'medium': continue # if cog_res['task_label'][task_num_table] == 'FA' or cog_res['task_label'][task_num_table] == 'HP': # continue if cog_res['task_label'][task_num_table] != 'NC': continue map_compl = {'low': 0, 'medium': 2, 'high': 1} for j in range(10): new_end = int(busy_n[task_num_table][1] - j * samp_rate) new_start = int(new_end - samp_rate * 30) dataextract = dataextractor.DataExtractor( data[0][new_start:new_end], data[1][new_start:new_end], samp_rate) try: tasks_data = np.vstack((tasks_data, dataextract.y)) tasks_y = np.vstack( (tasks_y, map_compl.get( cog_res['task_complexity'][task_num_table]))) except ValueError: print(ident) return tasks_data, tasks_y
def get_busy_vs_relax_timeframes(path, ident, seconds): """Returns raw data from either 'on task' or 'relax' time frames and their class (0 or 1). TODO: join functions""" dataread = datareader.DataReader(path, ident) # initialize path to data data = dataread.read_grc_data() # read from files samp_rate = int(round(len(data[1]) / max(data[0]))) cog_res = dataread.read_cognitive_load_study( str(ident) + '-primary-extract.txt') tasks_data = np.empty((0, seconds * samp_rate)) tasks_y = np.empty((0, 1)) busy_n = dataread.get_data_task_timestamps(return_indexes=True) relax_n = dataread.get_relax_timestamps(return_indexes=True) for i in cog_res['task_number']: task_num_table = i - 225 # 0 - 17 ### task versus relax (1 sample each) dataextract = dataextractor.DataExtractor( data[0][busy_n[task_num_table][0]:busy_n[task_num_table][1]], data[1][busy_n[task_num_table][0]:busy_n[task_num_table][1]], samp_rate) dataextract_relax = dataextractor.DataExtractor( data[0][relax_n[task_num_table][0]:relax_n[task_num_table][1]], data[1][relax_n[task_num_table][0]:relax_n[task_num_table][1]], samp_rate) try: tasks_data = np.vstack( (tasks_data, dataextract.y[-samp_rate * seconds:])) tasks_y = np.vstack((tasks_y, 1)) tasks_data = np.vstack( (tasks_data, dataextract_relax.y[-samp_rate * seconds:])) tasks_y = np.vstack((tasks_y, 0)) except ValueError: print(ident) # ignore short windows return tasks_data, tasks_y
def do_validation(m_valid, sess, valid_path): # dataset reader setting # valid_dr = dr.DataReader(valid_path["valid_input_path"], valid_path["valid_output_path"], valid_path["norm_path"], dist_num=config.dist_num, is_shuffle=False, is_val=True) valid_cost_list = [] while True: # valid_inputs, valid_labels = valid_dr.next_batch(config.batch_size) valid_inputs, valid_labels = valid_dr.next_batch(config.batch_size) feed_dict = {m_valid.inputs: valid_inputs, m_valid.labels: valid_labels, m_valid.keep_prob: 1.0} # valid_cost, valid_softpred, valid_raw_labels\ # = sess.run([m_valid.cost, m_valid.softpred, m_valid.raw_labels], feed_dict=feed_dict) # # fpr, tpr, thresholds = metrics.roc_curve(valid_raw_labels, valid_softpred, pos_label=1) # valid_auc = metrics.auc(fpr, tpr) valid_cost = sess.run(m_valid.raw_cost, feed_dict=feed_dict) valid_cost_list.append(np.expand_dims(valid_cost, axis=1)) if valid_dr.file_change_checker(): valid_dr.file_change_initialize() if valid_dr.eof_checker(): valid_dr.reader_initialize() print('Valid data reader was initialized!') # initialize eof flag & num_file & start index break valid_cost_list = np.concatenate(valid_cost_list, axis=0) total_avg_valid_cost = np.asscalar(np.mean(valid_cost_list)) return total_avg_valid_cost
import time import numpy as np # ========================================================================== # plt.style.use('classic') size = 20 size_config = .8 legendfont = 10 thickness = 0.3 width = .5 resolution = 300 color_value = '#6b8ba4' # ========================================================================== # start = '2020-04-03' end = '2021-04-03' dates = (start, end) BTC = dr.DataReader('BTCUSDT', 'binance', dates) BTCVariations = vr.Variations(BTC.Dates, BTC.Closes, normalized=True) STORJ = dr.DataReader('STORJUSDT', 'binance', dates) STORJVariations = vr.Variations(STORJ.Dates, STORJ.Closes, normalized=True) ETH = dr.DataReader('ETHUSDT', 'binance', dates) ETHVariations = vr.Variations(ETH.Dates, ETH.Closes, normalized=True) # ========================================================================== # max_variations = [max(BTCVariations.Variations), max(ETHVariations.Variations), max(STORJVariations.Variations)] min_variations = [min(BTCVariations.Variations), min(ETHVariations.Variations), min(STORJVariations.Variations)]
def update_extraction_files(path, ident): dataread = datareader.DataReader(path, ident) # initialize path to data dataread.extract_cognitive_load_study(save_path=path + '/' + str(ident) + '/' + str(ident) + '-primary-extract.txt') dataread.read_grc_data() dataread.fix_grc_data_and_save()
def conv_net_model_train(learning_rate, train_dir, save_dir): """ The feed forward convolutional neural network model Hyper parameters include learning rate, number of convolutional layers and fully connected layers. (Currently TBD) """ # Reset graphs tf.reset_default_graph() # Create placeholders x = tf.placeholder(dtype=tf.float32, shape=[ None, INPUT_IMAGE_DIMENSION, INPUT_IMAGE_DIMENSION, INPUT_IMAGE_CHANNELS ], name="x") y = tf.placeholder(dtype=tf.float32, shape=[None, OUTPUT_VECTOR_SIZE], name="y") weight1 = tf.Variable(tf.truncated_normal([4, 4, 3, 16], stddev=0.1), dtype=tf.float32, name="W1") bias1 = tf.Variable(tf.constant(0.1, shape=[16]), dtype=tf.float32, name="B1") weight2 = tf.Variable(tf.truncated_normal([4, 4, 16, 32], stddev=0.1), dtype=tf.float32, name="W2") bias2 = tf.Variable(tf.constant(0.1, shape=[32]), dtype=tf.float32, name="B2") weight3 = tf.Variable(tf.truncated_normal([4608, 2], stddev=0.1), dtype=tf.float32, name="W3") bias3 = tf.Variable(tf.constant(0.1, shape=[2]), dtype=tf.float32, name="B3") # First convolutional layer conv1 = ly.conv_layer(x, weight1, bias1, False) # First pooling pool1 = ly.pool_layer(conv1) # Second convolutional layer conv2 = ly.conv_layer(pool1, weight2, bias2, True) # Second pooling pool2 = ly.pool_layer(conv2) # Flatten input flattened = tf.reshape(pool2, shape=[-1, 12 * 12 * 32]) # Create fully connected layer logits = ly.fully_connected_layer(flattened, weight3, bias3) # Create loss function with tf.name_scope("cross_entropy"): cross_entropy = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y)) # Create optimizer with tf.name_scope("train"): train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize( cross_entropy) # Compute accuracy with tf.name_scope("accuracy"): # argmax gets the highest value in a given dimension (in this case, dimension 1) # equal checks if the label is equal to the computed logits correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1)) # tf.reduce_mean computes the mean across the vector accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) saver = tf.train.Saver() with tf.Session() as sess: # Run model sess.run(tf.global_variables_initializer()) data_reader = dr.DataReader(sess, train_dir, INPUT_IMAGE_DIMENSION, OUTPUT_VECTOR_SIZE, INPUT_IMAGE_CHANNELS) coord = tf.train.Coordinator() # Train the model for i in range(STEP_SIZE): images, labels = data_reader.get_train_batch(coord, BATCH_SIZE) if i % 10 == 0: a = sess.run(accuracy, feed_dict={x: images, y: labels}) print("step", i, "of ", STEP_SIZE) print("Acc: ", a) # Run the training step sess.run(train_step, feed_dict={x: images, y: labels}) saver.save(sess, save_dir) coord.request_stop()
def compare_extracted_hr_and_band(path, ident): """Compater heart rates acquired wirelessly and with Microfost Band. :param path: (str) main path to data, where user data is located in specific folders :param ident: (str) user identifier :return: MAE, MSE, CORRelation values of the aligned HR time series """ dataread = datareader.DataReader(path, ident) # initialize path to data data = dataread.read_grc_data() # read from files data = dataread.unwrap_grc_data() # unwrap phase. returns time and y values samp_rate = round(len(data[1]) / max(data[0])) dataextract = dataextractor.DataExtractor(data[0], data[1], samp_rate) cog_res = dataread.read_cognitive_load_study(ident + '-primary-extract.txt') end_epoch_time = dataread.get_end_time_cognitive_load_study() # end t extracted_br_features = dataextract.raw_windowing_breathing(30, 1) extracted_br_features['br_rate'] = np.array(extracted_br_features['br_rate'].rolling(6).mean()) extracted_br_features_roll_avg = extracted_br_features.loc[:, extracted_br_features.columns != 'times'].rolling( 6).mean() extracted_br_features_roll_avg['times'] = extracted_br_features['times'] extracted_br_features_roll_avg['br_ok'] = extracted_br_features['br_ok'] extracted_hr_features = dataextract.raw_windowing_heartrate(10, 1) extracted_hr_features = extracted_hr_features.drop(['hr_HRV_lf', 'hr_HRV_hf', 'hr_HRV_lf_hf'], axis=1) extracted_hr_features_roll_avg = extracted_hr_features.loc[:, extracted_hr_features.columns != 'times'].rolling( 10).mean() extracted_hr_features_roll_avg['times'] = extracted_hr_features['times'] extracted_hr_features_roll_avg['hr_ok1'] = extracted_hr_features['hr_ok'] bandread = bandreader.HeartRateBand(path + '_Hrates/', ident) band_data = bandread.load() band_data_time_start = bisect(band_data[0][:], end_epoch_time - data[0][-1] * 1000) band_data_time_stop = bisect(band_data[0][:], end_epoch_time) band_data = [band_data[0][band_data_time_start:band_data_time_stop], band_data[1][band_data_time_start:band_data_time_stop]] band_data_new_data = [(band_data[0] - band_data[0][0]) / 1000, band_data[1]] plt.figure(1) plt.clf() plt.plot(extracted_hr_features_roll_avg['times'], extracted_hr_features_roll_avg['hr_rate'], color='orange', label='Wi-Mind heart rate') plt.plot(band_data_new_data[0], band_data_new_data[1], color='green', label='Microsoft Band heart rate') plt.xlabel('time (s)') plt.ylabel('heart rate') plt.legend() plt.show() hr_data = extracted_hr_features_roll_avg[['times', 'hr_rate']] hr_data['times'] = hr_data['times'].astype(int) band_data = pd.DataFrame() band_data['times'] = band_data_new_data[0] band_data['times'] = band_data['times'].astype(int) band_data['rate'] = band_data_new_data[1] band_data = band_data.drop_duplicates(subset=['times']) together_data = pd.merge(hr_data, band_data, on='times') together_data = together_data.dropna() # new_hr = res_ind[intersect] # new_band = band_data_new__data[1][intersect] mae = metrics.mean_absolute_error(together_data['rate'], together_data['hr_rate']) mse = metrics.mean_squared_error(together_data['rate'], together_data['hr_rate']) corr = stats.pearsonr(together_data['rate'], together_data['hr_rate']) # print('mae amd mse: ', mae, mse) return mae, mse, corr
maxXPoints = statx[1][1] xpoints = numpy.linspace(0.0, maxXPoints) plt.plot(x, y, 'x', xpoints, poly(xpoints), '-') pdf.savefig(fig) print("---") if __name__ == '__main__': start = int(round(time.time() * 1000)) files = {} for filename in sys.argv[1:]: data = datareader.DataReader(filename) files[data.getBasename()] = data helper = {} for fileno, filename in enumerate(files.keys()): for attribute in files[filename].get_numerical_attributes(): helper[str(fileno) + "." + str(attribute['position'])] = (filename, attribute['attr_name']) for combination in list(itertools.combinations(helper.keys(), 2)): filename1, attr1 = helper[combination[0]] filename2, attr2 = helper[combination[1]] data1 = files[filename1].getData(attr1)
def full_signal_extract(path, ident): """Extract breathing and heartbeat features from one user and save features to file. :param path: (str) main path to data, where user data is located in specific folders :param ident: (str) user identifier :return: Nothing. It saves features (dataframe) to a .csv file """ dataread = datareader.DataReader(path, ident) # initialize path to data data = dataread.read_grc_data() # read from files data = dataread.unwrap_grc_data() # unwrap phase. returns time and y values samp_rate = round(len(data[1]) / max(data[0])) dataextract = dataextractor.DataExtractor(data[0], data[1], samp_rate) cog_res = dataread.read_cognitive_load_study(ident + '-primary-extract.txt') end_epoch_time = dataread.get_end_time_cognitive_load_study() # end t extracted_br_features = dataextract.raw_windowing_breathing(30, 1) extracted_br_features['br_rate'] = np.array(extracted_br_features['br_rate'].rolling(6).mean()) extracted_br_features_roll_avg = extracted_br_features.loc[:, extracted_br_features.columns != 'times'].rolling( 6).mean() extracted_br_features_roll_avg['times'] = extracted_br_features['times'] extracted_br_features_roll_avg['br_ok'] = extracted_br_features['br_ok'] extracted_hr_features = dataextract.raw_windowing_heartrate(10, 1) extracted_hr_features = extracted_hr_features.drop(['hr_HRV_lf', 'hr_HRV_hf', 'hr_HRV_lf_hf'], axis=1) extracted_hr_features_roll_avg = extracted_hr_features.loc[:, extracted_hr_features.columns != 'times'].rolling( 10).mean() extracted_hr_features_roll_avg['times'] = extracted_hr_features['times'] extracted_hr_features_roll_avg['hr_ok'] = extracted_hr_features['hr_ok'] extracted_hr_features2 = dataextract.raw_windowing_heartrate(100, 1) # longer time to extract HRV frequency feat. extracted_hr_features2 = extracted_hr_features2[['hr_HRV_lf', 'hr_HRV_hf', 'hr_HRV_lf_hf', 'times']] extracted_hr_features2_roll_avg = extracted_hr_features2.loc[:, extracted_hr_features2.columns != 'times'].rolling( 10).mean() extracted_hr_features2_roll_avg['times'] = extracted_hr_features2['times'] all_features = extracted_br_features_roll_avg all_features = pd.merge(all_features, extracted_hr_features_roll_avg, on='times') all_features = pd.merge(all_features, extracted_hr_features2_roll_avg, on='times') task_timestamps = dataread.get_data_task_timestamps() relax_timestamps = dataread.get_relax_timestamps() bandread = bandreader.HeartRateBand(path + '_Hrates/', ident) band_data = bandread.load() band_data_time_start = bisect(band_data[0][:], end_epoch_time - data[0][-1] * 1000) band_data_time_stop = bisect(band_data[0][:], end_epoch_time) band_data = [band_data[0][band_data_time_start:band_data_time_stop], band_data[1][band_data_time_start:band_data_time_stop]] band_data_new__data = [(band_data[0] - band_data[0][0]) / 1000, band_data[1]] hr_data = extracted_hr_features_roll_avg[['times', 'hr_rate']] hr_data['times'] = hr_data['times'].astype(int) band_data = pd.DataFrame() band_data['times'] = band_data_new__data[0] band_data['times'] = band_data['times'].astype(int) band_data['band_rate'] = band_data_new__data[1] band_data = band_data.drop_duplicates(subset=['times']) together_data = pd.merge(hr_data, band_data, on='times') together_data = together_data.dropna() for i in range(len(all_features['times'])): find_in_hr_data = bisect(together_data['times'], all_features['times'][i]) all_features.ix[i, 'band_rate'] = together_data['band_rate'][find_in_hr_data] for i in range(len(cog_res)): all_feat_ind_task_start = bisect(all_features['times'], task_timestamps[i][0]) all_feat_ind_task_end = bisect(all_features['times'], task_timestamps[i][1]) for j in cog_res.columns: all_features.ix[all_feat_ind_task_start:all_feat_ind_task_end, j] = cog_res.iloc[i][j] if cog_res.iloc[i][j] == 'GC' or cog_res.iloc[i][j] == 'PT': all_features.ix[all_feat_ind_task_start:all_feat_ind_task_end, 'keyboard_task'] = True elif cog_res.iloc[i][j] == 'HP' or cog_res.iloc[i][j] == 'FA' or cog_res.iloc[i][j] == 'NC' or \ cog_res.iloc[i][j] == 'SX': all_features.ix[all_feat_ind_task_start:all_feat_ind_task_end, 'keyboard_task'] = False for k in range(all_feat_ind_task_end - all_feat_ind_task_start + 1): all_features.ix[k + all_feat_ind_task_start, 'on_task_or_break_index'] = k for k in range(all_feat_ind_task_end - all_feat_ind_task_start, -1, -1): all_features.ix[all_feat_ind_task_end - k, 'on_task_or_break_index_down'] = k all_features.ix[all_feat_ind_task_start:all_feat_ind_task_end, 'on_task'] = True for i in range(len(relax_timestamps)): all_feat_ind_task_start = bisect(all_features['times'], relax_timestamps[i][0]) all_feat_ind_task_end = bisect(all_features['times'], relax_timestamps[i][1]) new_end = all_feat_ind_task_end + 30 # if i==0: # continue for k in range(all_feat_ind_task_end - all_feat_ind_task_start + 1): all_features.ix[k + all_feat_ind_task_start, 'on_task_or_break_index'] = k all_features.ix[k + all_feat_ind_task_start, 'consecutive_break'] = i for k in range(new_end - all_feat_ind_task_start + 1): all_features.ix[k + all_feat_ind_task_start, 'on_break_and_after_index'] = k if k <= 15: all_features.ix[k + all_feat_ind_task_start, 'engagement_increase'] = False elif k <= 30: all_features.ix[k + all_feat_ind_task_start, 'engagement_increase'] = np.nan else: all_features.ix[k + all_feat_ind_task_start, 'engagement_increase'] = True for k in range(all_feat_ind_task_end - all_feat_ind_task_start, -1, -1): all_features.ix[all_feat_ind_task_end - k, 'on_task_or_break_index_down'] = k all_features.ix[all_feat_ind_task_start:all_feat_ind_task_end, 'on_task'] = False all_features['person_id'] = cog_res['person_id'][0] all_features.to_csv(path_or_buf=path + ident + '/' + ident + '-data.csv', index=False)
# -*-coding:utf-8 -*- # @Author:king # @time:2020/2/24 11:05 # @File:tiaojianlvbo.py # @Software:PyCharm # 条件滤波 import datareader x, y, z = datareader.DataReader() with open("tiaojian.txt", 'w') as file2write: for i in range(len(x)): if int(x[i]) > -3000 and int(x[i]) < 8000 and int(y[i]) > 4000 and int(y[i]) < 10000 and int(z[i]) > 40 and int( z[i]) < 2000: file2write.write(str(x[i]) + " " + str(y[i]) + " " + str(z[i]) + "\n")
def main(argv=None): assert not (argv is None), "The project path must be provided." # set train path train_input_path = argv[0] + '/data/train/noisy' train_output_path = argv[0] + '/data/train/clean' norm_path = argv[0] + '/data/train/norm' # set valid path valid_input_path = argv[0] + '/data/valid/noisy' valid_output_path = argv[0] + '/data/valid/clean' logs_dir = argv[1] # Graph Part # print("Graph initialization...") global_step = tf.Variable(0, trainable=False) with tf.device(config.device): with tf.variable_scope("model", reuse=None): m_train = trnmodel.Model(is_training=True, global_step=global_step) with tf.variable_scope("model", reuse=True): m_valid = trnmodel.Model(is_training=False, global_step=global_step) print("Done") # Summary Part # tensor_names = [n.name for n in tf.get_default_graph().as_graph_def().node] with open(logs_dir + "/tensor_names.txt", 'w') as f: for t_name in tensor_names: f.write("%s\n" % str(t_name)) print("Setting up summary op...") writer = SummaryWriter(log_dir=logs_dir + '/summary') print("Done") # Model Save Part # print("Setting up Saver...") saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(logs_dir) print("Done") # Session Part # sess_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) sess_config.gpu_options.allow_growth = True sess = tf.Session(config=sess_config) if ckpt and ckpt.model_checkpoint_path: # model restore print("Model restored...") saver.restore(sess, ckpt.model_checkpoint_path) print("Done") else: sess.run(tf.global_variables_initializer()) # if the checkpoint doesn't exist, do initialization # datareader initialization train_dr = dr.DataReader(train_input_path, train_output_path, norm_path, dist_num=config.dist_num, is_training=True, is_shuffle=False) valid_path = {'valid_input_path': valid_input_path, 'valid_output_path': valid_output_path, 'norm_path': norm_path} train_path = {'train_input_path': train_input_path, 'train_output_path': train_output_path, 'norm_path': norm_path} for itr in range(config.max_epoch): start_time = time.time() train_inputs, train_labels = train_dr.next_batch(config.batch_size) feed_dict = {m_train.inputs: train_inputs, m_train.labels: train_labels, m_train.keep_prob: config.keep_prob} sess.run(m_train.train_op, feed_dict=feed_dict) elapsed_time = time.time() - start_time # print("time_per_step:%.4f" % elapsed_time) if itr % 5 == 0 and itr >= 0: train_cost, train_lr = sess.run([m_train.cost, m_train.lr], feed_dict=feed_dict) # print("Step: %d, train_cost: %.4f, train_accuracy=%4.4f, train_time=%.4f" # % (itr, train_cost, train_accuracy * 100, el_tim)) print("Step: %d, train_cost: %.4f, learning_rate: %.7f" % (itr, train_cost, train_lr)) writer.add_scalars('training_procedure', {'train': train_cost}, itr) if itr % config.val_step == 0 and itr > 0: saver.save(sess, logs_dir + "/model.ckpt", itr) # model save print('validation start!') valid_cost = do_validation(m_valid, sess, valid_path) print("valid_cost: %.4f" % valid_cost) writer.add_scalars('training_procedure', {'train': train_cost, 'valid': valid_cost}, itr) if itr % config.summary_step == 0 and itr > 0: if itr == config.summary_step: train_summary = Summary(train_path, logs_dir, name='train') valid_summary = Summary(valid_path, logs_dir, name='valid') train_summary.do_summary(m_valid, sess, itr) valid_summary.do_summary(m_valid, sess, itr) else: train_summary.do_summary(m_valid, sess, itr) valid_summary.do_summary(m_valid, sess, itr) writer.close()
counter = 0 if counter == optimizing_parameter: backtester.buy(percent, i) elif positions > 0: for previous_value in range(1, 2 * optimizing_parameter + 1): if current_close > self.Closes[i - previous_value]: counter += 1 else: counter = 0 if counter == optimizing_parameter: backtester.sell(i) if __name__ == '__main__': start = '2021-03-02' end = '2021-04-09' symbol = 'DOGEUSDT' dates = (start, end) BTC = dr.DataReader(symbol, 'binance', dates, tick='1h') Strat = ExampleStrategy(BTC.Closes, BTC.Dates, symbol) # Strat.BackTester.runner() optimize_range = range(1, 10) for i in optimize_range: Strat.N = i Strat.BackTester.runner() plt.scatter(optimize_range, Strat.BackTester.Gains, marker='x') plt.plot(optimize_range, Strat.BackTester.Gains, lw=.5) Strat.BackTester.optimizer()
print("Done") # Model Save Part # print("Setting up Saver...") saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(logs_dir) print("Done") # Session Part # sess_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) sess_config.gpu_options.allow_growth = True sess = tf.Session(config=sess_config) if ckpt and ckpt.model_checkpoint_path: # model restore print("Model restored...") print(logs_dir + ckpt_name) saver.restore(sess, logs_dir + ckpt_name) print("Done") else: sess.run(tf.global_variables_initializer() ) # if the checkpoint doesn't exist, do initialization test_dataset = dr.DataReader(test_in, test_out, max_len=max_len, is_shuffle=False) acc = utils.evaluation_last(m_valid, sess, test_dataset) print('test_acc = %f' % acc)
def do_summary(self, m_summary, sess, itr): valid_path = self.valid_path clean_speech = self.clean_speech clean_speech = utils.identity_trans(clean_speech) noisy_speech = self.noisy_speech noisy_speech = utils.identity_trans(noisy_speech) temp_dir = self.temp_dir name = self.name logs_dir = self.logs_dir writer = SummaryWriter(log_dir=self.logs_dir + '/summary') summary_dr = dr.DataReader(temp_dir, '', valid_path["norm_path"], dist_num=config.dist_num, is_training=False, is_shuffle=False) pred = [] while True: summary_inputs, summary_labels = summary_dr.next_batch(config.batch_size) feed_dict = {m_summary.inputs: summary_inputs, m_summary.labels: summary_labels, m_summary.keep_prob: 1.0} pred_temp = sess.run(m_summary.pred, feed_dict=feed_dict) pred.append(pred_temp) if summary_dr.file_change_checker(): phase = summary_dr.phase[0] lpsd = np.expand_dims( np.reshape(np.concatenate(pred, axis=0), [-1, config.freq_size])[0:phase.shape[0], :], axis=2) mean, std = summary_dr.norm_process(valid_path["norm_path"] + '/norm_noisy.mat') lpsd = np.squeeze((lpsd * std * 1.18) + mean) # denorm recon_speech = utils.get_recon(np.transpose(lpsd, (1, 0)), np.transpose(phase, (1, 0)), win_size=config.win_size, win_step=config.win_step, fs=config.fs) # plt.plot(recon_speech) # plt.show() # lab = np.reshape(np.asarray(lab), [-1, 1]) summary_dr.reader_initialize() break # write summary if itr == config.summary_step: writer.close() self.noisy_measure = utils.se_eval(clean_speech, np.squeeze(noisy_speech), float(config.fs)) summary_fname = tf.summary.text(name + '_filename', tf.convert_to_tensor(self.noisy_dir)) if name == 'train': config_str = "<br>sampling frequency: %d</br>" \ "<br>window step: %d ms</br>" \ "<br>window size: %d ms</br>" \ "<br>fft size: %d</br>" \ "<br>learning rate: %f</br><br>learning rate decay: %.4f</br><br>learning" \ " rate decay frequency: %.4d</br>" \ "<br>dropout rate: %.4f</br><br>max epoch:" \ " %.4e</br><br>batch size: %d</br><br>model type: %s</br>"\ % (config.fs, (config.win_step/config.fs*1000), (config.win_size/config.fs*1000), config.nfft, config.lr, config.lrDecayRate, config.lrDecayFreq, config.keep_prob, config.max_epoch, config.batch_size, config.mode) summary_config = tf.summary.text(name + '_configuration', tf.convert_to_tensor(config_str)) code_list = [] read_flag = False with open('./lib/trnmodel.py', 'r') as f: while True: line = f.readline() if "def inference(self, inputs):" in line: read_flag = True if "return fm" in line: code_list.append('<br>' + line.replace('\n', '') + '</br>') break if read_flag: code_list.append('<br>' + line.replace('\n', '') + '</br>') code_list = "<pre>" + "".join(code_list) + "</pre>" summary_model = tf.summary.text('train_model', tf.convert_to_tensor(code_list)) summary_op = tf.summary.merge([summary_fname, summary_config, summary_model]) else: summary_op = tf.summary.merge([summary_fname]) with tf.Session() as sess: summary_writer = tf.summary.FileWriter(logs_dir + '/summary/text') text = sess.run(summary_op) summary_writer.add_summary(text, 1) summary_writer.close() writer = SummaryWriter(log_dir=logs_dir + '/summary') writer.add_audio(name + '_audio_ref' + '/clean', clean_speech /np.max(np.abs(clean_speech)), itr, sample_rate=config.fs) writer.add_audio(name + '_audio_ref' + '/noisy', noisy_speech /np.max(np.abs(noisy_speech)), itr, sample_rate=config.fs) clean_S = get_spectrogram(clean_speech) noisy_S = get_spectrogram(noisy_speech) writer.add_image(name + '_spectrogram_ref' + '/clean', clean_S, itr) # image_shape = (C, H, W) writer.add_image(name + '_spectrogram_ref' + '/noisy', noisy_S, itr) # image_shape = (C, H, W) enhanced_measure = utils.se_eval(clean_speech, recon_speech, float(config.fs)) writer.add_scalars(name + '_speech_quality' + '/pesq', {'enhanced': enhanced_measure['pesq'], 'ref': self.noisy_measure['pesq']}, itr) writer.add_scalars(name + '_speech_quality' + '/stoi', {'enhanced': enhanced_measure['stoi'], 'ref': self.noisy_measure['stoi']}, itr) writer.add_scalars(name + '_speech_quality' + '/lsd', {'enhanced': enhanced_measure['lsd'], 'ref': self.noisy_measure['lsd']}, itr) writer.add_scalars(name + '_speech_quality' + '/ssnr', {'enhanced': enhanced_measure['ssnr'], 'ref': self.noisy_measure['ssnr']}, itr) writer.add_audio(name + '_audio_enhanced' + '/enhanced', recon_speech/np.max(np.abs(recon_speech)), itr, sample_rate=config.fs) enhanced_S = get_spectrogram(recon_speech) writer.add_image(name + '_spectrogram_enhanced' + '/enhanced', enhanced_S, itr) # image_shape = (C, H, W) writer.close()
# self.resnet = mobilenet.MobileFaceHead([2, 8, 16, 4]) self.resnet = Inception.Inception3(512) self.classifier = losspart.MarginalCosineLayer(num_classes) def forward(self, x, label): feat = self.resnet(x) # feat = tf.nn.dropout(feat, 0.4) logits = self.classifier(feat, label, 1.0, 0.5, 0.0) logits = logits * 64 return logits BSIZE = 2 print(BSIZE) EPOCH = 30 data_reader = datareader.DataReader('imglist_iccv.txt', BSIZE) tf.keras.backend.set_learning_phase(True) def lr_decay(step): lr = 0.1 step = step / 30000 step = tf.math.floor(step) step = tf.math.pow(0.1, step) lr = lr * step return lr def grad_loss(x, model): data, label = x with tf.GradientTape() as tape: