def spearman_r(r1, r2): r = 0 for i in range(len(r1.acts)): r1copy = deepcopy(r1.acts[i]) r2copy = deepcopy(r2.acts[i]) maxes1 = maxindex(r1copy, 5) maxes2 = maxindex(r2copy, 5) mask1 = np.ones(len(r1copy), np.bool) mask1[ints(maxes1)] = 0 # mask2 = np.ones(len(r2copy), np.bool) # mask2[ints(maxes2)] = 0 # mask = np.bitwise_and(mask1, mask2) # r1copy[mask1] = np.nan # r2copy[mask1] = np.nan # r1copy[mask1] = None # r2copy[mask1] = None r1copy = r1copy[ints(maxes1)] r2copy = r2copy[ints(maxes1)] # zs = np.bitwise_or(r1copy == 0, r2copy == 0) if all(r1copy == 0) or all(r2copy == 0): r += np.nan else: r += scipy.stats.spearmanr(r1copy, r2copy, nan_policy='omit')[0] return r
def rpeak_detect(self, ecg_raw, Fs, ecg_flt, ecg_raw_nopl_high): # just fixpeaks qrs = arr(self.MANUAL_INPUT_FILE['heartbeatevents']['py'][0]).flatten()[0].flatten() qrs = ints(qrs) qrs = qrs[qrs >= self.rawslice.start] qrs = qrs[qrs < self.rawslice.stop] if vers(self.version) >= vers(1): qrs = find_local_maxima( qrs, ecg_flt, AUTO=True, CROP_FIRST_LAST=True ) return qrs
def gen_main(FLAGS, _IMAGES_FOLDER, HUMAN_IMAGE_FOLDER): log('in gen!') _IMAGES_FOLDER.clearIfExists() HUMAN_IMAGE_FOLDER.clearIfExists() gen_cfg = FLAGS.cfg_cfg['gen_cfg'] # these numbers might be lower now that I'm excluding images that aren't squares # nevermind. I think trying to only take squares didn't work cats = ['Egyptian cat', # >=200 'Siamese cat', # 196 'Persian cat', # >=200 'tiger cat', # 182 'tabby cat'] # >=100 dogs = [ 'Afghan hound', # >=200 'basset hound', # >=200 'beagle', # 198 'bloodhound', # 199 'bluetick' # >=100 ] classes = cats + dogs not_trained = ['tabby cat', 'bluetick'] for d in dogs: nnstate.reduced_map[d] = 'dog' for c in cats: nnstate.reduced_map[c] = 'cat' if FLAGS.salience: log('in gen salience!') root = DATA_FOLDER.resolve('ImageNet/output_tf') filenames = root.glob('train*').map(lambda x: x.abspath).tolist() # validation import tensorflow as tf ds = tf.data.TFRecordDataset(filenames) # for subroot in root: # for imgfile in subroot: image_feature_description = { 'image/height' : tf.io.FixedLenFeature([], tf.int64), 'image/width' : tf.io.FixedLenFeature([], tf.int64), 'image/colorspace' : tf.io.FixedLenFeature([], tf.string), 'image/channels' : tf.io.FixedLenFeature([], tf.int64), 'image/class/label' : tf.io.FixedLenFeature([], tf.int64), 'image/class/synset': tf.io.FixedLenFeature([], tf.string), 'image/class/text' : tf.io.FixedLenFeature([], tf.string), # 'image/object/bbox/xmin' : tf.io.FixedLenFeature([], tf.float32), # 'image/object/bbox/xmax' : tf.io.FixedLenFeature([], tf.float32), # 'image/object/bbox/ymin' : tf.io.FixedLenFeature([], tf.float32), # 'image/object/bbox/ymax' : tf.io.FixedLenFeature([], tf.float32), # 'image/object/bbox/label': tf.io.FixedLenFeature([], tf.int64), 'image/format' : tf.io.FixedLenFeature([], tf.string), 'image/filename' : tf.io.FixedLenFeature([], tf.string), 'image/encoded' : tf.io.FixedLenFeature([], tf.string), } # imap = {} # current_i = -1 # def input_gen(): log('looping imagenet') _IMAGES_FOLDER[f'Training/{FLAGS.REGEN_NTRAIN}'].mkdirs() _IMAGES_FOLDER['Testing'].mkdirs() # classes = [ # 'barn spider', # 'garden spider', # 'black widow', # 'wolf spider', # 'black and gold garden spider', # # 'emmet' ,#ant # 'grasshopper', # 'cricket', # 'stick insect', # 'cockroach' # ] class_count = {cn: 0 for cn in classes} for i, raw_record in enum(ds): example = tf.io.parse_single_example(raw_record, image_feature_description) # r[f'tf']['y_true'][i] = example['image/class/label'].numpy() # return tf.image.decode_jpeg(example['image/encoded'], channels=3).numpy() # if example['image/height'] != example['image/width']: # continue if i % 100 == 0: log(f'on image {i}') classname = utf_decode(example['image/class/text'].numpy()) for cn in classes: if (cn in classname) and ( class_count[cn] < (FLAGS.REGEN_NTRAIN if cn in not_trained else (FLAGS.REGEN_NTRAIN * 2))): log(f'saving {cn} {class_count[cn] + 1}') rrr = tf.image.decode_jpeg(example['image/encoded'], channels=3).numpy() if class_count[cn] < FLAGS.REGEN_NTRAIN: _IMAGES_FOLDER['Testing'][cn][f'{i}.png'].save(rrr) else: _IMAGES_FOLDER[f'Training/{FLAGS.REGEN_NTRAIN}']['dog' if cn in dogs else 'cat'][ f'{i}.png'].save(rrr) class_count[cn] += 1 break break_all = True for cn, cc in listitems(class_count): if (cn in not_trained and cc != FLAGS.REGEN_NTRAIN) or ( cn not in not_trained and cc != (FLAGS.REGEN_NTRAIN * 2)): break_all = False if break_all: break # current_i = current_i + 1 # imap[i] = rrr # yield rrr # igen = input_gen() # def get_input(index): # # log(f'trying to get index {index}') # # log(f'current indices range from {safemin(list(imap.keys()))} to {safemax(list(imap.keys()))}') # if index not in imap: # # log('coud not get it') # next(igen) # return get_input(index) # else: # # log('got it!') # rr = imap[index] # for k in list(imap.keys()): # if k < index: # del imap[k] # return rr # # for raw_record in ds: # # example = tf.io.parse_single_example(raw_record, image_feature_description) # # r[f'tf']['y_true'][index] = example['image/class/label'].numpy() # # return tf.image.decode_jpeg(example['image/encoded'], channels=3).numpy() # # yield example # # y_true = [] # # ifs_for_labels = input_files() # # for i in range(SANITY_SET.num): # # y_true.append(next(ifs_for_labels)['image/class/label'].numpy()) # # r[f'tf']['y_true'] = y_true # # def input_file_raws(): # # gen = input_files() # # for example in gen: # # yield tf.image.decode_jpeg(example['image/encoded'], channels=3).numpy() # # IN_files = input_file_raws() # IN_files = get_input # test_class_pairs = [ # pair for pair in chain(*[ # ( # SymAsymClassPair(n, False), # SymAsymClassPair(n, True) # ) for n in ints(np.linspace(0, 10, 6)) # ]) # ] # class_pairs = [ # SymAsymClassPair(0, False), # SymAsymClassPair(4, False) # ] # human_class_pairs = [ # SymAsymClassPair(0, False), # SymAsymClassPair(2, False), # SymAsymClassPair(4, False), # SymAsymClassPair(6, False), # SymAsymClassPair(8, False) # ] # gen_cfg = FLAGS.cfg_cfg['gen_cfg'] # gen_images( # folder=HUMAN_IMAGE_FOLDER['TimePilot'], # class_pairs=human_class_pairs, # ims_per_class=10 # ) else: test_class_pairs = [ pair for pair in chain(*[ ( SymAsymClassPair(n, False), SymAsymClassPair(n, True) ) for n in ints(np.linspace(0, 10, 6)) ]) ] class_pairs = [ SymAsymClassPair(0, False), SymAsymClassPair(4, False) ] human_class_pairs = [ SymAsymClassPair(0, False), SymAsymClassPair(2, False), SymAsymClassPair(4, False), SymAsymClassPair(6, False), SymAsymClassPair(8, False) ] gen_images( folder=HUMAN_IMAGE_FOLDER['TimePilot'], class_pairs=human_class_pairs, ims_per_class=10 ) gen_images( folder=_IMAGES_FOLDER['RSA'], class_pairs=test_class_pairs, ims_per_class=10, # ims_per_class=1 ) gen_images( folder=_IMAGES_FOLDER['Testing'], class_pairs=test_class_pairs, ims_per_class=10, # ims_per_class=500, # ims_per_class=1 ) # for n in (25, 50, 100, 150, 200, 1000): for n in (10,): gen_images( folder=_IMAGES_FOLDER['Training'][n], class_pairs=class_pairs, ims_per_class=n ) log('doing thing with _temp_ims') with mlib.file.TempFolder('_temp_ims') as temp: log('temp_ims_1') if temp.exists and temp.isdir: temp.clear() log('temp_ims_2') temp.mkdirs() log('temp_ims_3') [_IMAGES_FOLDER.copy_to(temp[f'gpu{i + 1}']) for i in range(gen_cfg['num_gpus'])] log('temp_ims_4') _IMAGES_FOLDER.clear() log('temp_ims_5') [temp[f'gpu{i + 1}'].moveinto(_IMAGES_FOLDER) for i in range(gen_cfg['num_gpus'])] log('temp_ims_6') log('finished thing with _temp_ims') nn_init_fun.NRC_IS_FINISHED() # must be invoked this way since value of function changes
lmt = local_lmt if (ecg_flt[n] > lmt) and (n + 1 < sz): r_peak_indices.append( n ) # this n passed the threshod, so add it to the stored r peaks n += STEP # advance 350 ms and start scanning the next cardiac cycle else: n += STEP_SMALL # jump 10 ms forward # DOC: STOP prog.tick(n) # DOC: START # DOC: STOP fig_indices_indices = ints( np.nonzero(arr(r_peak_indices) >= _PLOT_SLICE.start)[0]) fig_indices_indices_small = ints( np.nonzero( bitwise_and( arr(r_peak_indices) >= _PLOT_SLICE_SMALL.start, arr(r_peak_indices) <= _PLOT_SLICE_SMALL.stop))[0]) r_peak_indices_fig = arr(r_peak_indices)[fig_indices_indices] r_peak_indices_small_fig = arr( r_peak_indices)[fig_indices_indices_small] # DOC: START DOC.fig([[ MultiPlot( PlotData(unitscale_demean(ecg_flt[_PLOT_SLICE]), hideYTicks=True, item_type='line', title='First Detected Peaks',
def rPeaks(self): if self.algmode == 'LOAD': peaks = self._loadpeaks() else: peaks = self._rpeak_detect() return ints(peaks)
def color(cls, *rgb): rgb = arr(rgb) * 128 # if len(rgb.shape) == 3: # if rgb.shape[0] == 1: # rgb = rgb[0] return '#%02x%02x%02x' % tuple(ints(rgb).tolist())