def spearman_r(r1, r2):
r = 0
for i in range(len(r1.acts)):
r1copy = deepcopy(r1.acts[i])
r2copy = deepcopy(r2.acts[i])
maxes1 = maxindex(r1copy, 5)
maxes2 = maxindex(r2copy, 5)
mask1 = np.ones(len(r1copy), np.bool)
mask1[ints(maxes1)] = 0
# mask2 = np.ones(len(r2copy), np.bool)
# mask2[ints(maxes2)] = 0
# mask = np.bitwise_and(mask1, mask2)
# r1copy[mask1] = np.nan
# r2copy[mask1] = np.nan
# r1copy[mask1] = None
# r2copy[mask1] = None
r1copy = r1copy[ints(maxes1)]
r2copy = r2copy[ints(maxes1)]
# zs = np.bitwise_or(r1copy == 0, r2copy == 0)
if all(r1copy == 0) or all(r2copy == 0):
r += np.nan
else:
r += scipy.stats.spearmanr(r1copy, r2copy, nan_policy='omit')[0]
return r
def rpeak_detect(self, ecg_raw, Fs, ecg_flt, ecg_raw_nopl_high):
# just fixpeaks
qrs = arr(self.MANUAL_INPUT_FILE['heartbeatevents']['py'][0]).flatten()[0].flatten()
qrs = ints(qrs)
qrs = qrs[qrs >= self.rawslice.start]
qrs = qrs[qrs < self.rawslice.stop]
if vers(self.version) >= vers(1):
qrs = find_local_maxima(
qrs,
ecg_flt,
AUTO=True,
CROP_FIRST_LAST=True
)
return qrs
def gen_main(FLAGS, _IMAGES_FOLDER, HUMAN_IMAGE_FOLDER):
log('in gen!')
_IMAGES_FOLDER.clearIfExists()
HUMAN_IMAGE_FOLDER.clearIfExists()
gen_cfg = FLAGS.cfg_cfg['gen_cfg']
# these numbers might be lower now that I'm excluding images that aren't squares
# nevermind. I think trying to only take squares didn't work
cats = ['Egyptian cat', # >=200
'Siamese cat', # 196
'Persian cat', # >=200
'tiger cat', # 182
'tabby cat'] # >=100
dogs = [
'Afghan hound', # >=200
'basset hound', # >=200
'beagle', # 198
'bloodhound', # 199
'bluetick' # >=100
]
classes = cats + dogs
not_trained = ['tabby cat', 'bluetick']
for d in dogs:
nnstate.reduced_map[d] = 'dog'
for c in cats:
nnstate.reduced_map[c] = 'cat'
if FLAGS.salience:
log('in gen salience!')
root = DATA_FOLDER.resolve('ImageNet/output_tf')
filenames = root.glob('train*').map(lambda x: x.abspath).tolist() # validation
import tensorflow as tf
ds = tf.data.TFRecordDataset(filenames)
# for subroot in root:
# for imgfile in subroot:
image_feature_description = {
'image/height' : tf.io.FixedLenFeature([], tf.int64),
'image/width' : tf.io.FixedLenFeature([], tf.int64),
'image/colorspace' : tf.io.FixedLenFeature([], tf.string),
'image/channels' : tf.io.FixedLenFeature([], tf.int64),
'image/class/label' : tf.io.FixedLenFeature([], tf.int64),
'image/class/synset': tf.io.FixedLenFeature([], tf.string),
'image/class/text' : tf.io.FixedLenFeature([], tf.string),
# 'image/object/bbox/xmin' : tf.io.FixedLenFeature([], tf.float32),
# 'image/object/bbox/xmax' : tf.io.FixedLenFeature([], tf.float32),
# 'image/object/bbox/ymin' : tf.io.FixedLenFeature([], tf.float32),
# 'image/object/bbox/ymax' : tf.io.FixedLenFeature([], tf.float32),
# 'image/object/bbox/label': tf.io.FixedLenFeature([], tf.int64),
'image/format' : tf.io.FixedLenFeature([], tf.string),
'image/filename' : tf.io.FixedLenFeature([], tf.string),
'image/encoded' : tf.io.FixedLenFeature([], tf.string),
}
# imap = {}
# current_i = -1
# def input_gen():
log('looping imagenet')
_IMAGES_FOLDER[f'Training/{FLAGS.REGEN_NTRAIN}'].mkdirs()
_IMAGES_FOLDER['Testing'].mkdirs()
# classes = [
# 'barn spider',
# 'garden spider',
# 'black widow',
# 'wolf spider',
# 'black and gold garden spider',
#
# 'emmet' ,#ant
# 'grasshopper',
# 'cricket',
# 'stick insect',
# 'cockroach'
# ]
class_count = {cn: 0 for cn in classes}
for i, raw_record in enum(ds):
example = tf.io.parse_single_example(raw_record, image_feature_description)
# r[f'tf']['y_true'][i] = example['image/class/label'].numpy()
# return tf.image.decode_jpeg(example['image/encoded'], channels=3).numpy()
# if example['image/height'] != example['image/width']:
# continue
if i % 100 == 0:
log(f'on image {i}')
classname = utf_decode(example['image/class/text'].numpy())
for cn in classes:
if (cn in classname) and (
class_count[cn] < (FLAGS.REGEN_NTRAIN if cn in not_trained else (FLAGS.REGEN_NTRAIN * 2))):
log(f'saving {cn} {class_count[cn] + 1}')
rrr = tf.image.decode_jpeg(example['image/encoded'], channels=3).numpy()
if class_count[cn] < FLAGS.REGEN_NTRAIN:
_IMAGES_FOLDER['Testing'][cn][f'{i}.png'].save(rrr)
else:
_IMAGES_FOLDER[f'Training/{FLAGS.REGEN_NTRAIN}']['dog' if cn in dogs else 'cat'][
f'{i}.png'].save(rrr)
class_count[cn] += 1
break
break_all = True
for cn, cc in listitems(class_count):
if (cn in not_trained and cc != FLAGS.REGEN_NTRAIN) or (
cn not in not_trained and cc != (FLAGS.REGEN_NTRAIN * 2)):
break_all = False
if break_all:
break
# current_i = current_i + 1
# imap[i] = rrr
# yield rrr
# igen = input_gen()
# def get_input(index):
# # log(f'trying to get index {index}')
# # log(f'current indices range from {safemin(list(imap.keys()))} to {safemax(list(imap.keys()))}')
# if index not in imap:
# # log('coud not get it')
# next(igen)
# return get_input(index)
# else:
# # log('got it!')
# rr = imap[index]
# for k in list(imap.keys()):
# if k < index:
# del imap[k]
# return rr
# # for raw_record in ds:
# # example = tf.io.parse_single_example(raw_record, image_feature_description)
# # r[f'tf']['y_true'][index] = example['image/class/label'].numpy()
# # return tf.image.decode_jpeg(example['image/encoded'], channels=3).numpy()
# # yield example
# # y_true = []
# # ifs_for_labels = input_files()
# # for i in range(SANITY_SET.num):
# # y_true.append(next(ifs_for_labels)['image/class/label'].numpy())
# # r[f'tf']['y_true'] = y_true
# # def input_file_raws():
# # gen = input_files()
# # for example in gen:
# # yield tf.image.decode_jpeg(example['image/encoded'], channels=3).numpy()
# # IN_files = input_file_raws()
# IN_files = get_input
# test_class_pairs = [
# pair for pair in chain(*[
# (
# SymAsymClassPair(n, False),
# SymAsymClassPair(n, True)
# ) for n in ints(np.linspace(0, 10, 6))
# ])
# ]
# class_pairs = [
# SymAsymClassPair(0, False),
# SymAsymClassPair(4, False)
# ]
# human_class_pairs = [
# SymAsymClassPair(0, False),
# SymAsymClassPair(2, False),
# SymAsymClassPair(4, False),
# SymAsymClassPair(6, False),
# SymAsymClassPair(8, False)
# ]
# gen_cfg = FLAGS.cfg_cfg['gen_cfg']
# gen_images(
# folder=HUMAN_IMAGE_FOLDER['TimePilot'],
# class_pairs=human_class_pairs,
# ims_per_class=10
# )
else:
test_class_pairs = [
pair for pair in chain(*[
(
SymAsymClassPair(n, False),
SymAsymClassPair(n, True)
) for n in ints(np.linspace(0, 10, 6))
])
]
class_pairs = [
SymAsymClassPair(0, False),
SymAsymClassPair(4, False)
]
human_class_pairs = [
SymAsymClassPair(0, False),
SymAsymClassPair(2, False),
SymAsymClassPair(4, False),
SymAsymClassPair(6, False),
SymAsymClassPair(8, False)
]
gen_images(
folder=HUMAN_IMAGE_FOLDER['TimePilot'],
class_pairs=human_class_pairs,
ims_per_class=10
)
gen_images(
folder=_IMAGES_FOLDER['RSA'],
class_pairs=test_class_pairs,
ims_per_class=10,
# ims_per_class=1
)
gen_images(
folder=_IMAGES_FOLDER['Testing'],
class_pairs=test_class_pairs,
ims_per_class=10,
# ims_per_class=500,
# ims_per_class=1
)
# for n in (25, 50, 100, 150, 200, 1000):
for n in (10,):
gen_images(
folder=_IMAGES_FOLDER['Training'][n],
class_pairs=class_pairs,
ims_per_class=n
)
log('doing thing with _temp_ims')
with mlib.file.TempFolder('_temp_ims') as temp:
log('temp_ims_1')
if temp.exists and temp.isdir:
temp.clear()
log('temp_ims_2')
temp.mkdirs()
log('temp_ims_3')
[_IMAGES_FOLDER.copy_to(temp[f'gpu{i + 1}']) for i in range(gen_cfg['num_gpus'])]
log('temp_ims_4')
_IMAGES_FOLDER.clear()
log('temp_ims_5')
[temp[f'gpu{i + 1}'].moveinto(_IMAGES_FOLDER) for i in range(gen_cfg['num_gpus'])]
log('temp_ims_6')
log('finished thing with _temp_ims')
nn_init_fun.NRC_IS_FINISHED() # must be invoked this way since value of function changes
lmt = local_lmt
if (ecg_flt[n] > lmt) and (n + 1 < sz):
r_peak_indices.append(
n
) # this n passed the threshod, so add it to the stored r peaks
n += STEP # advance 350 ms and start scanning the next cardiac cycle
else:
n += STEP_SMALL # jump 10 ms forward
# DOC: STOP
prog.tick(n)
# DOC: START
# DOC: STOP
fig_indices_indices = ints(
np.nonzero(arr(r_peak_indices) >= _PLOT_SLICE.start)[0])
fig_indices_indices_small = ints(
np.nonzero(
bitwise_and(
arr(r_peak_indices) >= _PLOT_SLICE_SMALL.start,
arr(r_peak_indices) <= _PLOT_SLICE_SMALL.stop))[0])
r_peak_indices_fig = arr(r_peak_indices)[fig_indices_indices]
r_peak_indices_small_fig = arr(
r_peak_indices)[fig_indices_indices_small]
# DOC: START
DOC.fig([[
MultiPlot(
PlotData(unitscale_demean(ecg_flt[_PLOT_SLICE]),
hideYTicks=True,
item_type='line',
title='First Detected Peaks',
def rPeaks(self):
if self.algmode == 'LOAD':
peaks = self._loadpeaks()
else:
peaks = self._rpeak_detect()
return ints(peaks)
def color(cls, *rgb):
rgb = arr(rgb) * 128
# if len(rgb.shape) == 3:
# if rgb.shape[0] == 1:
# rgb = rgb[0]
return '#%02x%02x%02x' % tuple(ints(rgb).tolist())