def count_zeros():
file_base = sys.argv[1]
with open('zero_channels.txt', 'w') as out_dat:
out_dat.write('Proportion of samples at zero per event \n \n \n')
for ifile in sorted(gl(file_base+'*.h5')):
with tb.open_file(ifile) as data_in:
try:
sipmrwf = data_in.root.RD.sipmrwf
except tb.NoSuchNodeError:
print('The files do not contain raw data')
exit()
sens_ids = np.fromiter((ch[1] for ch in data_in.root.Sensors.DataSiPM), np.int)
evts = np.fromiter((ev[0] for ev in data_in.root.Run.events), np.int)
buffer_len = len(sipmrwf[0][0])
print('file: '+ifile)
for evt_no, evtrwf in zip(evts, sipmrwf):
out = str(evt_no)
indx = np.argwhere(sens_ids == 5010)[0][0]
prop_zero_5010 = 1 - np.count_nonzero(evtrwf[indx]) / buffer_len
out += ' ch 5010 '+str(prop_zero_5010)
indx = np.argwhere(sens_ids == 10010)[0][0]
prop_zero_10010 = 1 - np.count_nonzero(evtrwf[indx]) / buffer_len
out += ' ch 10010 '+str(prop_zero_10010)
indx = np.argwhere(sens_ids == 11010)[0][0]
prop_zero_11010 = 1 - np.count_nonzero(evtrwf[indx]) / buffer_len
out += ' ch 11010 '+str(prop_zero_11010)+'\n'
out_dat.write(out)
def _find_all_md(repo) -> []:
DEBUG("Looking for all readme...")
files = [
f for ext in _EXTENSIONS
for f in gl(os.path.join(repo, '**', ext), recursive=True)
]
return [i for i in files if "readme" in PATH.basename(i.lower())]
def read_data():
train_data_list = []
train_lable_list = []
test_data_list = []
test_lable_list = []
for class_index in range(class_num):
all_pics_path = gl(
os.path.join(train_image_path, '%d_*.png' % class_index))
random.shuffle(all_pics_path)
# nn = range(len(all_pics_path))
# np.random.seed(23)
# np.random.shuffle(nn)
# all_pics_path = all_pics_path[nn]
pics_num = len(all_pics_path)
train_data_list += all_pics_path[:int(pics_num * 0.8)]
test_data_list += all_pics_path[int(pics_num * 0.8):]
train_lable_list += [
class_index
for i in range(len(all_pics_path[:int(pics_num * 0.8)]))
]
test_lable_list += [
class_index
for i in range(len(all_pics_path[int(pics_num * 0.8):]))
]
assert len(train_lable_list) == len(train_data_list)
print len(train_lable_list)
assert len(test_data_list) == len(test_lable_list)
print len(test_data_list)
for index in range(len(train_data_list)):
temp_path = train_data_list[index]
print temp_path
if not os.path.exists(temp_path):
continue
temp_image = cv2.imread(temp_path)
if temp_image is None:
continue
with open('1206_train.txt', 'a') as f:
f.write('%s %d\n' % (temp_path, train_lable_list[index]))
for index in range(len(test_data_list)):
temp_path = test_data_list[index]
print temp_path
if not os.path.exists(temp_path):
continue
temp_image = cv2.imread(temp_path)
if temp_image is None:
continue
with open('1206_test.txt', 'a') as f:
f.write('%s %d\n' % (temp_path, test_lable_list[index]))
import os
import warnings
from glob import glob as gl
import cv2
import numpy as np
from keras.datasets import mnist, cifar10, cifar100
from keras.preprocessing.image import ImageDataGenerator
from scipy.io import loadmat
glob = lambda k: sorted(gl(k))
pjoin = os.path.join
if 'DATASET_ROOT' not in os.environ:
warnings.warn("$DATASET_ROOT is not set, defaulting to author's path.")
os.environ['DATASET_ROOT'] = '/media/braf3002/hdd2'
DATASET_ROOT = os.environ['DATASET_ROOT']
assert os.path.exists(DATASET_ROOT), '$DATASET_ROOT is not a valid path!'
# Paths for all datasets
notMNIST = pjoin(DATASET_ROOT, 'Datasets/notMNIST_small')
INRIA = pjoin(DATASET_ROOT, 'Datasets/INRIA/INRIAPerson')
SEEFOOD = pjoin(DATASET_ROOT, 'Datasets/seefood/seefood')
SVHN = pjoin(DATASET_ROOT, 'Datasets/SVHN/')
STL10 = pjoin(DATASET_ROOT, 'Datasets/stl10/')
COMP_CARS = pjoin(DATASET_ROOT, 'Datasets/compCars/data')
MIOTCD = pjoin(DATASET_ROOT, 'Datasets/mio_tcd_classification')
CHINA = pjoin(DATASET_ROOT, 'Datasets/ChinaSet_AllFiles')
import numpy as np
import cv2
from glob import glob as gl
import os
import shutil
from __config__ import *
import json
images = gl(path_labeled + '/*/*.jpg')
def get_point(img, color):
global line1
lower_green = np.array([50, 230, 230])
upper_green = np.array([70, 255, 255])
lower_blue = np.array([110, 220, 180])
upper_blue = np.array([130, 255, 255])
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
mask = cv2.inRange(hsv, lower_green, upper_green)
if color == 'blue':
mask = cv2.inRange(hsv, lower_blue, upper_blue)
contours, hierarchy = cv2.findContours(mask, cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
point = []
center = [WIDTH // 2, 0]
for cnt in contours:
area = cv2.contourArea(cnt)
if area > 10:
x, y, w, h = cv2.boundingRect(cnt)
def pos(s, remove_tag=[], c_tag=[], _opt = '-tip1+sw', _in = 'sample.in', _out = 'sample.out', thread=True):
try:
c_tag = c_tags
# -- 멀티 세션 돌릴때 파일이름 겹치면 오류발생함
if thread:
def _idGenerator():
return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6))
_in = _idGenerator()+'.in'
_out = _idGenerator()+'.out'
_i = gl('*.in')
if len(_i):
for i in _i:
os.remove(i)
_o = gl('*.out')
if len(_o):
for o in _o:
os.remove(o)
# 50 단어 마다 줄바꿈 ( 안하면 KLT에서 WARNING 발생 )
words = s.split()
s = [ ' '.join(words[i:i + 50]) for i in range(0, len(words), 50) ]
s = '\n'.join(s)
f = codecs.open(_in, 'w+', encoding='KSC5601')
f.write(s)
f.close()
command = ["kma.exe",_opt,_in,_out]
check_call(command, stdout=DEVNULL, stderr=STDOUT)
os.remove(_in) # 파일 지우기
f = codecs.open(_out, encoding='KSC5601')
tokend_text = f.read()
f.close()
os.remove(_out) # 파일 지우기
str_token = re.findall(pattern='\([\w ]+\)', string=tokend_text)
poses = list(map(_parse, str_token))
# -- 불용태그 제거
if len(remove_tag):
poses = [(w,t) for w,t in poses if t not in remove_tag ]
chunker = RegexpParser('JOSA:{<t|c><e>}')
chunks = chunker.parse(poses)
chunks = [chunk.leaves() if type(chunk) != tuple else chunk for chunk in chunks]
poses = []
for pos in chunks:
if type(pos) == list:
w1, t1 = pos[0]
jong, t2 = pos[1]
try:
chojung = decompose(w1)
w = compose(chojung[0], chojung[1], jong)
except:
w = w1+jong
if w1 == '하' and jong == '어':
w = '해'
pos = (w, t1+t2)
for org,cus in c_tag:
if org == pos:
pos = cus
poses.append(pos)
# 불용어 제거
stop_words = [('의','N'),('을','N'),('를','N'),('대한','N'),('인해','N'),('중','N'),('등','N')]
poses = [ pos for pos in poses if pos not in stop_words]
return poses
except:
return []
def compare_sims():
"""
Just a copy (more or less) of Paola's notebook for
simulation low level plots adjusted to compare simulation
with and without the low frequency noise.
"""
run_number = -4735
DataSiPM = DB.DataSiPM(run_number)
data_xs = DataSiPM.X.values
data_ys = DataSiPM.Y.values
## conf parameters
## n_rebin = 1
n_files = 10
drift_v = 1.
## Containers for plots ##
s1Q_NO = []
s1H_NO = []
s1W_NO = []
s1per_evt_NO = []
s1Q_O = []
s1H_O = []
s1W_O = []
s1per_evt_O = []
#maxPMTtot_ratio = []
s2per_evt_NO = []
s2Q_NO = []
s2H_NO = []
s2W_NO = []
s2per_evt_O = []
s2Q_O = []
s2H_O = []
s2W_O = []
no_iter = gl("/Users/laingandrew/Documents/NEXT/IC_stuff/simdats/noOsc_paola/Tl_DISK_CLOSE_TO_ANODE_7bar_noiseSiPM_pmaps.*.h5")
o_iter = gl("/Users/laingandrew/Documents/NEXT/IC_stuff/simdats/Tl_DISK_CLOSE_TO_ANODE_7bar_noiseSiPM_pmaps.*.h5")
for no_file, o_file in zip(no_iter, o_iter):
##for ifile in range(n_files):
##PATH_IN_NOOSC = "/Users/laingandrew/Documents/NEXT/IC_stuff/simdats/noOsc_paola/Tl_DISK_CLOSE_TO_ANODE_7bar_noiseSiPM_pmaps.{}.h5".format(ifile)
##PATH_IN_OSC = "/Users/laingandrew/Documents/NEXT/IC_stuff/simdats/Tl_DISK_CLOSE_TO_ANODE_7bar_noiseSiPM_pmaps.{}.h5".format(ifile)
print(no_file)
#pmapNO_dict = load_pmaps(no_file)
s1sNO, s2sNO, _, _, _ = load_pmaps_as_df(no_file)
print('NO pmaps got')
#pmapO_dict = load_pmaps(o_file)
s1sO, s2sO, _, _, _ = load_pmaps_as_df(o_file)
print('O pmaps got')
## event_numbers_NO, timestamps_NO = get_event_numbers_and_timestamps_from_file_name(PATH_IN_NOOSC)
## event_numbers_O, timestamps_O = get_event_numbers_and_timestamps_from_file_name(PATH_IN_OSC)
event_numbers_NO = s2sNO['event'].unique()
event_numbers_O = s2sO['event'].unique()
print(len(event_numbers_NO), len(event_numbers_NO))
for evtNO, evtO in zip(event_numbers_NO, event_numbers_O):
## pmapNO = pmapNO_dict.get(evtNO, None)
## pmapO = pmapO_dict.get(evtO, None)
s1s_NO = s1sNO[s1sNO['event']==evtNO]
s1s_O = s1sO[s1sO['event']==evtO]
s2s_NO = s2sNO[s2sNO['event']==evtNO]
s2s_O = s2sO[s2sO['event']==evtO]
#print(len(s1s_NO), len(s1s_O), len(s2s_NO), len(s2s_O))
#print('evtNO = ', evtNO, ' evtO = ', evtO,' pmaps got')
if len(s1s_NO) == 0 or len(s1s_O) == 0 or len(s2s_NO) == 0 or len(s2s_O) == 0:
continue
## if pmapNO:
## s1s_NO = pmapNO.s1s
## s2sraw_NO = pmapNO.s2s
## else:
## continue
## if pmapO:
## s1s_O = pmapO.s1s
## s2sraw_O = pmapO.s2s
## else:
## continue
#if not s1s_NO or not s2sraw_NO or not s1s_O or not s2sraw_O: continue
s1per_evt_NO.append(s1s_NO['peak'].nunique())
s1per_evt_O .append(s1s_O['peak'].nunique())
#print('n_s1_nosc = ', s1s_NO['peak'].nunique(), ', n_s1_osc = ', s1s_O['peak'].nunique())
#if s1s_NO['peak'].nunique() != 1: continue
#if s1s_O['peak'].nunique() != 1: continue
## s2sNO = [rebin_peak(s2raw_NO, n_rebin) for s2raw_NO in s2sraw_NO]
## s2sO = [rebin_peak(s2raw_O, n_rebin) for s2raw_O in s2sraw_O]
## s1tNO = s1s_NO[0].times
## s1eNO = s1s_NO[0].pmts
## S1tNO = s1t_NO[np.argmax(s1eNO)]
## s1tO = s1s_O[0].times
## s1eO = s1s_O[0].pmts
## S1tO = s1t_O[np.argmax(s1eO)]
#print('Aqui?')
#fill histograms
for is1 in range(s1s_NO['peak'].nunique()):
s1 = s1s_NO[s1s_NO['peak'] == is1]
s1H_NO.append(s1['ene'].max())
s1W_NO.append(s1['time'].iloc[-1] - s1['time'].iloc[0])
s1Q_NO.append(s1['ene'].sum())
for js1 in range(s1s_O['peak'].nunique()):
s1 = s1s_O[s1s_O['peak'] == js1]
s1H_O.append(s1['ene'].max())
s1W_O.append(s1['time'].iloc[-1] - s1['time'].iloc[0])
s1Q_O.append(s1['ene'].sum())
s2per_evt_NO.append(s2s_NO['peak'].nunique())
s2per_evt_O.append(s2s_O['peak'].nunique())
for is2 in range(s2s_NO['peak'].nunique()):
s2 = s2s_NO[s2s_NO['peak'] == is2]
s2Q_NO.append(s2['ene'].sum())
## if s2['time'].iloc[-1] - s2['time'].iloc[0] < 0:
## print(evtNO, ' has -ve width S2 in NO')
s2W_NO.append(s2['time'].iloc[-1] - s2['time'].iloc[0])
s2H_NO.append(s2['ene'].max())
for js2 in range(s2s_O['peak'].nunique()):
s2 = s2s_O[s2s_O['peak'] == js2]
s2Q_O.append(s2['ene'].sum())
## if s2['time'].iloc[-1] - s2['time'].iloc[0] < 0:
## print(evtO, ' has -ve width S2 in O')
s2W_O.append(s2['time'].iloc[-1] - s2['time'].iloc[0])
s2H_O.append(s2['ene'].max())
print(min(s1per_evt_NO), max(s1per_evt_NO), min(s1per_evt_O), max(s1per_evt_O))
print(min(s2per_evt_NO), max(s2per_evt_NO), min(s2per_evt_O), max(s2per_evt_O))
print(min(s1Q_NO), max(s1Q_NO), min(s2Q_O), max(s2Q_O))
print(min(s1H_NO), max(s1H_NO), min(s2H_O), max(s2H_O))
print(min(s1W_NO), max(s1W_NO), min(s2W_NO), max(s2W_NO))
print(min(s1W_O), max(s1W_O), min(s2W_O), max(s2W_O))
fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(20,6))
axes[0][0].hist(s1per_evt_NO, bins=10, range=(0, 10), label='No. S1s no low frequency noise', log=True, histtype='step')
axes[0][0].hist(s1per_evt_O , bins=10, range=(0, 10), label='No. S1s with low frequency noise', log=True, histtype='step')
axes[0][0].legend()
axes[0][0].set_xlabel('No. s1 per event')
axes[0][0].set_ylabel('AU')
axes[0][1].hist(s1H_NO, bins=200, range=(0, 200), label='S1 height no low frequency noise', log=True, histtype='step')
axes[0][1].hist(s1H_O , bins=200, range=(0, 200), label='S1 height with low frequency noise', log=True, histtype='step')
axes[0][1].legend()
axes[0][1].set_xlabel('S1 height (pes)')
axes[0][1].set_ylabel('AU')
axes[1][0].hist(s1Q_NO, bins=100, range=(0, 1500), label='S1 charge no low frequency noise', log=True, histtype='step')
axes[1][0].hist(s1Q_O , bins=100, range=(0, 1500), label='S1 charge with low frequency noise', log=True, histtype='step')
axes[1][0].legend()
axes[1][0].set_xlabel('S1 charge (pes)')
axes[1][0].set_ylabel('AU')
axes[1][1].hist(s1W_NO, bins=40, range=(0, 1000), label='S1 width no low frequency noise', log=True, histtype='step')
axes[1][1].hist(s1W_O , bins=40, range=(0, 1000), label='S1 width with low frequency noise', log=True, histtype='step')
axes[1][1].legend()
axes[1][1].set_xlabel('S1 width (ns)')
axes[1][1].set_ylabel('AU')
fig.show()
fig.savefig('s1_plots_pmaps_oscNoosc.png')
fig2, axes2 = plt.subplots(nrows=2, ncols=2, figsize=(20,6))
axes2[0][0].hist(s2per_evt_NO, bins=10, range=(0, 10), label='No. S2s no low frequency noise', log=True, histtype='step')
axes2[0][0].hist(s2per_evt_O , bins=10, range=(0, 10), label='No. S2s with low frequency noise', log=True, histtype='step')
axes2[0][0].legend()
axes2[0][0].set_xlabel('No. s2 per event')
axes2[0][0].set_ylabel('AU')
axes2[0][1].hist(s2H_NO, bins=1000, range=(10, 65000), label='S2 height no low frequency noise', log=True, histtype='step')
axes2[0][1].hist(s2H_O , bins=1000, range=(10, 65000), label='S2 height with low frequency noise', log=True, histtype='step')
axes2[0][1].legend()
axes2[0][1].set_xlabel('S2 height (pes)')
axes2[0][1].set_ylabel('AU')
axes2[1][0].hist(s2Q_NO, bins=1000, range=(10, 750000), label='S2 charge no low frequency noise', log=True, histtype='step')
axes2[1][0].hist(s2Q_O , bins=1000, range=(10, 750000), label='S2 charge with low frequency noise', log=True, histtype='step')
axes2[1][0].legend()
axes2[1][0].set_xlabel('S2 charge (pes)')
axes2[1][0].set_ylabel('AU')
axes2[1][1].hist(s2W_NO, bins=235, range=(1500, 350000), label='S2 width no low frequency noise', log=True, histtype='step')
axes2[1][1].hist(s2W_O , bins=235, range=(1500, 350000), label='S2 width with low frequency noise', log=True, histtype='step')
axes2[1][1].legend()
axes2[1][1].set_xlabel('S2 width (ns)')
axes2[1][1].set_ylabel('AU')
fig2.show()
fig2.savefig('s2_plots_pmaps_oscNoosc.png')
input('Done with plots?')
file2 = output + "images/" + file_name + "&&.jpg"
file3 = output + "images/" + file_name + "&&&.jpg"
file4 = output + "images/" + file_name + "&&&&.jpg"
cv2.imwrite(file, outputs[0])
cv2.imwrite(file1, outputs[1])
cv2.imwrite(file2, outputs[2])
cv2.imwrite(file3, outputs[3])
cv2.imwrite(file4, outputs[4])
the_file.close()
HEIGHT = effect_config.get("height")
WIDTH = HEIGHT * 4
gosh = np.ones((256, 64, 3), np.uint8) * 255
gosh = Image.from_array(gosh)
words = np.load("words.npy")[:1000]
dic_length = len(words)
output_path = "out/"
fonts_file = gl("all_fonts/*.ttf")
bg_imgs = []
bg_files = gl("backgrounds/*.jpg")
print("num_bg:", len(bg_files))
print("num fronts:", len(fonts_file))
for i in range(len(bg_files)):
img = cv2.imread(bg_files[i], 1)
bg_imgs.append(img)
output = "out/"