parser.add_argument('--trsh',
required=False,
help='choose threshold',
type=float,
default=0.1)
parser.add_argument('--test', action="store_true", default=False)
args = parser.parse_args()
threshold = args.trsh
features_root = args.feat
files_list = sorted(glob.glob('../data/data/training/*.h5'))
ws = 400
dp = rfc.DataProvider(ny=ws,
a_min=0,
a_max=200,
files=files_list,
label_name='gt_mask',
n_class=2)
_, nx, ny, _ = dp(1)[0].shape
training_iters = 100
epochs = 300
model_dir = './models/unet_' + str(features_root) + '_' + str(threshold)
net = unet.Unet(channels=dp.channels,
n_class=dp.n_class,
layers=3,
features_root=features_root,
cost_kwargs=dict(regularizer=0.001))
if not args.test:
learning_rate = args.learning_rate
rfc.the_print('Chosen architecture is: ' + args.arch + '; learning rate = ' +
str(learning_rate),
bgc='green')
name = arch + '_' + str(nx) + 'x' + str(ny)
model_add = './models/kat7_model_' + name
files_list = sorted(glob.glob('../../data/kat7/dataset2/training*.h5'))
rfc.the_print('number of files: ' + str(len(files_list)))
dpt = rfc.DataProvider(nx=nx,
ny=ny,
a_min=0,
a_max=200,
files=files_list,
label_name='mask')
#print dp(5)[0].shape
#_,nx,ny,_ = dp(1)[0].shape
#if args.arch=='1':
# if nx>450:
# dtype = tf.float16
#if args.arch=='2':
# if nx>450:
# dtype = tf.float16
#if args.arch=='3':
# if nx>350:
# dtype = tf.float16
threshold = args.trsh
time_limit = args.time_limit
learning_rate = args.learning_rate
rfc.the_print('Chosen architecture is: ' + args.arch + '; threshold =: ' +
str(threshold) + '; learning rate = ' + str(learning_rate),
bgc='green')
model_add = './models/model_' + arch + '_' + str(threshold)
files_list = sorted(glob.glob('../../data/hide_sims_train/calib_1year/*.fits'))
rfc.the_print('number of files: ' + str(len(files_list)))
ws = 400
dp = rfc.DataProvider(ny=ws,
a_min=0,
a_max=200,
files=files_list,
label_name='RFI_MASK',
threshold=threshold)
#print dp(5)[0].shape
_, nx, ny, _ = dp(1)[0].shape
conv = rfc.ConvolutionalLayers(nx=nx,
ny=ny,
n_channel=1,
restore=os.path.exists(model_add),
model_add=model_add,
arch_file_name='arch_' + args.arch)
n_rounds = 10
if args.train:
for i in range(n_rounds):
learning_rate = args.learning_rate
n_class = 10
thresholds = 10**np.linspace(-8,1,n_class,endpoint=1)
th_labels = np.linspace(0.2,1,n_class+1,endpoint=1)[1:]
rfc.the_print('Chosen architecture is: '+args.arch+'; threshold =: '+str(threshold)+'; learning rate = '+str(learning_rate),bgc='green')
model_add = './models/multiclass_model_'+arch+'_'+str(threshold)
files_list = sorted(glob.glob('../../data/hide_sims_train/calib_1year/*.fits'))
rfc.the_print('number of files: '+str(len(files_list)))
ws = 400
dp = rfc.DataProvider(files=files_list,label_name='RFI',
ny=ws,
thresholds=thresholds,
th_labels=th_labels,
a_min=0, a_max=200)
#print dp(5)[0].shape
_,nx,ny,_ = dp(1)[0].shape
conv = rfc.ConvolutionalLayers(nx=nx,ny=ny,n_channel=1,restore=os.path.exists(model_add),model_add=model_add,arch_file_name='arch_'+args.arch)
n_rounds = 10
if args.train:
for i in range(n_rounds):
rfc.the_print('ROUND: '+str(i)+', learning rate='+str(learning_rate),bgc='blue')
conv.train(data_provider=dp,training_epochs = 10000000,n_s = 100,learning_rate = learning_rate, dropout=0.7, time_limit=time_limit//n_rounds, verbose=1)
learning_rate = learning_rate/4.
mode = 'one_hot'
thresholds = [1e-10, 0.1]
th_labels = [0,1,2]
rfc.the_print('Chosen architecture is: '+args.arch,bgc='green')
model_add = './models/multiclass_model_'+arch+'_'+mode
test_files = sorted(glob.glob('/home/anke/HIDE_simulations/hide_sims_test/calib_1month/*.fits'))
rfc.the_print('number of files: '+str(len(test_files)))
ws = 400
dp = rfc.DataProvider(files=test_files,label_name='RFI',
ny=ws,
one_hot=1,
thresholds=thresholds,
th_labels=th_labels,
a_min=0, a_max=200)
_,nx,ny,nc = dp(1)[1].shape
print(dp(1)[1].shape)
conv = rfc.ConvolutionalLayers(nx=nx,ny=ny,n_channel=1,n_class=nc,
restore=os.path.exists(model_add),
model_add=model_add,
arch_file_name=arch)
res_file = 'results/threeclass_'+arch+'_'+mode
rfc.ch_mkdir('results')
rfc.ch_mkdir('predictions')
import pylab as plt
from matplotlib.colors import LogNorm
import rficnn as rfc
dss = ['training', 'validation', 'test']
rfc.ch_mkdir('plots')
for ds in dss:
test_files = sorted(glob.glob('../../../data/kat7/dataset/' + ds +
'/*.h5'))
for fil in test_files:
fname = fil.split('/')[-1]
dp = rfc.DataProvider(a_min=0,
a_max=100,
files=[fil],
label_name='mask')
data, mask = dp(1)
mask = mask[0, :, :, 0]
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(12, 8))
ax1.imshow(data[0, :, :, 0], norm=LogNorm(), aspect='auto')
ax2.imshow(mask, aspect='auto')
plt.subplots_adjust(left=0.04, right=0.99, top=0.99, bottom=0.04)
plt.savefig('./plots/' + ds + '_' + fname + '.jpg', dpi=100)
plt.close()
