def load_all():
print('Carrying out initial reduction for NGC147')
n147 = data_loader('ngc147', cls_bands=cls_bands)
print('Carrying out initial reduction for NGC185')
n185 = data_loader('ngc185', cls_bands=cls_bands)
print('Carrying out initial reduction for NGC205')
n205 = data_loader('ngc205', cls_bands=cls_bands)
print('Carrying out initial reduction for M32')
m32 = data_loader('m32', cls_bands=cls_bands)
return [n147, n185, n205, m32]
def main(args):
#load data
if args.mode == 'train':
data_loader_train = data_load.data_loader(args.train_feat,
args.train_phn,
args.batch_size,
meta_path=args.meta,
max_length=args.max_length,
is_training=True)
else:
data_loader_train = data_load.data_loader(args.train_feat,
args.train_phn,
args.batch_size,
meta_path=args.meta,
max_length=args.max_length,
is_training=True)
data_loader_test = data_load.data_loader(args.test_feat,
args.test_phn,
args.batch_size,
max_length=args.max_length,
is_training=False)
#add some feature to args
args.feat_dim = data_loader_train.feat_dim
args.vocab_size = data_loader_train.vocab_size
#build model graph
if args.mode == 'train':
g = model(args)
else:
g = model(args, is_training=False)
print("Graph loaded")
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
#create sess
with tf.Session(graph=g.graph) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(max_to_keep=3)
if (args.mode != 'train') or (args.load == 'load'):
print('load_model')
saver.restore(sess, tf.train.latest_checkpoint(args.save_dir))
if args.mode == 'train':
print('training')
train(sess, g, args, saver, data_loader_train)
else:
print('evaluating')
evaluation(sess, g, args, data_loader_train, data_loader_test)
def train(DATA_URL, SAVE_URL):
x_train, y_train, x_test = data_loader(DATA_URL)
model = create_cnn_model(x_train)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
# 파일 이름에 에포크 번호를 포함시킵니다(`str.format` 포맷)
checkpoint_path = os.path.join(SAVE_URL,"my_model.h5")
# checkpoint_dir = SAVE_URL
model.fit(x_train, y_train, epochs=20, verbose=0, callbacks=[TqdmCallback(verbose=2)])
model.save(checkpoint_path)
return model
def main(args):
#load data
if args.mode == 'train':
data_loader = data_load.data_loader(args.max_length, args.train_idx, args.train_audio2vec, args.train_oracle, args.mapping, target_data_path=args.target)
else:
data_loader = data_load.data_loader(args.max_length, args.test_idx, args.test_audio2vec, args.test_oracle, args.mapping, target_data_path=None)
#add some feature to args
args.idx_size = data_loader.idx_size
args.phn_size = data_loader.vocab_size
args.feat_dim = data_loader.feat_dim
#build model graph
if args.mode == 'train':
g = model(args)
else:
g = model(args, is_training=False)
print("Graph loaded")
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
#create sess
with tf.Session(graph=g.graph) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(max_to_keep=3)
if (args.mode != 'train') or (args.load == 'load'):
print('load_model')
saver.restore(sess, tf.train.latest_checkpoint(args.save_dir))
if args.mode == 'train':
print('training')
train(sess, g, args, saver, data_loader)
else:
print('evaluating')
evaluation(sess, g, args, data_loader)
def main():
test_path = './colon_artificial'
train_path = './colon_artificial'
types = 'EVENT'
loader = data_loader(train_path, types=types)
loader.write_sent_tokenizer('sent_tokenize.pickle')
[sent_list, sent_labels], [train_span_list,
train_labels_span] = loader.train_data()
[test_list, test_labels], [test_span_list,
test_labels_span] = loader.test_data(test_path)
v = vocabulary(sent_list)
#tag_to_indx = {"O":0,"B-TIMEX3":1,"IN-TIMEX3":2}
v.write('vocabulary.dict')
def load_sph_all():
and1 = data_loader('and1', cls_bands=cls_bands)
and2 = data_loader('and2', cls_bands=cls_bands)
and3 = data_loader('and3', cls_bands=cls_bands)
and6 = data_loader('and6', cls_bands=cls_bands)
and7 = data_loader('and7', cls_bands=cls_bands)
and10 = data_loader('and10', cls_bands=cls_bands)
and14 = data_loader('and14', cls_bands=cls_bands)
and15 = data_loader('and15', cls_bands=cls_bands)
and16 = data_loader('and16', cls_bands=cls_bands)
and17 = data_loader('and17', cls_bands=cls_bands)
and18 = data_loader('and18', cls_bands=cls_bands)
and19 = data_loader('and19', cls_bands=cls_bands)
and20 = data_loader('and20', cls_bands=cls_bands)
return [
and1, and2, and3, and6, and7, and10, and14, and15, and16,
and17, and18, and19, and20
]
PATHS = create_paths()
def model_save(fn):
with open(fn, 'wb') as f:
torch.save([model, criterion, optimizer], f)
def model_load(fn):
global model, criterion, optimizer
with open(fn, 'rb') as f:
model, criterion, optimizer = torch.load(f)
print("loading data")
from data_load import data_loader
dl, TEXT = data_loader(PATHS, bs=args.batch_size, bptt=args.bptt)
eval_batch_size = 10
test_batch_size = 1
train_data = dl.trn_dl
val_data = dl.val_dl
test_data = dl.test_dl
ntokens = dl.nt
# dump param dict
dump_param_dict(PATHS, TEXT, ntokens, args.batch_size, args.bptt, args.emsize, args.nhid, args.nlayers)
###############################################################################
# Build the model
###############################################################################
from splitcross import SplitCrossEntropyLoss
criterion = None
from data_load import data_loader
from preprocess import vocabulary
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from MultiTask import MultiTaskLSTM
from utils import tag, convert, metrics, linkMetrics
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
test_path = './colon_artificial/Dev'
train_path = './colon_artificial/train'
#types = 'TIMEX3'
types = 'EVENT'
loader = data_loader(train_path,
types=types,
sent_tokenizer='sent_tokenize.pickle')
[sent_list,
sent_labels], [train_span_list,
train_labels_span], [train_linkSpans,
train_linkLabels] = loader.train_data()
[test_list, test_labels], [test_span_list, test_labels_span
], [test_linkSpans,
test_linkLabels] = loader.test_data(test_path)
v = vocabulary()
#tag_to_indx = {"O":0,"B-"+types:1,"IN-"+types:2}
tag_to_indx = {"O": 0, "B-EVENT": 1, "IN-EVENT": 2}
v.load('vocabulary.dict')
def test(model, threshold=0.5):
def make_table():
processed_stages = ['cls_cut', 'fore_cut', 'cm']
# galaxies=['ngc147','ngc185','ngc205','m32']
dr = data_readall(stage='cls_cut')
n147_cls = dr.n147.data
n185_cls = dr.n185.data
n205_cls = dr.n205.data
m32_cls = dr.m32.data
dr = data_readall(stage='cls_crossed')
n147_cls_crossed = dr.n147.data
n185_cls_crossed = dr.n185.data
n205_cls_crossed = dr.n205.data
m32_cls_crossed = dr.m32.data
dr = data_readall(stage='fore_cut')
n147_fore_cut = dr.n147.data
n185_fore_cut = dr.n185.data
n205_fore_cut = dr.n205.data
m32_fore_cut = dr.m32.data
dr = data_readall(stage='cm')
n147_c = dr.n147.cdata
n185_c = dr.n185.cdata
n205_c = dr.n205.cdata
m32_c = dr.m32.cdata
n147_m = dr.n147.mdata
n185_m = dr.n185.mdata
n205_m = dr.n205.mdata
m32_m = dr.m32.mdata
n147_agb = dr.n147.data
n185_agb = dr.n185.data
n205_agb = dr.n205.data
m32_agb = dr.m32.data
n147_fore = pd.concat([n147_fore_cut,
n147_cls]).drop_duplicates(keep=False)
n185_fore = pd.concat([n185_fore_cut,
n185_cls]).drop_duplicates(keep=False)
n205_fore = pd.concat([n205_fore_cut,
n205_cls]).drop_duplicates(keep=False)
m32_fore = pd.concat([m32_fore_cut, m32_cls]).drop_duplicates(keep=False)
# trgb_cuts = [18.137, 17.862, 17.930, 17.8]
n147_rgb = pd.concat([n147_fore_cut,
n147_agb]).drop_duplicates(subset=['RA', 'DEC'],
keep=False)
for i in n147_rgb.index:
if n147_rgb['kmag'].at[i] < 18.137:
n147_rgb.loc[i] = np.nan
n185_rgb = pd.concat([n185_fore_cut,
n185_agb]).drop_duplicates(subset=['RA', 'DEC'],
keep=False)
for i in n185_rgb.index:
if n185_rgb['kmag'].at[i] < 17.862:
n185_rgb.loc[i] = np.nan
n205_rgb = pd.concat([n205_fore_cut,
n205_agb]).drop_duplicates(subset=['RA', 'DEC'],
keep=False)
for i in n205_rgb.index:
if n205_rgb['kmag'].at[i] < 17.930:
n205_rgb.loc[i] = np.nan
m32_rgb = pd.concat([m32_fore_cut,
m32_agb]).drop_duplicates(subset=['RA', 'DEC'],
keep=False)
for i in m32_rgb.index:
if m32_rgb['kmag'].at[i] < 17.8:
m32_rgb.loc[i] = np.nan
fore_cuts = [0.992, 0.964, 1.00, 0.92]
n147_gai = pd.concat([n147_cls, n147_cls_crossed
]).drop_duplicates(subset=['RA', 'DEC'], keep=False)
n147_gai_orig = n147_gai.copy()
for i in n147_gai.index:
if n147_gai['kmag'].at[i] > 18.137 or n147_gai['jmag'].at[
i] - n147_gai['kmag'].at[i] < 0.992:
n147_gai.loc[i] = np.nan
n185_gai = pd.concat([n185_cls, n185_cls_crossed
]).drop_duplicates(subset=['RA', 'DEC'], keep=False)
n185_gai_orig = n185_gai.copy()
for i in n185_gai.index:
if n185_gai['kmag'].at[i] > 17.862 or n185_gai['jmag'].at[
i] - n185_gai['kmag'].at[i] < 0.964:
n185_gai.loc[i] = np.nan
n205_gai = pd.concat([n205_cls, n205_cls_crossed
]).drop_duplicates(subset=['RA', 'DEC'], keep=False)
n205_gai_orig = n205_gai.copy()
for i in n205_gai.index:
if n205_gai['kmag'].at[i] > 17.930 or n205_gai['jmag'].at[
i] - n205_gai['kmag'].at[i] < 1.00:
n205_gai.loc[i] = np.nan
m32_gai = pd.concat([m32_cls, m32_cls_crossed
]).drop_duplicates(subset=['RA', 'DEC'], keep=False)
m32_gai_orig = m32_gai.copy()
for i in m32_gai.index:
if m32_gai['kmag'].at[i] > 17.8 or m32_gai['jmag'].at[i] - m32_gai[
'kmag'].at[i] < 0.92:
m32_gai.loc[i] = np.nan
dl = data_loader(galaxy='ngc147',
CLS=False,
cls_bands='norm',
mag=False,
ext=True,
path_to_file='initial_data/')
n147_raw = dl.data
dl = data_loader(galaxy='ngc185',
CLS=False,
cls_bands='norm',
mag=False,
ext=True,
path_to_file='initial_data/')
n185_raw = dl.data
dl = data_loader(galaxy='ngc205',
CLS=False,
cls_bands='norm',
mag=False,
ext=True,
path_to_file='initial_data/')
n205_raw = dl.data
dl = data_loader(galaxy='m32',
CLS=False,
cls_bands='norm',
mag=False,
ext=True,
path_to_file='initial_data/')
m32_raw = dl.data
dl = data_loader(galaxy='ngc147',
CLS=True,
cls_bands='norm',
mag=False,
ext=True,
path_to_file='initial_data/')
n147_mag = dl.data
dl = data_loader(galaxy='ngc185',
CLS=True,
cls_bands='norm',
mag=False,
ext=True,
path_to_file='initial_data/')
n185_mag = dl.data
dl = data_loader(galaxy='ngc205',
CLS=True,
cls_bands='norm',
mag=False,
ext=True,
path_to_file='initial_data/')
n205_mag = dl.data
dl = data_loader(galaxy='m32',
CLS=True,
cls_bands='norm',
mag=False,
ext=True,
path_to_file='initial_data/')
m32_mag = dl.data
n147_mag_fail = pd.concat([n147_cls, n147_mag]).drop_duplicates(keep=False)
n185_mag_fail = pd.concat([n185_cls, n185_mag]).drop_duplicates(keep=False)
n205_mag_fail = pd.concat([n205_cls, n205_mag]).drop_duplicates(keep=False)
m32_mag_fail = pd.concat([m32_cls, m32_mag]).drop_duplicates(keep=False)
n147_cls_fail = pd.concat([n147_mag, n147_raw]).drop_duplicates(keep=False)
n185_cls_fail = pd.concat([n185_mag, n185_raw]).drop_duplicates(keep=False)
n205_cls_fail = pd.concat([n205_mag, n205_raw]).drop_duplicates(keep=False)
m32_cls_fail = pd.concat([m32_mag, m32_raw]).drop_duplicates(keep=False)
def add_flag(frame, flag, galaxy):
frame['class'] = flag
frame['galaxy'] = galaxy
n147_gai_orig = n147_gai_orig.dropna(how='all')
n185_gai_orig = n185_gai_orig.dropna(how='all')
n205_gai_orig = n205_gai_orig.dropna(how='all')
m32_gai_orig = m32_gai_orig.dropna(how='all')
add_flag(n147_gai_orig, 'GAIA', 'NGC147')
add_flag(n185_gai_orig, 'GAIA', 'NGC185')
add_flag(n205_gai_orig, 'GAIA', 'NGC205')
add_flag(m32_gai_orig, 'GAIA', 'M32')
gai_table = pd.concat(
[n147_gai_orig, n185_gai_orig, n205_gai_orig, m32_gai_orig])
n147_c = n147_c.dropna(how='all')
n185_c = n185_c.dropna(how='all')
n205_c = n205_c.dropna(how='all')
m32_c = m32_c.dropna(how='all')
n147_m = n147_m.dropna(how='all')
n185_m = n185_m.dropna(how='all')
n205_m = n205_m.dropna(how='all')
m32_m = m32_m.dropna(how='all')
n147_fore = n147_fore.dropna(how='all')
n185_fore = n185_fore.dropna(how='all')
n205_fore = n205_fore.dropna(how='all')
m32_fore = m32_fore.dropna(how='all')
n147_rgb = n147_rgb.dropna(how='all')
n185_rgb = n185_rgb.dropna(how='all')
n205_rgb = n205_rgb.dropna(how='all')
m32_rgb = m32_rgb.dropna(how='all')
n147_gai = n147_gai.dropna(how='all')
n185_gai = n185_gai.dropna(how='all')
n205_gai = n205_gai.dropna(how='all')
m32_gai = m32_gai.dropna(how='all')
n147_mag_fail = n147_mag_fail.dropna(how='all')
n185_mag_fail = n185_mag_fail.dropna(how='all')
n205_mag_fail = n205_mag_fail.dropna(how='all')
m32_mag_fail = m32_mag_fail.dropna(how='all')
n147_cls_fail = n147_cls_fail.dropna(how='all')
n185_cls_fail = n185_cls_fail.dropna(how='all')
n205_cls_fail = n205_cls_fail.dropna(how='all')
m32_cls_fail = m32_cls_fail.dropna(how='all')
add_flag(n147_c, 'C-AGB', 'NGC147')
add_flag(n185_c, 'C-AGB', 'NGC185')
add_flag(n205_c, 'C-AGB', 'NGC205')
add_flag(m32_c, 'C-AGB', 'M32')
add_flag(n147_m, 'M-AGB', 'NGC147')
add_flag(n185_m, 'M-AGB', 'NGC185')
add_flag(n205_m, 'M-AGB', 'NGC205')
add_flag(m32_m, 'M-AGB', 'M32')
add_flag(n147_rgb, 'RGB', 'NGC147')
add_flag(n185_rgb, 'RGB', 'NGC185')
add_flag(n205_rgb, 'RGB', 'NGC205')
add_flag(m32_rgb, 'RGB', 'M32')
add_flag(n147_gai, 'AGB_CROSS', 'NGC147')
add_flag(n185_gai, 'AGB_CROSS', 'NGC185')
add_flag(n205_gai, 'AGB_CROSS', 'NGC205')
add_flag(m32_gai, 'AGB_CROSS', 'M32')
add_flag(n147_fore, 'FORE_SEQ', 'NGC147')
add_flag(n185_fore, 'FORE_SEQ', 'NGC185')
add_flag(n205_fore, 'FORE_SEQ', 'NGC205')
add_flag(m32_fore, 'FORE_SEQ', 'M32')
add_flag(n147_cls_fail, 'NOISE-CLS', 'NGC147')
add_flag(n185_cls_fail, 'NOISE-CLS', 'NGC185')
add_flag(n205_cls_fail, 'NOISE-CLS', 'NGC205')
add_flag(m32_cls_fail, 'NOISE-CLS', 'M32')
add_flag(n147_mag_fail, 'NOISE-MAG', 'NGC147')
add_flag(n185_mag_fail, 'NOISE-MAG', 'NGC185')
add_flag(n205_mag_fail, 'NOISE-MAG', 'NGC205')
add_flag(m32_mag_fail, 'NOISE-MAG', 'M32')
table = pd.concat([
n147_cls_fail, n185_cls_fail, n205_cls_fail, m32_cls_fail,
n147_mag_fail, n185_mag_fail, n205_mag_fail, m32_mag_fail, n147_fore,
n185_fore, n205_fore, m32_fore, n147_rgb, n185_rgb, n205_rgb, m32_rgb,
n147_gai, n185_gai, n205_gai, m32_gai, n147_m, n185_m, n205_m, m32_m,
n147_c, n185_c, n205_c, m32_c
])
table.to_parquet('master_table')
gai_table.to_parquet('gaia_table')