def parse_acquisitions(self):
src_paths = []
for f in self._files:
# Check if it is a DICOM
try:
ds = dicom.read_file(f)
except InvalidDicomError:
utils.warning('Not a DICOM file: {}'.format(f))
continue
# Filter by serie description
dsDesc = ds.SeriesDescription
if any(_ in dsDesc for _ in self.exclusionCriteria):
self._excluded.add(dsDesc)
else:
src_paths.append(f)
groups = dcmstack.parse_and_group(src_paths)
for key, group in groups.iteritems():
if self._isFromOneDirectory(group):
#utils.ok("Same directory: {}".format(key[2]))
stack = dcmstack.stack_group(group)
inDir = os.path.dirname(group[0][2])
self._acquisitions.append(
Acquisition(inDir, stack, self._session))
else:
#TODO: regroup dicoms in a tmp directory
utils.new_line()
utils.fail(
"DICOM of '{}' are not in the same directory.\nThis structure won't be compatible with dcm2niibatch"
.format(key[2]))
def main():
awareness = Awareness(BROADCAST_IP, BROADCAST_PORT, HEARBEAT_INTERVAL)
awareness.start()
acquisition = Acquisition(UNICAST_PORT)
acquisition.start()
time.sleep(DEFAULT_EXECUTION_TIME)
awareness.stop()
acquisition.stop()
def test_acquisition_bad_retrieval():
acquisition_obj = Acquisition()
try:
acquisition_obj.retrieve(
"http://s1.twnmm.com/images/en_ca/icons/flags/all/16/ca.png",
"ca2.png")
assert False
except Exception as e:
assert True
def __init__(self, config_file_path):
"""
__init__ - self
"""
LOGGER.info("Initializing UATU.")
self.config_file = config_file_path
self.cfg_organizer = ConfigOrganizer(config_file=self.config_file)
self.cfg_organizer.read_config_data()
self.acq_obj = Acquisition()
self.current_max_count()
LOGGER.info("Completed initialization.")
self.img_processing = ImageProcessing(
yolo_path=self.cfg_organizer.config_handler['system']['yolo_dir'])
def __init__(self):
super().__init__()
self.acquisition = Acquisition()
self.preview = CaptureDevicePreview(self.acquisition.captureDevices)
self.mainLayout = True
self.readyForAcquisition = False
# THREAD: Main Thread which works continuously
self.stopMainThread = False
self.mainThread = threading.Thread(target=self.runMainThread, args=(lambda: self.stopMainThread,))
self.mainThread.start()
# THREAD: Acquisition
self.createAcquisitionThread()
def main(args):
task_seq = [
# acquire_method(sub_acquire_method): random(""), no-dete("DAL","BALD"), dete("coreset","entropy",...)
# ../../datasets/answer_selection/YahooCQA/data/data-FD/
#evidence, diversity # BiLSTM CNN
{
"model_name": "BiLSTM",
"group_name": "[tkde]BiLSTM+Pets+MRR+160+160",
"max_performance": 0.80,
"data_path":
"../../datasets/answer_selection/YahooCQA/data/data-Pets/",
"acquire_method": "no-dete",
"sub_acquire_method": "DAL",
"unsupervised_method": '',
"submodular_k": 1.5,
"num_acquisitions_round": 50,
"init_question_num": 32,
"acquire_question_num_per_round": 32,
"warm_start_random_seed": 0,
"sample_method": "No-Deterministic+DALLL2Layer+0",
},
{
"model_name": "BiLSTM",
"group_name": "[tkde]BiLSTM+Pets+MRR+160+160",
"max_performance": 0.80,
"data_path":
"../../datasets/answer_selection/YahooCQA/data/data-Pets/",
"acquire_method": "no-dete",
"sub_acquire_method": "DAL",
"unsupervised_method": '',
"submodular_k": 1.5,
"num_acquisitions_round": 50,
"init_question_num": 32,
"acquire_question_num_per_round": 32,
"warm_start_random_seed": 16,
"sample_method": "No-Deterministic+DALLL2Layer+16",
},
{
"model_name": "BiLSTM",
"group_name": "[tkde]BiLSTM+Pets+MRR+160+160",
"max_performance": 0.80,
"data_path":
"../../datasets/answer_selection/YahooCQA/data/data-Pets/",
"acquire_method": "no-dete",
"sub_acquire_method": "DAL",
"unsupervised_method": '',
"submodular_k": 1.5,
"num_acquisitions_round": 50,
"init_question_num": 32,
"acquire_question_num_per_round": 32,
"warm_start_random_seed": 32,
"sample_method": "No-Deterministic+DALLL2Layer+32",
},
{
"model_name": "BiLSTM",
"group_name": "[tkde]BiLSTM+Pets+MRR+160+160",
"max_performance": 0.80,
"data_path":
"../../datasets/answer_selection/YahooCQA/data/data-Pets/",
"acquire_method": "no-dete",
"sub_acquire_method": "DAL",
"unsupervised_method": '',
"submodular_k": 1.5,
"num_acquisitions_round": 50,
"init_question_num": 32,
"acquire_question_num_per_round": 32,
"warm_start_random_seed": 64,
"sample_method": "No-Deterministic+DALLL2Layer+64",
},
{
"model_name": "BiLSTM",
"group_name": "[tkde]BiLSTM+Pets+MRR+160+160",
"max_performance": 0.80,
"data_path":
"../../datasets/answer_selection/YahooCQA/data/data-Pets/",
"acquire_method": "no-dete",
"sub_acquire_method": "DAL",
"unsupervised_method": '',
"submodular_k": 1.5,
"num_acquisitions_round": 50,
"init_question_num": 32,
"acquire_question_num_per_round": 32,
"warm_start_random_seed": 128,
"sample_method": "No-Deterministic+DALLL2Layer+128",
},
{
"model_name": "BiLSTM",
"group_name": "[tkde]BiLSTM+Pets+MRR+160+160",
"max_performance": 0.80,
"data_path":
"../../datasets/answer_selection/YahooCQA/data/data-Pets/",
"acquire_method": "random",
"sub_acquire_method": "",
"unsupervised_method": '',
"submodular_k": 1.5,
"num_acquisitions_round": 50,
"init_question_num": 32,
"acquire_question_num_per_round": 32,
"warm_start_random_seed": 0,
"sample_method": "No-Deterministic+randommm2Layer+0",
},
{
"model_name": "BiLSTM",
"group_name": "[tkde]BiLSTM+Pets+MRR+160+160",
"max_performance": 0.80,
"data_path":
"../../datasets/answer_selection/YahooCQA/data/data-Pets/",
"acquire_method": "random",
"sub_acquire_method": "",
"unsupervised_method": '',
"submodular_k": 1.5,
"num_acquisitions_round": 50,
"init_question_num": 32,
"acquire_question_num_per_round": 32,
"warm_start_random_seed": 16,
"sample_method": "No-Deterministic+randommm2Layer+16",
},
{
"model_name": "BiLSTM",
"group_name": "[tkde]BiLSTM+Pets+MRR+160+160",
"max_performance": 0.80,
"data_path":
"../../datasets/answer_selection/YahooCQA/data/data-Pets/",
"acquire_method": "random",
"sub_acquire_method": "",
"unsupervised_method": '',
"submodular_k": 1.5,
"num_acquisitions_round": 50,
"init_question_num": 32,
"acquire_question_num_per_round": 32,
"warm_start_random_seed": 32,
"sample_method": "No-Deterministic+randommm2Layer+32",
},
{
"model_name": "BiLSTM",
"group_name": "[tkde]BiLSTM+Pets+MRR+160+160",
"max_performance": 0.80,
"data_path":
"../../datasets/answer_selection/YahooCQA/data/data-Pets/",
"acquire_method": "random",
"sub_acquire_method": "",
"unsupervised_method": '',
"submodular_k": 1.5,
"num_acquisitions_round": 50,
"init_question_num": 32,
"acquire_question_num_per_round": 32,
"warm_start_random_seed": 64,
"sample_method": "No-Deterministic+randommm2Layer+64",
},
{
"model_name": "BiLSTM",
"group_name": "[tkde]BiLSTM+Pets+MRR+160+160",
"max_performance": 0.80,
"data_path":
"../../datasets/answer_selection/YahooCQA/data/data-Pets/",
"acquire_method": "random",
"sub_acquire_method": "",
"unsupervised_method": '',
"submodular_k": 1.5,
"num_acquisitions_round": 50,
"init_question_num": 32,
"acquire_question_num_per_round": 32,
"warm_start_random_seed": 128,
"sample_method": "No-Deterministic+randommm2Layer+128",
},
]
allMethods_results = [
] #Record the performance results of each method during active learning
for config in task_seq:
print("-------------------{}-{}-------------------".format(
config["group_name"], config["sample_method"]))
####################################### initial setting ###########################################
data_path = config["data_path"]
model_name = config[
"model_name"] if "model_name" in config else 'BiLSTM'
num_acquisitions_round = config["num_acquisitions_round"]
acquire_method = config["acquire_method"]
sub_acquire_method = config["sub_acquire_method"]
init_question_num = config[
"init_question_num"] if "init_question_num" in config else 160 # number of initial training samples
acquire_question_num_per_round = config[
"acquire_question_num_per_round"] if "acquire_question_num_per_round" in config else 20 #Number of samples collected per round
warm_start_random_seed = config[
"warm_start_random_seed"] # the random seed for selecting the initial training set
sample_method = config["sample_method"]
loader = Loader()
print('model:', model_name)
print('dataset:', data_path)
print('acquisition method:', acquire_method, "+", sub_acquire_method)
if not os.path.exists(args.result_path):
os.makedirs(args.result_path)
if not os.path.exists(os.path.join(args.result_path, model_name)):
os.makedirs(os.path.join(args.result_path, model_name))
if not os.path.exists(
os.path.join(args.result_path, model_name, 'active_checkpoint',
acquire_method)):
os.makedirs(
os.path.join(args.result_path, model_name, 'active_checkpoint',
acquire_method))
#### If the data is not compiled, compile; otherwise load directly
if (os.path.exists(os.path.join(data_path, 'mappings.pkl'))
and os.path.exists(os.path.join(data_path, 'train.pkl'))
and os.path.exists(os.path.join(data_path, 'test.pkl'))):
mappings = pkl.load(
open(os.path.join(data_path, 'mappings.pkl'), 'rb'))
train_data = pkl.load(
open(os.path.join(data_path, 'train.pkl'), 'rb'))
test_data = pkl.load(
open(os.path.join(data_path, 'test.pkl'), 'rb'))
else:
train_data, test_data, mappings = loader.load_yahoo(
data_path, args.pretrained_word_embedding,
args.word_embedding_dim, args.answer_count)
pkl.dump(mappings,
open(os.path.join(data_path, 'mappings.pkl'), 'wb'))
pkl.dump(train_data,
open(os.path.join(data_path, 'train.pkl'), 'wb'))
pkl.dump(test_data, open(os.path.join(data_path, 'test.pkl'),
'wb'))
#word embedding
word_to_id = mappings['word_to_id']
tag_to_id = mappings['tag_to_id']
word_embeds = mappings[
'word_embeds'] if args.use_pretrained_word_embedding else None
word_vocab_size = len(word_to_id)
print(
' The total amount of training data：%d\n' % len(
train_data
), # Total number of training samples (number of question answer pair)
'The total amount of val data：%d\n' % len(test_data))
acquisition_function = Acquisition(train_data,
seed=warm_start_random_seed,
answer_count=args.answer_count,
cuda_device=args.device[0],
batch_size=args.sampling_batch_size,
submodular_k=config["submodular_k"])
checkpoint_path = os.path.join(args.result_path, 'active_checkpoint',
config["group_name"], sample_method)
if not os.path.exists(checkpoint_path):
os.makedirs(checkpoint_path)
method_result = [
] # Record the performance results of each method during active learning
####################################### acquire data and retrain ###########################################
for i in range(num_acquisitions_round):
print("current round：{}".format(i))
#-------------------acquisition---------------------
if i == 0: #first round
acq = init_question_num
a_m = "random"
m_p = ""
else:
acq = acquire_question_num_per_round
a_m = acquire_method
m_p = os.path.join(checkpoint_path, 'modelweights')
acquisition_function.obtain_data(
train_data,
model_path=m_p,
model_name=model_name,
acquire_questions_num=acq,
method=a_m,
sub_method=sub_acquire_method,
unsupervised_method=config["unsupervised_method"])
# -------------------prepare training data---------------------
'''
train_data format：
{
'str_words_q': str_words_q, # question word segmentation
'str_words_a': str_words_a, # answer word segmentation
'words_q': words_q, # question word id
'words_a': words_a, # answer word id
'tag': tag, # sample tag id
}
'''
new_train_index = (acquisition_function.train_index).copy()
sorted_train_index = list(new_train_index)
sorted_train_index.sort()
labeled_train_data = [train_data[i] for i in sorted_train_index]
print("Labeled training samples: {}".format(
len(acquisition_function.train_index)))
# -------------------------------------train--------------------------------------
print('.............Recreate the model...................')
if model_name == 'BiLSTM':
model = BiLSTM(word_vocab_size,
args.word_embedding_dim,
args.word_hidden_dim,
args.target_size,
pretrained=word_embeds,
with_sim_features=args.with_sim_feature,
cuda_device=args.device[0])
if model_name == 'CNN':
model = CNN(word_vocab_size,
args.word_embedding_dim,
args.word_out_channels,
args.target_size,
pretrained=word_embeds,
cuda_device=args.device[0])
model.cuda(args.device[0])
trainer = Trainer(model,
model_name,
tag_to_id,
answer_count=args.answer_count,
cuda_device=args.device[0],
sampling_number=args.sampling_number)
test_performance = trainer.train_supervisedLearning(
args.num_epochs,
labeled_train_data,
test_data,
args.learning_rate,
checkpoint_path=checkpoint_path,
batch_size=args.batch_size)
print('.' * 50)
print("Test performance: {}".format(test_performance))
print('*' * 50)
#--------------------------Send data for a visual web page------------------------------
max_performance = config[
"max_performance"] if "max_performance" in config else 0
if "group_name" in config:
updateLineChart(str(test_performance),
sample_method,
gp_name=config["group_name"],
max=max_performance)
else:
updateLineChart(str(test_performance),
sample_method,
max=max_performance)
# method_result.append(test_performance)
#
# print("acquire_method: {}，sub_acquire_method: {}, warm_start_random_seed{}"
# .format(acquire_method, sub_acquire_method, warm_start_random_seed))
# print(method_result)
# allMethods_results.append(method_result)
shutil.rmtree(checkpoint_path)
def __init__(self, ws_host="", ws_port=5678):
self.daq = Acquisition()
self.ws_handler = WebSocketHandler(ws_host, ws_port, self.daq)
def test_acquisition_retrieval():
acquisition_obj = Acquisition()
acquisition_obj.retrieve(
"http://s1.twnmm.com/images/en_ca/icons/flags/all/16/ca.png", "ca.png")
assert os.path.exists("ca.png") == True
def test_acquisition_creation():
acquisition_obj = Acquisition()
assert acquisition_obj is not None