def write_predictions(self):
"""Write final predictions to the json file."""
unique_id_to_result = {}
for result in self._all_results:
unique_id_to_result[result.unique_id] = result
all_predictions = collections.OrderedDict()
for example in self._eval_examples:
example_id = example.qas_id
features = self._task.featurize(example, False, for_eval=True)[0]
result = unique_id_to_result[features[self._name + "_eid"]]
max_index = np.argmax(result.logits)
_max_index = max_index
decoded_indexes = []
for i in range(self._config.max_options_num - 1, -1, -1):
if _max_index >= 2**i:
decoded_indexes.append(i)
_max_index -= 2**i
elif _max_index <= 0:
break
options_tags = features[self._name + "_options_tags"]
combination_options = features[self._name + "_combination_options"]
answer = None
if features[self._name + "_type"] == "0":
# 异常捕获，如果有异常要解决掉，下同
if len(decoded_indexes) != 1 or decoded_indexes[0] >= len(
options_tags):
utils.log(
"decode single answer error, set answer C for default."
)
answer = "C"
else:
answer = options_tags[decoded_indexes[0]]
elif features[self._name + "_type"] == "1":
if len(decoded_indexes
) == 0 or decoded_indexes[0] >= len(options_tags):
utils.log(
"decode combination single answer error, set answer C for default."
)
answer = "C"
else:
comb_ops_pred = []
for ind in decoded_indexes:
comb_ops_pred.append(options_tags[ind])
answer = None
for op, comb_ops in combination_options.items():
if set(comb_ops) == set(comb_ops_pred):
answer = op
break
if answer is None:
utils.log(
"decode combination single answer error, set answer C for default."
)
answer = "C"
all_predictions[example_id] = [answer] if answer else []
utils.write_json(
dict(all_predictions),
self._config.qa_preds_file(self._name + "_" + self._split))
def run_finetuning(config: configure_finetuning.FinetuningConfig):
"""Run finetuning."""
hvd.init()
config.model_dir = config.model_dir if hvd.rank() == 0 else \
os.path.join(config.model_dir, str(hvd.rank()))
config.train_batch_size = config.train_batch_size // hvd.size()
# Setup for training
results = []
trial = 1
heading_info = "model={:}, trial {:}/{:}".format(
config.model_name, trial, config.num_trials)
heading = lambda msg: utils.heading(msg + ": " + heading_info)
heading("Config")
utils.log_config(config)
generic_model_dir = config.model_dir
tasks = task_builder.get_tasks(config)
# Train and evaluate num_trials models with different random seeds
while config.num_trials < 0 or trial <= config.num_trials:
config.model_dir = generic_model_dir + "_" + str(trial)
if config.do_train:
utils.rmkdir(config.model_dir)
model_runner = ModelRunner(config, tasks, hvd)
if config.do_train:
heading("Start training")
model_runner.train()
utils.log()
if config.do_eval:
heading("Run dev set evaluation")
results.append(model_runner.evaluate())
write_results(config, results)
if config.write_test_outputs and trial <= config.n_writes_test:
heading("Running on the test set and writing the predictions")
for task in tasks:
# Currently only writing preds for GLUE and SQuAD 2.0 is supported
if task.name in ["cola", "mrpc", "mnli", "sst", "rte", "qnli", "qqp",
"sts"]:
for split in task.get_test_splits():
model_runner.write_classification_outputs([task], trial, split)
elif task.name == "squad":
scorer = model_runner.evaluate_task(task, "test", False)
scorer.write_predictions()
preds = utils.load_json(config.qa_preds_file("squad"))
null_odds = utils.load_json(config.qa_na_file("squad"))
for q, _ in preds.items():
if null_odds[q] > config.qa_na_threshold:
preds[q] = ""
utils.write_json(preds, config.test_predictions(
task.name, "test", trial))
else:
utils.log("Skipping task", task.name,
"- writing predictions is not supported for this task")
if trial != config.num_trials and (not config.keep_all_models):
utils.rmrf(config.model_dir)
trial += 1
def _process_dir(self, dir_path, json_path, relabel):
if osp.exists(json_path):
print("=> {} generated before, awesome!".format(json_path))
split = read_json(json_path)
return split['tracklets'], split['num_tracklets'], split['num_pids'], split['num_imgs_per_tracklet']
print("=> Automatically generating split (might take a while for the first time, have a coffe)")
pdirs = glob.glob(osp.join(dir_path, '*')) # avoid .DS_Store
print("Processing {} with {} person identities".format(dir_path, len(pdirs)))
pid_container = set()
for pdir in pdirs:
pid = int(osp.basename(pdir))
pid_container.add(pid)
pid2label = {pid:label for label, pid in enumerate(pid_container)}
tracklets = []
num_imgs_per_tracklet = []
for pdir in pdirs:
pid = int(osp.basename(pdir))
if relabel: pid = pid2label[pid]
tdirs = glob.glob(osp.join(pdir, '*'))
for tdir in tdirs:
raw_img_paths = glob.glob(osp.join(tdir, '*.jpg'))
num_imgs = len(raw_img_paths)
if num_imgs < self.min_seq_len:
continue
num_imgs_per_tracklet.append(num_imgs)
img_paths = []
for img_idx in range(num_imgs):
# some tracklet starts from 0002 instead of 0001
img_idx_name = 'F' + str(img_idx+1).zfill(4)
res = glob.glob(osp.join(tdir, '*' + img_idx_name + '*.jpg'))
if len(res) == 0:
print("Warn: index name {} in {} is missing, jump to next".format(img_idx_name, tdir))
continue
img_paths.append(res[0])
img_name = osp.basename(img_paths[0])
camid = int(img_name[5]) - 1 # index-0
img_paths = tuple(img_paths)
tracklets.append((img_paths, pid, camid))
num_pids = len(pid_container)
num_tracklets = len(tracklets)
print("Saving split to {}".format(json_path))
split_dict = {
'tracklets': tracklets,
'num_tracklets': num_tracklets,
'num_pids': num_pids,
'num_imgs_per_tracklet': num_imgs_per_tracklet,
}
write_json(split_dict, json_path)
return tracklets, num_tracklets, num_pids, num_imgs_per_tracklet
def write_predictions(self):
"""Write final predictions to the json file."""
all_predictions = collections.OrderedDict()
unique_id_to_text_a = {}
for example in self._eval_examples:
unique_id_to_text_a[example.eid] = example.text_a
for eid, pred in zip(self._eid, self._preds):
text_a = unique_id_to_text_a[eid]
org_pred = pred * (self._max_value -
self._min_value) + self._min_value
all_predictions[text_a] = max(1, round(org_pred))
utils.write_json(
dict(all_predictions),
self._config.cl_preds_file(self._name + "_" + self._split))
def write_predictions(self):
"""Write final predictions to the json file."""
all_predictions = collections.OrderedDict()
unique_id_to_text_a = {}
label_id_to_label_text = {}
for example in self._eval_examples:
unique_id_to_text_a[example.eid] = example.text_a
for label_id, label_text in enumerate(self._label_list):
label_id_to_label_text[label_id] = label_text
for eid, pred in zip(self._eid, self._preds):
text_a = unique_id_to_text_a[eid]
label_text = label_id_to_label_text[int(pred)]
all_predictions[text_a] = label_text
utils.write_json(
dict(all_predictions),
self._config.cl_preds_file(self._name + "_" + self._split))
def _split_dataset(self):
train_raw, train_gt = self._generate_list(self.train_split)
val_raw, val_gt = self._generate_list(self.val_split)
val_selected_raw = self._generate_list_selected(
osp.join(self.val_selected_split, 'velodyne_raw'))
val_selected_gt = self._generate_list_selected(
osp.join(self.val_selected_split, 'groundtruth_depth'))
test_raw = self._generate_list_selected(self.test_dir)
splits = {
'train_raw': train_raw,
'train_gt': train_gt,
'val_raw': val_raw,
'val_gt': val_gt,
'val_selected_raw': val_selected_raw,
'val_selected_gt': val_selected_gt,
'test_raw': test_raw
}
write_json(splits, self.splits)
def _serialize_balanced_dataset(self, tasks, is_training, split):
"""Write out the dataset as tfrecords."""
labels = [
self._tasks[i]._label_list for i in range(len(self_tasks))
]
dataset_name = "_".join(sorted([task.name for task in tasks]))
dataset_name += "_" + split
dataset_prefix = os.path.join(self._config.preprocessed_data_dir,
dataset_name)
tfrecords_path = dataset_prefix + ".tfrecord"
metadata_path = dataset_prefix + ".metadata"
batch_size = (self._config.train_batch_size
if is_training else self._config.eval_batch_size)
utils.log("Loading dataset", dataset_name)
n_examples = None
if (self._config.use_tfrecords_if_existing
and tf.io.gfile.exists(metadata_path)):
n_examples = utils.load_json(metadata_path)["n_examples"]
if n_examples is None:
utils.log("Existing tfrecords not found so creating")
examples = []
for task in tasks:
task_examples = task.get_examples(split)
examples += task_examples
if is_training:
random.shuffle(examples)
utils.mkdir(tfrecords_path.rsplit("/", 1)[0])
n_examples = self.serialize_examples(examples, is_training,
tfrecords_path,
batch_size)
utils.write_json({"n_examples": n_examples}, metadata_path)
input_fn = self._input_fn_builder(tfrecords_path, is_training)
if is_training:
steps = int(n_examples // batch_size *
self._config.num_train_epochs)
else:
steps = n_examples // batch_size
return input_fn, steps
def write_predictions(self):
"""Write final predictions to the json file."""
all_predictions = collections.OrderedDict()
all_pred_results = self._get_improved_span_labels(False, True)
assert len(self._eval_examples) == len(all_pred_results)
for example, span_preds in zip(self._eval_examples, all_pred_results):
words = example.words
orig_id = example.orig_id
word_to_char_mapping = self._word_to_char_mapping[orig_id]
answers = collections.OrderedDict()
for s, e, l in span_preds:
if l not in answers.keys():
answers[l] = []
answers[l].append(
("".join(words[s:e + 1]), word_to_char_mapping[s],
word_to_char_mapping[e]))
all_predictions[orig_id] = answers
utils.write_json(
dict(all_predictions),
self._config.ner_preds_file(self._name + "_" + self._split))
def _prepare_split(self):
if not osp.exists(self.split_path):
print("Creating splits")
mat_split_data = loadmat(self.split_mat_path)['ls_set']
num_splits = mat_split_data.shape[0]
num_total_ids = mat_split_data.shape[1]
assert num_splits == 10
assert num_total_ids == 300
num_ids_each = num_total_ids/2
# pids in mat_split_data are indices, so we need to transform them
# to real pids
person_cam1_dirs = os.listdir(self.cam_1_path)
person_cam2_dirs = os.listdir(self.cam_2_path)
# make sure persons in one camera view can be found in the other camera view
assert set(person_cam1_dirs) == set(person_cam2_dirs)
splits = []
for i_split in range(num_splits):
# first 50% for testing and the remaining for training, following Wang et al. ECCV'14.
train_idxs = sorted(list(mat_split_data[i_split,num_ids_each:]))
test_idxs = sorted(list(mat_split_data[i_split,:num_ids_each]))
train_idxs = [int(i)-1 for i in train_idxs]
test_idxs = [int(i)-1 for i in test_idxs]
# transform pids to person dir names
train_dirs = [person_cam1_dirs[i] for i in train_idxs]
test_dirs = [person_cam1_dirs[i] for i in test_idxs]
split = {'train': train_dirs, 'test': test_dirs}
splits.append(split)
print("Totally {} splits are created, following Wang et al. ECCV'14".format(len(splits)))
print("Split file is saved to {}".format(self.split_path))
write_json(splits, self.split_path)
print("Splits created")
def vote1(dataset, all_nbest, all_odds, qid_answers, split, output_dir):
bagging_preds = collections.OrderedDict()
bagging_odds = collections.OrderedDict()
bagging_all_nbest = collections.OrderedDict()
for qid in qid_answers:
bagging_preds[qid] = \
(seq([nbest[qid][0] for nbest in all_nbest]).sorted(key=lambda x: x['probability'])).list()[-1]['text']
bagging_all_nbest[qid] = \
[(seq([nbest[qid][0] for nbest in all_nbest]).sorted(key=lambda x: x['probability'])).list()[-1]]
bagging_odds[qid] = np.mean([odds[qid] for odds in all_odds])
utils.write_json(
bagging_preds,
os.path.join(output_dir, 'vote1',
'ccks42bagging_{}_preds.json'.format(split)))
utils.write_pickle(
bagging_all_nbest,
os.path.join(output_dir, 'vote1',
'ccks42bagging_{}_all_nbest.pkl'.format(split)))
utils.write_json(
bagging_odds,
os.path.join(output_dir, 'vote1',
'ccks42bagging_{}_null_odds.json'.format(split)))
if split in ['train', 'dev']:
out_eval = main2(dataset, bagging_preds, bagging_odds)
utils.log('vote1')
utils.log(out_eval)
elif split == 'eval':
for qid in bagging_preds.keys():
if bagging_odds[qid] > -2.75:
bagging_preds[qid] = ""
utils.write_json(
bagging_preds,
os.path.join(output_dir, 'vote1',
'ccks42bagging_{}_1_preds.json'.format(split)))
else:
utils.log('{} split is not supported'.format(split))
def vote2(dataset, all_nbest, all_odds, qid_answers, split, output_dir):
bagging_preds = collections.OrderedDict()
bagging_odds = collections.OrderedDict()
for qid in qid_answers:
preds_scores = (seq(all_nbest).map(lambda x: x[qid][0]).map(
lambda x: (x['text'], x['probability']))).dict()
compare = collections.defaultdict(lambda: 0.)
for pred, score in preds_scores.items():
compare[pred] += score
compare = seq(compare.items()).sorted(lambda x: x[1]).reverse().list()
bagging_preds[qid] = compare[0][0]
bagging_odds[qid] = np.mean([odds[qid] for odds in all_odds])
utils.write_json(
bagging_preds,
os.path.join(output_dir, 'vote2',
'ccks42bagging_{}_preds.json'.format(split)))
utils.write_json(
bagging_odds,
os.path.join(output_dir, 'vote2',
'ccks42bagging_{}_null_odds.json'.format(split)))
if split in ['train', 'dev']:
out_eval = main2(dataset, bagging_preds, bagging_odds)
utils.log('vote2')
utils.log(out_eval)
elif split == 'eval':
for qid in bagging_preds.keys():
if bagging_odds[qid] > -2.75:
bagging_preds[qid] = ""
utils.write_json(
bagging_preds,
os.path.join(output_dir, 'vote2',
'ccks42bagging_{}_1_preds.json'.format(split)))
else:
utils.log('{} split is not supported'.format(split))
def vote3(dataset, all_nbest, all_odds, qid_answers, qid_questions, models,
split, output_dir):
bagging_preds = collections.OrderedDict()
bagging_odds = collections.OrderedDict()
def post_process(question, candi, weight=1):
question = question.lower()
first_token = candi['text'].split()[0]
th = 0.
if "when" in question:
if first_token in [
'before', 'after', 'about', 'around', 'from', 'during'
]:
candi['probability'] += th
elif "where" in question:
if first_token in [
'in', 'at', 'on', 'behind', 'from', 'through', 'between',
'throughout'
]:
candi['probability'] += th
elif "whose" in question:
if "'s" in candi['text']:
candi['probability'] += th
elif "which" in question:
if first_token == "the":
candi['probability'] += th
candi['probability'] *= weight
return candi
cof = 0.2
for qid in qid_answers:
question = qid_questions[qid]
post_process_candidates = (seq(zip(all_nbest, models)).map(lambda x: (
x[0][qid], cof if 'lr_epoch_results' in x[1] else 1.)).map(
lambda x: seq(x[0]).map(lambda y: post_process(
question, y, x[1])).list()).flatten()).list()
preds_probs = collections.defaultdict(lambda: [])
for pred in post_process_candidates:
preds_probs[pred['text']].append(pred['probability'])
for pred in post_process_candidates:
preds_probs[pred['text']] = np.mean(
preds_probs[pred['text']]).__float__()
bagging_preds[qid] = (seq(preds_probs.items()).sorted(
lambda x: x[1]).reverse().map(lambda x: x[0])).list()[0]
bagging_odds[qid] = np.mean([
odds[qid] * cof if 'lr_epoch_results' in model else odds[qid]
for odds, model in zip(all_odds, models)
])
utils.write_json(
bagging_preds,
os.path.join(output_dir, 'vote3',
'ccks42bagging_{}_preds.json'.format(split)))
utils.write_json(
bagging_odds,
os.path.join(output_dir, 'vote3',
'ccks42bagging_{}_null_odds.json'.format(split)))
if split in ['train', 'dev']:
out_eval = main2(dataset, bagging_preds, bagging_odds)
utils.log('vote3')
utils.log(out_eval)
elif split == 'eval':
for qid in bagging_preds.keys():
if bagging_odds[qid] > -2.75:
bagging_preds[qid] = ""
utils.write_json(
bagging_preds,
os.path.join(output_dir, 'vote3',
'ccks42bagging_{}_1_preds.json'.format(split)))
else:
utils.log('{} split is not supported'.format(split))
def write_predictions(self):
"""Write final predictions to the json file."""
unique_id_to_result = {}
for result in self._all_results:
unique_id_to_result[result.unique_id] = result
_PrelimPrediction = collections.namedtuple( # pylint: disable=invalid-name
"PrelimPrediction", [
"feature_index", "start_index", "end_index", "start_logit",
"end_logit"
])
all_predictions = collections.OrderedDict()
all_nbest_json = collections.OrderedDict()
scores_diff_json = collections.OrderedDict()
for example in self._eval_examples:
example_id = example.qas_id if "squad" in self._name else example.qid
features = self._task.featurize(example, False, for_eval=True)
prelim_predictions = []
# keep track of the minimum score of null start+end of position 0
score_null = 1000000 # large and positive
for (feature_index, feature) in enumerate(features):
result = unique_id_to_result[feature[self._name + "_eid"]]
if self._config.joint_prediction:
start_indexes = result.start_top_index
end_indexes = result.end_top_index
else:
start_indexes = _get_best_indexes(result.start_logits,
self._config.n_best_size)
end_indexes = _get_best_indexes(result.end_logits,
self._config.n_best_size)
# if we could have irrelevant answers, get the min score of irrelevant
if self._v2:
if self._config.answerable_classifier:
feature_null_score = result.answerable_logit
else:
feature_null_score = result.start_logits[
0] + result.end_logits[0]
if feature_null_score < score_null:
score_null = feature_null_score
for i, start_index in enumerate(start_indexes):
for j, end_index in enumerate(
end_indexes[i] if self._config.
joint_prediction else end_indexes):
# We could hypothetically create invalid predictions, e.g., predict
# that the start of the span is in the question. We throw out all
# invalid predictions.
if start_index >= len(feature[self._name + "_tokens"]):
continue
if end_index >= len(feature[self._name + "_tokens"]):
continue
if start_index == 0:
continue
if start_index not in feature[self._name +
"_token_to_orig_map"]:
continue
if end_index not in feature[self._name +
"_token_to_orig_map"]:
continue
if not feature[self._name +
"_token_is_max_context"].get(
start_index, False):
continue
if end_index < start_index:
continue
length = end_index - start_index + 1
if length > self._config.max_answer_length:
continue
start_logit = (result.start_top_log_probs[i]
if self._config.joint_prediction else
result.start_logits[start_index])
end_logit = (result.end_top_log_probs[i, j]
if self._config.joint_prediction else
result.end_logits[end_index])
prelim_predictions.append(
_PrelimPrediction(feature_index=feature_index,
start_index=start_index,
end_index=end_index,
start_logit=start_logit,
end_logit=end_logit))
if self._v2:
if len(prelim_predictions) == 0 and self._config.debug:
tokid = sorted(feature[self._name +
"_token_to_orig_map"].keys())[0]
prelim_predictions.append(
_PrelimPrediction(feature_index=0,
start_index=tokid,
end_index=tokid + 1,
start_logit=1.0,
end_logit=1.0))
prelim_predictions = sorted(prelim_predictions,
key=lambda x:
(x.start_logit + x.end_logit),
reverse=True)
_NbestPrediction = collections.namedtuple( # pylint: disable=invalid-name
"NbestPrediction", ["text", "start_logit", "end_logit"])
seen_predictions = {}
nbest = []
for pred in prelim_predictions:
if len(nbest) >= self._config.n_best_size:
break
feature = features[pred.feature_index]
tok_tokens = feature[self._name + "_tokens"][pred.start_index:(
pred.end_index + 1)]
orig_doc_start = feature[self._name + "_token_to_orig_map"][
pred.start_index]
orig_doc_end = feature[self._name +
"_token_to_orig_map"][pred.end_index]
orig_tokens = example.doc_tokens[orig_doc_start:(orig_doc_end +
1)]
tok_text = " ".join(tok_tokens)
# De-tokenize WordPieces that have been split off.
tok_text = tok_text.replace(" ##", "")
tok_text = tok_text.replace("##", "")
# Clean whitespace
tok_text = tok_text.strip()
tok_text = " ".join(tok_text.split())
orig_text = " ".join(orig_tokens)
final_text = get_final_text(self._config, tok_text, orig_text)
if final_text in seen_predictions:
continue
seen_predictions[final_text] = True
nbest.append(
_NbestPrediction(text=final_text,
start_logit=pred.start_logit,
end_logit=pred.end_logit))
# In very rare edge cases we could have no valid predictions. So we
# just create a nonce prediction in this case to avoid failure.
if not nbest:
nbest.append(
_NbestPrediction(text="empty",
start_logit=0.0,
end_logit=0.0))
assert len(nbest) >= 1
total_scores = []
best_non_null_entry = None
for entry in nbest:
total_scores.append(entry.start_logit + entry.end_logit)
if not best_non_null_entry:
if entry.text:
best_non_null_entry = entry
probs = _compute_softmax(total_scores)
nbest_json = []
for (i, entry) in enumerate(nbest):
output = collections.OrderedDict()
output["text"] = entry.text
output["probability"] = probs[i]
output["start_logit"] = entry.start_logit
output["end_logit"] = entry.end_logit
nbest_json.append(dict(output))
assert len(nbest_json) >= 1
if not self._v2:
all_predictions[example_id] = nbest_json[0]["text"]
else:
# predict "" iff the null score - the score of best non-null > threshold
if self._config.answerable_classifier:
score_diff = score_null
else:
score_diff = score_null - best_non_null_entry.start_logit - (
best_non_null_entry.end_logit)
scores_diff_json[example_id] = score_diff
all_predictions[example_id] = best_non_null_entry.text
all_nbest_json[example_id] = nbest_json
utils.write_json(dict(all_predictions),
self._config.qa_preds_file(self._name))
if self._v2:
utils.write_json(
{k: float(v)
for k, v in six.iteritems(scores_diff_json)},
self._config.qa_na_file(self._name))
def _preprocess(self):
"""
This function is a bit complex and ugly, what it does is
1. Extract data from cuhk-03.mat and save as png images.
2. Create 20 classic splits. (Li et al. CVPR'14)
3. Create new split. (Zhong et al. CVPR'17)
"""
print("Note: if root path is changed, the previously generated json files need to be re-generated (delete them first)")
if osp.exists(self.imgs_labeled_dir) and \
osp.exists(self.imgs_detected_dir) and \
osp.exists(self.split_classic_det_json_path) and \
osp.exists(self.split_classic_lab_json_path) and \
osp.exists(self.split_new_det_json_path) and \
osp.exists(self.split_new_lab_json_path):
return
mkdir_if_missing(self.imgs_detected_dir)
mkdir_if_missing(self.imgs_labeled_dir)
print("Extract image data from {} and save as png".format(self.raw_mat_path))
mat = h5py.File(self.raw_mat_path, 'r')
def _deref(ref):
return mat[ref][:].T
def _process_images(img_refs, campid, pid, save_dir):
img_paths = [] # Note: some persons only have images for one view
for imgid, img_ref in enumerate(img_refs):
img = _deref(img_ref)
# skip empty cell
if img.size == 0 or img.ndim < 3: continue
# images are saved with the following format, index-1 (ensure uniqueness)
# campid: index of camera pair (1-5)
# pid: index of person in 'campid'-th camera pair
# viewid: index of view, {1, 2}
# imgid: index of image, (1-10)
viewid = 1 if imgid < 5 else 2
img_name = '{:01d}_{:03d}_{:01d}_{:02d}.png'.format(campid+1, pid+1, viewid, imgid+1)
img_path = osp.join(save_dir, img_name)
imsave(img_path, img)
img_paths.append(img_path)
return img_paths
def _extract_img(name):
print("Processing {} images (extract and save) ...".format(name))
meta_data = []
imgs_dir = self.imgs_detected_dir if name == 'detected' else self.imgs_labeled_dir
for campid, camp_ref in enumerate(mat[name][0]):
camp = _deref(camp_ref)
num_pids = camp.shape[0]
for pid in range(num_pids):
img_paths = _process_images(camp[pid,:], campid, pid, imgs_dir)
assert len(img_paths) > 0, "campid{}-pid{} has no images".format(campid, pid)
meta_data.append((campid+1, pid+1, img_paths))
print("done camera pair {} with {} identities".format(campid+1, num_pids))
return meta_data
meta_detected = _extract_img('detected')
meta_labeled = _extract_img('labeled')
def _extract_classic_split(meta_data, test_split):
train, test = [], []
num_train_pids, num_test_pids = 0, 0
num_train_imgs, num_test_imgs = 0, 0
for i, (campid, pid, img_paths) in enumerate(meta_data):
if [campid, pid] in test_split:
for img_path in img_paths:
camid = int(osp.basename(img_path).split('_')[2])
test.append((img_path, num_test_pids, camid))
num_test_pids += 1
num_test_imgs += len(img_paths)
else:
for img_path in img_paths:
camid = int(osp.basename(img_path).split('_')[2])
train.append((img_path, num_train_pids, camid))
num_train_pids += 1
num_train_imgs += len(img_paths)
return train, num_train_pids, num_train_imgs, test, num_test_pids, num_test_imgs
print("Creating classic splits (# = 20) ...")
splits_classic_det, splits_classic_lab = [], []
for split_ref in mat['testsets'][0]:
test_split = _deref(split_ref).tolist()
# create split for detected images
train, num_train_pids, num_train_imgs, test, num_test_pids, num_test_imgs = \
_extract_classic_split(meta_detected, test_split)
splits_classic_det.append({
'train': train, 'query': test, 'gallery': test,
'num_train_pids': num_train_pids, 'num_train_imgs': num_train_imgs,
'num_query_pids': num_test_pids, 'num_query_imgs': num_test_imgs,
'num_gallery_pids': num_test_pids, 'num_gallery_imgs': num_test_imgs,
})
# create split for labeled images
train, num_train_pids, num_train_imgs, test, num_test_pids, num_test_imgs = \
_extract_classic_split(meta_labeled, test_split)
splits_classic_lab.append({
'train': train, 'query': test, 'gallery': test,
'num_train_pids': num_train_pids, 'num_train_imgs': num_train_imgs,
'num_query_pids': num_test_pids, 'num_query_imgs': num_test_imgs,
'num_gallery_pids': num_test_pids, 'num_gallery_imgs': num_test_imgs,
})
write_json(splits_classic_det, self.split_classic_det_json_path)
write_json(splits_classic_lab, self.split_classic_lab_json_path)
def _extract_set(filelist, pids, pid2label, idxs, img_dir, relabel):
tmp_set = []
unique_pids = set()
for idx in idxs:
img_name = filelist[idx][0]
camid = int(img_name.split('_')[2])
pid = pids[idx]
if relabel: pid = pid2label[pid]
img_path = osp.join(img_dir, img_name)
tmp_set.append((img_path, int(pid), camid))
unique_pids.add(pid)
return tmp_set, len(unique_pids), len(idxs)
def _extract_new_split(split_dict, img_dir):
train_idxs = split_dict['train_idx'].flatten() - 1 # index-0
pids = split_dict['labels'].flatten()
train_pids = set(pids[train_idxs])
pid2label = {pid: label for label, pid in enumerate(train_pids)}
query_idxs = split_dict['query_idx'].flatten() - 1
gallery_idxs = split_dict['gallery_idx'].flatten() - 1
filelist = split_dict['filelist'].flatten()
train_info = _extract_set(filelist, pids, pid2label, train_idxs, img_dir, relabel=True)
query_info = _extract_set(filelist, pids, pid2label, query_idxs, img_dir, relabel=False)
gallery_info = _extract_set(filelist, pids, pid2label, gallery_idxs, img_dir, relabel=False)
return train_info, query_info, gallery_info
print("Creating new splits for detected images (767/700) ...")
train_info, query_info, gallery_info = _extract_new_split(
loadmat(self.split_new_det_mat_path),
self.imgs_detected_dir,
)
splits = [{
'train': train_info[0], 'query': query_info[0], 'gallery': gallery_info[0],
'num_train_pids': train_info[1], 'num_train_imgs': train_info[2],
'num_query_pids': query_info[1], 'num_query_imgs': query_info[2],
'num_gallery_pids': gallery_info[1], 'num_gallery_imgs': gallery_info[2],
}]
write_json(splits, self.split_new_det_json_path)
print("Creating new splits for labeled images (767/700) ...")
train_info, query_info, gallery_info = _extract_new_split(
loadmat(self.split_new_lab_mat_path),
self.imgs_labeled_dir,
)
splits = [{
'train': train_info[0], 'query': query_info[0], 'gallery': gallery_info[0],
'num_train_pids': train_info[1], 'num_train_imgs': train_info[2],
'num_query_pids': query_info[1], 'num_query_imgs': query_info[2],
'num_gallery_pids': gallery_info[1], 'num_gallery_imgs': gallery_info[2],
}]
write_json(splits, self.split_new_lab_json_path)
def run_finetuning(config: configure_finetuning.FinetuningConfig):
"""Run finetuning."""
# Setup for training
results = []
trial = 1
heading_info = "model={:}, trial {:}/{:}".format(config.model_name, trial,
config.num_trials)
heading = lambda msg: utils.heading(msg + ": " + heading_info)
heading("Config")
utils.log_config(config)
generic_model_dir = config.model_dir
tasks = task_builder.get_tasks(config)
# Train and evaluate num_trials models with different random seeds
while config.num_trials < 0 or trial <= config.num_trials:
config.model_dir = generic_model_dir + "_" + str(trial)
if config.do_train:
utils.rmkdir(config.model_dir)
model_runner = ModelRunner(config, tasks)
if config.do_train:
heading("Start training")
model_runner.train()
utils.log()
if config.do_eval:
heading("Run dev set evaluation")
results.append(model_runner.evaluate())
if config.do_test:
for task in tasks:
test_score = model_runner.evaluate_task_test(
task, results[-1][task.name]['checkpoint_path'])
results[-1][task.name]["test_results"] = test_score
write_results(config, results)
if config.write_test_outputs and trial <= config.n_writes_test:
heading("Running on the test set and writing the predictions")
for task in tasks:
# Currently only writing preds for GLUE and SQuAD 2.0 is supported
if task.name in [
"cola", "mrpc", "mnli", "sst", "rte", "qnli",
"qqp", "sts"
]:
for split in task.get_test_splits():
model_runner.write_classification_outputs([task],
trial,
split)
elif task.name == "squad":
scorer = model_runner.evaluate_task(
task, "test", False)
scorer.write_predictions()
preds = utils.load_json(config.qa_preds_file("squad"))
null_odds = utils.load_json(config.qa_na_file("squad"))
for q, _ in preds.items():
if null_odds[q] > config.qa_na_threshold:
preds[q] = ""
utils.write_json(
preds,
config.test_predictions(task.name, "test", trial))
else:
utils.log(
"Skipping task", task.name,
"- writing predictions is not supported for this task"
)
if config.do_predict:
if "dev" in config.predict_split:
results = model_runner.predict(tasks[0],
config.predict_checkpoint_path,
"dev")
import pickle
with open("predict_dev.pickle", "bw") as outfile:
pickle.dump(results, outfile)
if "train" in config.predict_split:
results = model_runner.predict(tasks[0],
config.predict_checkpoint_path,
"train")
import pickle
with open("predict_train.pickle", "bw") as outfile:
pickle.dump(results, outfile)
if "test" in config.predict_split:
results = model_runner.predict(tasks[0],
config.predict_checkpoint_path,
"test")
import pickle
with open("predict_test.pickle", "bw") as outfile:
pickle.dump(results, outfile)
if trial != config.num_trials and (not config.keep_all_models):
utils.rmrf(config.model_dir)
trial += 1
def run_finetuning(config: configure_finetuning.FinetuningConfig):
"""Run finetuning."""
tf.get_variable_scope().reuse_variables() #import pdb; pdb.set_trace()
# Setup for training
results = []
trial = 1
heading_info = "model={:}, trial {:}/{:}".format(
config.model_name, trial, config.num_trials)
heading = lambda msg: utils.heading(msg + ": " + heading_info)
heading("Config")
utils.log_config(config)
generic_model_dir = config.model_dir
tasks = task_builder.get_tasks(config)
# Train and evaluate num_trials models with different random seeds
while config.num_trials < 0 or trial <= config.num_trials:
config.model_dir = generic_model_dir + "_" + str(trial)
if config.do_train:
utils.rmkdir(config.model_dir)
model_runner = ModelRunner(config, tasks)
if config.do_train:
heading("Start training")
model_runner.train()
utils.log()
if config.do_eval:
heading("Run dev set evaluation")
results.append(model_runner.evaluate())
write_results(config, results)
if config.write_test_outputs and trial <= config.n_writes_test:
heading("Running on the test set and writing the predictions")
for task in tasks:
# Currently only writing preds for GLUE and SQuAD 2.0 is supported
if task.name in ["cola", "mrpc", "mnli", "sst", "rte", "qnli", "qqp",
"sts"]:
for split in task.get_test_splits():
model_runner.write_classification_outputs([task], trial, split)
elif task.name == "squad":
scorer = model_runner.evaluate_task(task, "test", False)
scorer.write_predictions()
preds = utils.load_json(config.qa_preds_file("squad"))
null_odds = utils.load_json(config.qa_na_file("squad"))
for q, _ in preds.items():
if null_odds[q] > config.qa_na_threshold:
preds[q] = ""
utils.write_json(preds, config.test_predictions(
task.name, "test", trial))
else:
utils.log("Skipping task", task.name,
"- writing predictions is not supported for this task")
if trial != config.num_trials and (not config.keep_all_models):
utils.rmrf(config.model_dir)
trial += 1
# exporting the model
if config.export_dir:
# with tf.variable_scope(tf.get_variable_scope(), reuse=True):
# model_runner = ModelRunner(config, tasks)
# tf.gfile.MakeDirs(config.export_dir)
# checkpoint_path = os.path.join(config.init_checkpoint, "model.ckpt-6315")
# squad_serving_input_fn = (
# build_squad_serving_input_fn(config.max_seq_length))
# utils.log("Starting to export model.")
# subfolder = model_runner._estimator.export_saved_model(
# export_dir_base=os.path.join(config.export_dir, "saved_model"),
# serving_input_receiver_fn=squad_serving_input_fn)
tf.get_variable_scope().reuse_variables()
model_runner = ModelRunner(config, tasks)
tf.gfile.MakeDirs(config.export_dir)
checkpoint_path = os.path.join(config.init_checkpoint, "model.ckpt-6315")
squad_serving_input_fn = (
build_squad_serving_input_fn(config.max_seq_length))
utils.log("Starting to export model.")
subfolder = model_runner._estimator.export_saved_model(
export_dir_base=os.path.join(config.export_dir, "saved_model"),
serving_input_receiver_fn=squad_serving_input_fn)
