def makeInputString(filesToSearch):
daytofilesmap = defdic(list)
for afile in filesToSearch:
yearmonthday = afile[0:8]
daytofilesmap[yearmonthday].append(afile)
# basepath = "/user/pdhakshi/SIE_DATA/BY_MULTIPARAMS/"
basepath = "/user/pdhakshi/pigouts/smart_data_2months/"
outputfilelist = []
for (aday, filelist) in daytofilesmap.items():
outputfilelist.append("%s/{%s}.gz/*.gz" % (aday, ",".join(filelist)))
return basepath + "{" + ",".join(outputfilelist) + "}"
def parseDateField(parser, dateRange):
datePeriods = dateRange.split(",")
days = []
for period in datePeriods:
rangevalue = period.split("-")
if len(rangevalue) == 1:
days.append(rangevalue[0])
elif len(rangevalue) == 2:
days = days + map (str, range(int(rangevalue[0]), int(rangevalue[1])+1))
else:
printHelpExit(parser, "Date period has more than one hiphen (-)")
yearmonthToDayMap= defdic(list)
for aday in days:
yearmonth = aday[0:6]
yearmonthToDayMap[yearmonth].append(aday)
return yearmonthToDayMap
def get_bbox_ious(bb_preds, bb_annos, mask, id2mean, thresh=0.5):
''' bb_preds = N(?) wise bb prediction tensor
'''
# import pdb
# pdb.set_trace()
bb_preds = var2np(bb_preds)
bb_annos = var2np(bb_annos)
# if bb_annos.size == 0:
# return {}
mask = var2np(mask)
bb_preds = bb_preds.reshape(-1, 4)
bb_annos = bb_annos.reshape(-1, 4)
# Must convert BOTH back to normal extent..
uniques = [u for u in np.unique(mask) if u not in [255]] # [0, 255]]
inters = defdic(float)
unions = defdic(float)
ious = defdic(list)
ok_pred = defdic(float)
total_pred = defdic(float)
def map_back(arr, meanw, meanh):
'''Convert back to xy space'''
arr[:, 2] = np.exp(arr[:, 2])
arr[:, 3] = np.exp(arr[:, 3])
arr[:, 0] *= meanw
arr[:, 1] *= meanh
arr[:, 2] *= meanw
arr[:, 3] *= meanh
if bb_preds.size == 0:
return inters, unions, ok_pred, total_pred
for u in uniques:
inds = mask == u
cwise_preds = bb_preds[inds]
cwise_annos = bb_annos[inds]
uind = str(int(u))
if roll and uind not in rolling_anno:
rolling_anno[uind] = _create_xy_list()
rolling_anno_post[uind] = _create_xy_list()
rolling_pred[uind] = _create_xy_list()
rolling_pred_post[uind] = _create_xy_list()
rolling_iou[uind] = _create_iou_dict()
for bb_pred, bb_anno in zip(cwise_preds, cwise_annos):
for xywh, ix in zip(['x', 'y', 'w', 'h'], [0, 1, 2, 3]):
rolling_anno[uind][xywh].append(float(bb_anno[ix]))
rolling_pred[uind][xywh].append(float(bb_pred[ix]))
means = id2mean[u]
meanh = means['h']
meanw = means['w']
map_back(cwise_preds, meanw, meanh)
map_back(cwise_annos, meanw, meanh)
for bb_pred, bb_anno in zip(cwise_preds, cwise_annos):
inters_, unis = get_bbox_iou(bb_pred, bb_anno, True)
if roll:
for xywh, ix in zip(['x', 'y', 'w', 'h'], [0, 1, 2, 3]):
rolling_anno_post[uind][xywh].append(float(bb_anno[ix]))
rolling_pred_post[uind][xywh].append(float(bb_pred[ix]))
rolling_iou[uind]['inters'] += inters_
rolling_iou[uind]['union'] += unis
rolling_iou[uind]['iou'].append(inters_ / unis)
# tqdm.tqdm.write("Prediction/anno:\n\t{}\n\t{}".format(bb_pred, bb_anno))
# tqdm.tqdm.write("IOU:\t{}".format(inters_ / unis))
# tqdm.tqdm.write(
# "Rolling: {} ({})".format(
# rolling_iou['inters'] / rolling_iou['union'],
# np.mean(rolling_iou['iou'])))
if inters_ > unis:
import pdb
pdb.set_trace()
inters[u] += inters_
unions[u] += unis
iou = inters_ / unis
ious[u].append(iou)
if iou > thresh:
ok_pred[u] += 1
ok_pred[u] += 0
total_pred[u] += 1
if roll:
with open('valdata/rolling_pred_{}.json'.format(model_name),
'w+') as f:
json.dump(rolling_pred, f)
with open('valdata/rolling_anno_{}.json'.format(model_name),
'w+') as f:
json.dump(rolling_anno, f)
with open('valdata/rolling_pred_post_{}.json'.format(model_name),
'w+') as f:
json.dump(rolling_pred_post, f)
with open('valdata/rolling_anno_post_{}.json'.format(model_name),
'w+') as f:
json.dump(rolling_anno_post, f)
with open('valdata/rolling_iou_{}.json'.format(model_name), 'w+') as f:
json.dump(rolling_iou, f)
return inters, unions, ious, ok_pred, total_pred
