def makeInputString(filesToSearch): daytofilesmap = defdic(list) for afile in filesToSearch: yearmonthday = afile[0:8] daytofilesmap[yearmonthday].append(afile) # basepath = "/user/pdhakshi/SIE_DATA/BY_MULTIPARAMS/" basepath = "/user/pdhakshi/pigouts/smart_data_2months/" outputfilelist = [] for (aday, filelist) in daytofilesmap.items(): outputfilelist.append("%s/{%s}.gz/*.gz" % (aday, ",".join(filelist))) return basepath + "{" + ",".join(outputfilelist) + "}"
def parseDateField(parser, dateRange): datePeriods = dateRange.split(",") days = [] for period in datePeriods: rangevalue = period.split("-") if len(rangevalue) == 1: days.append(rangevalue[0]) elif len(rangevalue) == 2: days = days + map (str, range(int(rangevalue[0]), int(rangevalue[1])+1)) else: printHelpExit(parser, "Date period has more than one hiphen (-)") yearmonthToDayMap= defdic(list) for aday in days: yearmonth = aday[0:6] yearmonthToDayMap[yearmonth].append(aday) return yearmonthToDayMap
def get_bbox_ious(bb_preds, bb_annos, mask, id2mean, thresh=0.5): ''' bb_preds = N(?) wise bb prediction tensor ''' # import pdb # pdb.set_trace() bb_preds = var2np(bb_preds) bb_annos = var2np(bb_annos) # if bb_annos.size == 0: # return {} mask = var2np(mask) bb_preds = bb_preds.reshape(-1, 4) bb_annos = bb_annos.reshape(-1, 4) # Must convert BOTH back to normal extent.. uniques = [u for u in np.unique(mask) if u not in [255]] # [0, 255]] inters = defdic(float) unions = defdic(float) ious = defdic(list) ok_pred = defdic(float) total_pred = defdic(float) def map_back(arr, meanw, meanh): '''Convert back to xy space''' arr[:, 2] = np.exp(arr[:, 2]) arr[:, 3] = np.exp(arr[:, 3]) arr[:, 0] *= meanw arr[:, 1] *= meanh arr[:, 2] *= meanw arr[:, 3] *= meanh if bb_preds.size == 0: return inters, unions, ok_pred, total_pred for u in uniques: inds = mask == u cwise_preds = bb_preds[inds] cwise_annos = bb_annos[inds] uind = str(int(u)) if roll and uind not in rolling_anno: rolling_anno[uind] = _create_xy_list() rolling_anno_post[uind] = _create_xy_list() rolling_pred[uind] = _create_xy_list() rolling_pred_post[uind] = _create_xy_list() rolling_iou[uind] = _create_iou_dict() for bb_pred, bb_anno in zip(cwise_preds, cwise_annos): for xywh, ix in zip(['x', 'y', 'w', 'h'], [0, 1, 2, 3]): rolling_anno[uind][xywh].append(float(bb_anno[ix])) rolling_pred[uind][xywh].append(float(bb_pred[ix])) means = id2mean[u] meanh = means['h'] meanw = means['w'] map_back(cwise_preds, meanw, meanh) map_back(cwise_annos, meanw, meanh) for bb_pred, bb_anno in zip(cwise_preds, cwise_annos): inters_, unis = get_bbox_iou(bb_pred, bb_anno, True) if roll: for xywh, ix in zip(['x', 'y', 'w', 'h'], [0, 1, 2, 3]): rolling_anno_post[uind][xywh].append(float(bb_anno[ix])) rolling_pred_post[uind][xywh].append(float(bb_pred[ix])) rolling_iou[uind]['inters'] += inters_ rolling_iou[uind]['union'] += unis rolling_iou[uind]['iou'].append(inters_ / unis) # tqdm.tqdm.write("Prediction/anno:\n\t{}\n\t{}".format(bb_pred, bb_anno)) # tqdm.tqdm.write("IOU:\t{}".format(inters_ / unis)) # tqdm.tqdm.write( # "Rolling: {} ({})".format( # rolling_iou['inters'] / rolling_iou['union'], # np.mean(rolling_iou['iou']))) if inters_ > unis: import pdb pdb.set_trace() inters[u] += inters_ unions[u] += unis iou = inters_ / unis ious[u].append(iou) if iou > thresh: ok_pred[u] += 1 ok_pred[u] += 0 total_pred[u] += 1 if roll: with open('valdata/rolling_pred_{}.json'.format(model_name), 'w+') as f: json.dump(rolling_pred, f) with open('valdata/rolling_anno_{}.json'.format(model_name), 'w+') as f: json.dump(rolling_anno, f) with open('valdata/rolling_pred_post_{}.json'.format(model_name), 'w+') as f: json.dump(rolling_pred_post, f) with open('valdata/rolling_anno_post_{}.json'.format(model_name), 'w+') as f: json.dump(rolling_anno_post, f) with open('valdata/rolling_iou_{}.json'.format(model_name), 'w+') as f: json.dump(rolling_iou, f) return inters, unions, ious, ok_pred, total_pred