def outputSvmFormatFile(fReader, LAST_ROW, outF, for_test, idxDic):
count = 0
preVN = -1
firstCol = -1
w = -1
vnFeatDic = {}
upcVnList = []
scVnList = []
ddVnList = []
flnVnList = []
next(fReader, None) # skip header
for row in fReader:
curVN = row[idxDic['VN_IDX']]
count = count + 1
if (count % 10000 == 0):
print('--count -> ' + str(count))
print('VisitNumber ->' + str(curVN))
print(vnFeatDic)
if ((preVN != curVN and len(vnFeatDic) > 0) or count == LAST_ROW):
## ************* CUSTOM FEATURES START ************* ##
vnFeatDic[emStart +
extraMetricsList.index('records')] = len(ddVnList)
if len(ddVnList) > 1:
vnFeatDic[emStart + extraMetricsList.index('uniqDD')] = len(
np.unique(ddVnList))
vnFeatDic[emStart + extraMetricsList.index('uniqFLN')] = len(
np.unique(flnVnList))
vnFeatDic[emStart +
extraMetricsList.index('sumSC')] = sum(scVnList)
# SC-VN Stat features
if len(scVnList) > 1:
vnFeatDic[emStart +
extraMetricsList.index('meanSC')] = np.mean(scVnList)
vnFeatDic[emStart +
extraMetricsList.index('medianSC')] = np.median(
scVnList)
vnFeatDic[emStart +
extraMetricsList.index('maxSC')] = max(scVnList)
vnFeatDic[emStart + extraMetricsList.index('sdSC')] = np.std(
scVnList, ddof=1)
vnFeatDic[emStart +
extraMetricsList.index('skewSC')] = stats.skew(
scVnList)
vnFeatDic[emStart +
extraMetricsList.index('kurtSC')] = stats.kurtosis(
scVnList)
vnFeatDic[emStart +
extraMetricsList.index('iqrSC')] = np.subtract(
*np.percentile(scVnList, [75, 25]))
# DD-VN-SC Stat features
ddVnScList = [
vnFeatDic.get(x) for x in list(range(ddStart, flnStart))
if vnFeatDic.get(x) != None
]
if len(ddVnScList) > 1:
vnFeatDic[emStart +
extraMetricsList.index('meanDDSC')] = np.mean(
ddVnScList)
vnFeatDic[emStart +
extraMetricsList.index('medianDDSC')] = np.median(
ddVnScList)
vnFeatDic[emStart +
extraMetricsList.index('maxDDSC')] = max(ddVnScList)
vnFeatDic[emStart + extraMetricsList.index('sdDDSC')] = np.std(
ddVnScList, ddof=1)
vnFeatDic[emStart +
extraMetricsList.index('skewDDSC')] = stats.skew(
ddVnScList)
vnFeatDic[emStart +
extraMetricsList.index('kurtDDSC')] = stats.kurtosis(
ddVnScList)
vnFeatDic[emStart +
extraMetricsList.index('iqrDDSC')] = np.subtract(
*np.percentile(ddVnScList, [75, 25]))
# FLN-VN Stat features
flnVnScList = [
vnFeatDic.get(x) for x in list(range(flnStart, upcStart))
if vnFeatDic.get(x) != None
]
if len(flnVnScList) > 1:
vnFeatDic[emStart +
extraMetricsList.index('meanFLNSC')] = np.mean(
flnVnScList)
vnFeatDic[emStart +
extraMetricsList.index('medianFLNSC')] = np.median(
flnVnScList)
vnFeatDic[emStart + extraMetricsList.index('maxFLNSC')] = max(
flnVnScList)
vnFeatDic[emStart +
extraMetricsList.index('sdFLNSC')] = np.std(
flnVnScList, ddof=1)
vnFeatDic[emStart +
extraMetricsList.index('skewFLNSC')] = stats.skew(
flnVnScList)
vnFeatDic[emStart + extraMetricsList.index(
'kurtFLNSC')] = stats.kurtosis(flnVnScList)
vnFeatDic[emStart +
extraMetricsList.index('iqrFLNSC')] = np.subtract(
*np.percentile(flnVnScList, [75, 25]))
# DD's with top two SC; Cond: SC1>1 and SC2>SC1/3
ddVnFeatDicSub = {
k: vnFeatDic.get(k)
for k in list(range(ddStart, flnStart))
if vnFeatDic.get(k) != None
}
if len(ddVnFeatDicSub) > 1:
sorted_ddVnFeatDicSub = sorted(ddVnFeatDicSub.items(),
key=operator.itemgetter(1))
ddVnMax1 = sorted_ddVnFeatDicSub[-1]
if ddVnMax1[1] > 1: # Cond1: SC1>1
vnFeatDic[emStart +
extraMetricsList.index('DDMaxSC1')] = ddVnMax1[0]
if len(ddVnFeatDicSub) > 2:
ddVnMax2 = sorted_ddVnFeatDicSub[-2]
if ddVnMax2[1] > 1 and ddVnMax2[1] > ddVnMax1[
1] / 3: # Cond2: SC2>1 and SC2>SC1/3
vnFeatDic[emStart + extraMetricsList.index(
'DDMaxSC2')] = ddVnMax2[0]
# DD adjacency usage
ddVnFeatNewDic = {
k - ddStart: v
for k, v in ddVnFeatDicSub.items()
} # sub ddStart from all keys of ddVnFeatDicSub
cosine_dist_all_list = get_sp_weights_from_matrix(ddVnFeatNewDic,
cosine_dist_all,
multiply_10=True)
add_metrics_to_main_dict(vnFeatDic, cosine_dist_all_list,
spAllStart, spMetricList)
sp_k_1_list = get_sp_weights_from_matrix(ddVnFeatNewDic,
sp_all_k_1)
add_metrics_to_main_dict(vnFeatDic, sp_k_1_list,
spAllStart + len(spMetricList),
spMetricList)
sp_k_3_list = get_sp_weights_from_matrix(ddVnFeatNewDic,
sp_all_k_3)
add_metrics_to_main_dict(vnFeatDic, sp_k_3_list,
spAllStart + 2 * len(spMetricList),
spMetricList)
spl_k_1_list = get_sp_length_from_dict(ddVnFeatNewDic, spl_all_k_1)
add_metrics_to_main_dict(vnFeatDic, spl_k_1_list,
spAllStart + 3 * len(spMetricList),
spMetricList)
spl_k_3_list = get_sp_length_from_dict(ddVnFeatNewDic, spl_all_k_3)
add_metrics_to_main_dict(vnFeatDic, spl_k_3_list,
spAllStart + 4 * len(spMetricList),
spMetricList)
# 3 features for UPC length
upcVnLenList = [len(j) for j in upcVnList]
if len(upcVnLenList) > 1:
vnFeatDic[emStart +
extraMetricsList.index('sdUpcLen')] = np.std(
upcVnLenList, ddof=1)
vnFeatDic[emStart +
extraMetricsList.index('skewUpcLen')] = stats.skew(
upcVnLenList)
vnFeatDic[emStart + extraMetricsList.index(
'kurtUpcLen')] = stats.kurtosis(upcVnLenList)
# upcLenList freq add
if upcVnLenList:
upcLenDic = {(upcLenStart + upcLenList.index(i)):
upcVnLenList.count(i)
for i in set(upcVnLenList)}
vnFeatDic.update(upcLenDic)
## ************* CUSTOM FEATURES END ************* ##
vnFeatDic[wStart + wList.index(w)] = 1 # WEEKDAY
#Start writing the 'vnFeatDic' to the output file
value = ['']
vnFeatDicSorted = collections.OrderedDict(sorted(
vnFeatDic.items()))
for kk, vv in vnFeatDicSorted.items():
value.append(' %s:%s' % (str(kk), str(vv)))
outF.write(str(firstCol) + ''.join(value) + '\n')
# Process the dict for VN and output a single line for that VN
# reset for next VN
vnFeatDic = {}
upcVnList = []
scVnList = []
ddVnList = []
flnVnList = []
if count == LAST_ROW:
break
if for_test:
firstCol = row[idxDic['VN_IDX']] # set VN for TEST
else:
firstCol = row[idxDic['TT_IDX']] # set TT for TRAIN
w = row[idxDic['W_IDX']] # doesn't change for a VN
sc = int(row[idxDic['SC_IDX']])
scVnList.append(sc)
upc = row[idxDic['UPC_IDX']]
if upc.isdigit():
upcVnList.append(upc)
upcKeys = returnUpcKeys(upc)
for ke in upcKeys:
keIndex = upcStart + upcList.index(ke)
if keIndex not in vnFeatDic:
vnFeatDic[keIndex] = sc
else:
vnFeatDic[keIndex] += sc
dd = row[idxDic['DD_IDX']]
if dd == 'MENSWEAR':
dd = 'MENS WEAR'
ddVnList.append(dd)
ddIdx = ddStart + ddList.index(dd)
if ddIdx not in vnFeatDic:
vnFeatDic[ddIdx] = sc
else:
vnFeatDic[ddIdx] += sc
fln = row[idxDic['FLN_IDX']]
if fln: #for empty fln # and sc>0:
flnVnList.append(fln)
flnIdx = flnStart + flnList.index(fln)
if flnIdx not in vnFeatDic:
vnFeatDic[flnIdx] = sc
else:
vnFeatDic[flnIdx] += sc
preVN = curVN
def outputSvmFormatFile(fReader,LAST_ROW,outF,for_test,idxDic):
count = 0
preVN = -1
firstCol = -1
w = -1
vnFeatDic = {}
upcVnList = []
scVnList = []
ddVnList = []
flnVnList = []
next(fReader, None) # skip header
for row in fReader:
curVN = row[idxDic['VN_IDX']]
count = count + 1
if(count % 10000 == 0):
print('--count -> ' + str(count))
print('VisitNumber ->' + str(curVN))
print(vnFeatDic)
if((preVN != curVN and len(vnFeatDic)>0) or count == LAST_ROW):
## ************* CUSTOM FEATURES START ************* ##
vnFeatDic[emStart + extraMetricsList.index('records')] = len(ddVnList)
if len(ddVnList)>1:
vnFeatDic[emStart + extraMetricsList.index('uniqDD')] = len(np.unique(ddVnList))
vnFeatDic[emStart + extraMetricsList.index('uniqFLN')] = len(np.unique(flnVnList))
vnFeatDic[emStart + extraMetricsList.index('sumSC')] = sum(scVnList)
# SC-VN Stat features
if len(scVnList)>1:
vnFeatDic[emStart + extraMetricsList.index('meanSC')] = np.mean(scVnList)
vnFeatDic[emStart + extraMetricsList.index('medianSC')] = np.median(scVnList)
vnFeatDic[emStart + extraMetricsList.index('maxSC')] = max(scVnList)
vnFeatDic[emStart + extraMetricsList.index('sdSC')] = np.std(scVnList,ddof=1)
vnFeatDic[emStart + extraMetricsList.index('skewSC')] = stats.skew(scVnList)
vnFeatDic[emStart + extraMetricsList.index('kurtSC')] = stats.kurtosis(scVnList)
vnFeatDic[emStart + extraMetricsList.index('iqrSC')] = np.subtract(*np.percentile(scVnList, [75, 25]))
# DD-VN-SC Stat features
ddVnScList = [vnFeatDic.get(x) for x in list(range(ddStart,flnStart)) if vnFeatDic.get(x)!=None]
if len(ddVnScList)>1:
vnFeatDic[emStart + extraMetricsList.index('meanDDSC')] = np.mean(ddVnScList)
vnFeatDic[emStart + extraMetricsList.index('medianDDSC')] = np.median(ddVnScList)
vnFeatDic[emStart + extraMetricsList.index('maxDDSC')] = max(ddVnScList)
vnFeatDic[emStart + extraMetricsList.index('sdDDSC')] = np.std(ddVnScList,ddof=1)
vnFeatDic[emStart + extraMetricsList.index('skewDDSC')] = stats.skew(ddVnScList)
vnFeatDic[emStart + extraMetricsList.index('kurtDDSC')] = stats.kurtosis(ddVnScList)
vnFeatDic[emStart + extraMetricsList.index('iqrDDSC')] = np.subtract(*np.percentile(ddVnScList, [75, 25]))
# FLN-VN Stat features
flnVnScList = [vnFeatDic.get(x) for x in list(range(flnStart,upcStart)) if vnFeatDic.get(x)!=None]
if len(flnVnScList)>1:
vnFeatDic[emStart + extraMetricsList.index('meanFLNSC')] = np.mean(flnVnScList)
vnFeatDic[emStart + extraMetricsList.index('medianFLNSC')] = np.median(flnVnScList)
vnFeatDic[emStart + extraMetricsList.index('maxFLNSC')] = max(flnVnScList)
vnFeatDic[emStart + extraMetricsList.index('sdFLNSC')] = np.std(flnVnScList,ddof=1)
vnFeatDic[emStart + extraMetricsList.index('skewFLNSC')] = stats.skew(flnVnScList)
vnFeatDic[emStart + extraMetricsList.index('kurtFLNSC')] = stats.kurtosis(flnVnScList)
vnFeatDic[emStart + extraMetricsList.index('iqrFLNSC')] = np.subtract(*np.percentile(flnVnScList, [75, 25]))
# DD's with top two SC; Cond: SC1>1 and SC2>SC1/3
ddVnFeatDicSub = {k: vnFeatDic.get(k) for k in list(range(ddStart,flnStart)) if vnFeatDic.get(k)!=None}
if len(ddVnFeatDicSub) > 1:
sorted_ddVnFeatDicSub = sorted(ddVnFeatDicSub.items(), key=operator.itemgetter(1))
ddVnMax1 = sorted_ddVnFeatDicSub[-1]
if ddVnMax1[1] > 1: # Cond1: SC1>1
vnFeatDic[emStart + extraMetricsList.index('DDMaxSC1')] = ddVnMax1[0]
if len(ddVnFeatDicSub) > 2:
ddVnMax2 = sorted_ddVnFeatDicSub[-2]
if ddVnMax2[1] > 1 and ddVnMax2[1] > ddVnMax1[1]/3: # Cond2: SC2>1 and SC2>SC1/3
vnFeatDic[emStart + extraMetricsList.index('DDMaxSC2')] = ddVnMax2[0]
# DD adjacency usage
ddVnFeatNewDic = {k-ddStart:v for k,v in ddVnFeatDicSub.items()} # sub ddStart from all keys of ddVnFeatDicSub
cosine_dist_all_dd_list = get_sp_weights_from_matrix(ddVnFeatNewDic, cosine_dist_all_dd,multiply_10=True)
add_metrics_to_main_dict(vnFeatDic, cosine_dist_all_dd_list, spAllStart+0*len(spMetricList), spMetricList)
sp_all_dd_k_1_list = get_sp_weights_from_matrix(ddVnFeatNewDic, sp_all_dd_k_1)
add_metrics_to_main_dict(vnFeatDic, sp_all_dd_k_1_list, spAllStart+1*len(spMetricList), spMetricList)
sp_all_dd_k_3_list = get_sp_weights_from_matrix(ddVnFeatNewDic, sp_all_dd_k_3)
add_metrics_to_main_dict(vnFeatDic, sp_all_dd_k_3_list, spAllStart+2*len(spMetricList), spMetricList)
spl_all_dd_k_1_list = get_sp_length_from_dict(ddVnFeatNewDic, spl_all_dd_k_1)
add_metrics_to_main_dict(vnFeatDic, spl_all_dd_k_1_list, spAllStart+3*len(spMetricList), spMetricList)
spl_all_dd_k_3_list = get_sp_length_from_dict(ddVnFeatNewDic, spl_all_dd_k_3)
add_metrics_to_main_dict(vnFeatDic, spl_all_dd_k_3_list, spAllStart+4*len(spMetricList), spMetricList)
# UPC adjacency usage
upcVnFeatDicSub = {k: vnFeatDic.get(k) for k in list(range(upcStart,nextStart)) if vnFeatDic.get(k)!=None}
upcVnFeatNewDic = {k-upcStart:v for k,v in upcVnFeatDicSub.items()} # sub upcStart from all keys of upcVnFeatDicSub
cosine_dist_all_upc_list = get_sp_weights_from_matrix(upcVnFeatNewDic, cosine_dist_all_upc,multiply_10=True)
add_metrics_to_main_dict(vnFeatDic, cosine_dist_all_upc_list, spAllStart+5*len(spMetricList), spMetricList)
sp_all_upc_k_1_list = get_sp_weights_from_matrix(upcVnFeatNewDic, sp_all_upc_k_1)
add_metrics_to_main_dict(vnFeatDic, sp_all_upc_k_1_list, spAllStart+6*len(spMetricList), spMetricList)
sp_all_upc_k_3_list = get_sp_weights_from_matrix(upcVnFeatNewDic, sp_all_upc_k_3)
add_metrics_to_main_dict(vnFeatDic, sp_all_upc_k_3_list, spAllStart+7*len(spMetricList), spMetricList)
spl_all_upc_k_1_list = get_sp_length_from_dict(upcVnFeatNewDic, spl_all_upc_k_1)
add_metrics_to_main_dict(vnFeatDic, spl_all_upc_k_1_list, spAllStart+8*len(spMetricList), spMetricList)
spl_all_upc_k_3_list = get_sp_length_from_dict(upcVnFeatNewDic, spl_all_upc_k_3)
add_metrics_to_main_dict(vnFeatDic, spl_all_upc_k_3_list, spAllStart+9*len(spMetricList), spMetricList)
# UPC add neighbors via lookup in dict- knn_upc_k_3 # knnmatrix[i].nonzero()[1] # nearest neighbors(k) for i'th element in matrix
upcVnFeatDicUpdate={} # will hold original UPC and their KNeighbors
for upcKey,upcVal in upcVnFeatNewDic.items():
if upcVal>0:
knn_idx_list = knn_upc_k_3[upcKey].nonzero()[1].tolist() # indexes of other UPC which are neighbor
if upcKey in knn_idx_list:
knn_idx_list.remove(upcKey)
ll = len(knn_idx_list)
if upcKey not in upcVnFeatDicUpdate:
upcVnFeatDicUpdate[upcKey]=(1-ll/10)*upcVal #adding original member with 70% value
else:
upcVnFeatDicUpdate[upcKey]+=(1-ll/10)*upcVal
# add neighbors(k) to upcVnFeatDicUpdate dictionary with a fraction of 'v' value
for neigh in knn_idx_list:
if neigh not in upcVnFeatDicUpdate:
upcVnFeatDicUpdate[neigh]=0.1*upcVal
else:
upcVnFeatDicUpdate[neigh]+=0.1*upcVal
upcVnFeatDicUpdate = {k+upcStart:v for k,v in upcVnFeatDicUpdate.items()}
vnFeatDic.update(upcVnFeatDicUpdate)
# 3 features for UPC length
upcVnLenList = [len(j) for j in upcVnList]
if len(upcVnLenList)>1:
vnFeatDic[emStart + extraMetricsList.index('sdUpcLen')] = np.std(upcVnLenList,ddof=1)
vnFeatDic[emStart + extraMetricsList.index('skewUpcLen')] = stats.skew(upcVnLenList)
vnFeatDic[emStart + extraMetricsList.index('kurtUpcLen')] = stats.kurtosis(upcVnLenList)
# upcLenList freq add
if upcVnLenList:
upcLenDic = {(upcLenStart + upcLenList.index(i)):upcVnLenList.count(i) for i in set(upcVnLenList)}
vnFeatDic.update(upcLenDic)
## ************* CUSTOM FEATURES END ************* ##
vnFeatDic[wStart + wList.index(w)] = 1 # WEEKDAY
#Start writing the 'vnFeatDic' to the output file
value = ['']
vnFeatDicSorted = collections.OrderedDict(sorted(vnFeatDic.items()))
for kk, vv in vnFeatDicSorted.items():
value.append(' %s:%s' % (str(kk), str(vv)))
outF.write(str(firstCol) + ''.join(value) + '\n');# Process the dict for VN and output a single line for that VN
# reset for next VN
vnFeatDic = {}
upcVnList = []
scVnList = []
ddVnList = []
flnVnList = []
if count == LAST_ROW:
break
if for_test:
firstCol = row[idxDic['VN_IDX']] # set VN for TEST
else:
firstCol = row[idxDic['TT_IDX']] # set TT for TRAIN
w = row[idxDic['W_IDX']] # doesn't change for a VN
sc = int(row[idxDic['SC_IDX']])
scVnList.append(sc)
upc = row[idxDic['UPC_IDX']]
if upc.isdigit():
upcVnList.append(upc)
upcKeys = returnUpcKeys(upc)
for ke in upcKeys:
if (not for_test) or (for_test and ke in upcList):
keIndex = upcStart + upcList.index(ke)
if keIndex not in vnFeatDic:
vnFeatDic[keIndex] = sc
else:
vnFeatDic[keIndex] += sc
dd = row[idxDic['DD_IDX']]
if dd == 'MENSWEAR':
dd = 'MENS WEAR'
ddVnList.append(dd)
if (not for_test) or (for_test and dd in ddList):
ddIdx = ddStart + ddList.index(dd)
if ddIdx not in vnFeatDic:
vnFeatDic[ddIdx] = sc
else:
vnFeatDic[ddIdx] += sc
fln = row[idxDic['FLN_IDX']]
if fln: #for empty fln # and sc>0:
flnVnList.append(fln)
if (not for_test) or (for_test and fln in flnList):
flnIdx = flnStart + flnList.index(fln)
if flnIdx not in vnFeatDic:
vnFeatDic[flnIdx] = sc
else:
vnFeatDic[flnIdx] += sc
preVN = curVN
def outputSvmFormatFile(fReader,LAST_ROW,outF,for_test,idxDic):
count = 0
preVN = -1
firstCol = -1
w = -1
vnFeatDic = {}
upcList = []
scVnList = []
ddVnList = []
flnVnList = []
next(fReader, None) # skip header
for row in fReader:
curVN = row[idxDic['VN_IDX']]
count = count + 1
if(count % 10000 == 0):
print('--count -> ' + str(count))
print('VisitNumber ->' + str(curVN))
print(vnFeatDic)
if((preVN != curVN and len(vnFeatDic)>0) or count == LAST_ROW):
## ************* CUSTOM FEATURES START ************* ##
vnFeatDic[emStart + extraMetricsList.index('records')] = len(ddVnList)
vnFeatDic[emStart + extraMetricsList.index('uniqDD')] = len(np.unique(ddVnList))
vnFeatDic[emStart + extraMetricsList.index('uniqFLN')] = len(np.unique(flnVnList))
vnFeatDic[emStart + extraMetricsList.index('sumSC')] = sum(scVnList)
# SC-VN Stat features
if len(scVnList)>1:
vnFeatDic[emStart + extraMetricsList.index('meanSC')] = np.mean(scVnList)
vnFeatDic[emStart + extraMetricsList.index('medianSC')] = np.median(scVnList)
vnFeatDic[emStart + extraMetricsList.index('maxSC')] = max(scVnList)
vnFeatDic[emStart + extraMetricsList.index('sdSC')] = np.std(scVnList,ddof=1)
vnFeatDic[emStart + extraMetricsList.index('skewSC')] = stats.skew(scVnList)
vnFeatDic[emStart + extraMetricsList.index('kurtSC')] = stats.kurtosis(scVnList)
vnFeatDic[emStart + extraMetricsList.index('iqrSC')] = np.subtract(*np.percentile(scVnList, [75, 25]))
# DD-VN-SC Stat features
ddVnScList = [vnFeatDic.get(x) for x in list(range(ddStart,flnStart)) if vnFeatDic.get(x)!=None]
if len(ddVnScList)>1:
vnFeatDic[emStart + extraMetricsList.index('meanDDSC')] = np.mean(ddVnScList)
vnFeatDic[emStart + extraMetricsList.index('medianDDSC')] = np.median(ddVnScList)
vnFeatDic[emStart + extraMetricsList.index('maxDDSC')] = max(ddVnScList)
vnFeatDic[emStart + extraMetricsList.index('sdDDSC')] = np.std(ddVnScList,ddof=1)
vnFeatDic[emStart + extraMetricsList.index('skewDDSC')] = stats.skew(ddVnScList)
vnFeatDic[emStart + extraMetricsList.index('kurtDDSC')] = stats.kurtosis(ddVnScList)
vnFeatDic[emStart + extraMetricsList.index('iqrDDSC')] = np.subtract(*np.percentile(ddVnScList, [75, 25]))
# FLN-VN Stat features
flnVnScList = [vnFeatDic.get(x) for x in list(range(flnStart,nextStart)) if vnFeatDic.get(x)!=None]
if len(flnVnScList)>1:
vnFeatDic[emStart + extraMetricsList.index('meanFLNSC')] = np.mean(flnVnScList)
vnFeatDic[emStart + extraMetricsList.index('medianFLNSC')] = np.median(flnVnScList)
vnFeatDic[emStart + extraMetricsList.index('maxFLNSC')] = max(flnVnScList)
vnFeatDic[emStart + extraMetricsList.index('sdFLNSC')] = np.std(flnVnScList,ddof=1)
vnFeatDic[emStart + extraMetricsList.index('skewFLNSC')] = stats.skew(flnVnScList)
vnFeatDic[emStart + extraMetricsList.index('kurtFLNSC')] = stats.kurtosis(flnVnScList)
vnFeatDic[emStart + extraMetricsList.index('iqrFLNSC')] = np.subtract(*np.percentile(flnVnScList, [75, 25]))
# DD's with top two SC; Cond: SC1>1 and SC2>SC1/3
ddVnFeatDicSub = {k: vnFeatDic.get(k) for k in list(range(ddStart,flnStart)) if vnFeatDic.get(k)!=None}
if len(ddVnFeatDicSub) > 1:
sorted_ddVnFeatDicSub = sorted(ddVnFeatDicSub.items(), key=operator.itemgetter(1))
ddVnMax1 = sorted_ddVnFeatDicSub[-1]
if ddVnMax1[1] > 1: # Cond1: SC1>1
vnFeatDic[emStart + extraMetricsList.index('DDMaxSC1')] = ddVnMax1[0]
if len(ddVnFeatDicSub) > 2:
ddVnMax2 = sorted_ddVnFeatDicSub[-2]
if ddVnMax2[1] > 1 and ddVnMax2[1] > ddVnMax1[1]/3: # Cond2: SC2>1 and SC2>SC1/3
vnFeatDic[emStart + extraMetricsList.index('DDMaxSC2')] = ddVnMax2[0]
# DD adjacency usage
ddVnFeatNewDic = {k-ddStart:v for k,v in ddVnFeatDicSub.items()} # sub ddStart from all keys of ddVnFeatDicSub
cosine_dist_all_list = get_sp_weights_from_matrix(ddVnFeatNewDic, cosine_dist_all,multiply_10=True)
add_metrics_to_main_dict(vnFeatDic, cosine_dist_all_list, spAllStart, spMetricList)
sp_k_1_list = get_sp_weights_from_matrix(ddVnFeatNewDic, sp_all_k_1)
add_metrics_to_main_dict(vnFeatDic, sp_k_1_list, spAllStart+len(spMetricList), spMetricList)
sp_k_3_list = get_sp_weights_from_matrix(ddVnFeatNewDic, sp_all_k_3)
add_metrics_to_main_dict(vnFeatDic, sp_k_3_list, spAllStart+2*len(spMetricList), spMetricList)
spl_k_1_list = get_sp_length_from_dict(ddVnFeatNewDic, spl_all_k_1)
add_metrics_to_main_dict(vnFeatDic, spl_k_1_list, spAllStart+3*len(spMetricList), spMetricList)
spl_k_3_list = get_sp_length_from_dict(ddVnFeatNewDic, spl_all_k_3)
add_metrics_to_main_dict(vnFeatDic, spl_k_3_list, spAllStart+4*len(spMetricList), spMetricList)
# 3 features for UPC length
upcVnLenList = [len(j) for j in upcList]
if len(upcVnLenList)>1:
vnFeatDic[emStart + extraMetricsList.index('sdUpcLen')] = np.std(upcVnLenList,ddof=1)
vnFeatDic[emStart + extraMetricsList.index('skewUpcLen')] = stats.skew(upcVnLenList)
vnFeatDic[emStart + extraMetricsList.index('kurtUpcLen')] = stats.kurtosis(upcVnLenList)
# upcLenList freq add
if upcVnLenList:
upcLenDic = {(upcLenStart + upcLenList.index(i)):upcVnLenList.count(i) for i in set(upcVnLenList)}
vnFeatDic.update(upcLenDic)
## ************* CUSTOM FEATURES END ************* ##
vnFeatDic[wStart + wList.index(w)] = 1 # WEEKDAY
#Start writing the 'vnFeatDic' to the output file
value = ['']
vnFeatDicSorted = collections.OrderedDict(sorted(vnFeatDic.items()))
for kk, vv in vnFeatDicSorted.items():
value.append(' %s:%s' % (str(kk), str(vv)))
outF.write(str(firstCol) + ''.join(value) + '\n');# Process the dict for VN and output a single line for that VN
# reset for next VN
vnFeatDic = {}
upcList = []
scVnList = []
ddVnList = []
flnVnList = []
if count == LAST_ROW:
break
if for_test:
firstCol = row[idxDic['VN_IDX']] # set VN for TEST
else:
firstCol = row[idxDic['TT_IDX']] # set TT for TRAIN
w = row[idxDic['W_IDX']] # doesn't change for a VN
upc = row[idxDic['UPC_IDX']]
upcList.append(upc)
sc = int(row[idxDic['SC_IDX']])
scVnList.append(sc)
dd = row[idxDic['DD_IDX']]
if dd == 'MENSWEAR':
dd = 'MENS WEAR'
ddVnList.append(dd)
ddIdx = ddStart + ddList.index(dd)
if ddIdx not in vnFeatDic:
vnFeatDic[ddIdx] = sc
else:
vnFeatDic[ddIdx] = vnFeatDic[ddIdx] + sc
fln = row[idxDic['FLN_IDX']]
if fln: #for empty fln # and sc>0:
flnVnList.append(fln)
flnIdx = flnStart + flnList.index(fln)
if flnIdx not in vnFeatDic:
vnFeatDic[flnIdx] = sc
else:
vnFeatDic[flnIdx] = vnFeatDic[flnIdx] + sc
preVN = curVN