def solveIndividualRiddles(detectionFolder, prefix, allSeedsDictionary,
inferenceFolder, seedsCentralityFile, pipelineStage,
imageNum, API_USED):
import pslModelOneNewOptimization_v2 as pslOne
from misc import WordWeightsOptimization2
sumIndividualAccuracy = 0
trainingImage = detectionFolder + prefix + "_" + str(imageNum) + ".txt"
WordWeightsOptimization2.VERBOSE = False
reorderedSeedsFiles = WordWeightsOptimization2.reorderWeightsBasedOnPopularity(allSeedsDictionary, \
detectionFolder, prefix, int(imageNum), int(imageNum), inferenceFolder, API_USED)
reweightedSeedsFileName = reorderedSeedsFiles[0]
print("\treweighting seeds completed..")
if pipelineStage == "clarifai":
## Note: We will not do parallel processing for this
centroid = conceptnet_util.calculateWord2vecCentroidAndHighestAcc(
allSeedsDictionary, reweightedSeedsFileName)
return [centroid, reweightedSeedsFileName]
#### Step 1: Merge targets from different seeds.
mergeTargets.VERBOSE = False
[sortedScoreAndIndexList, targetWordsList, targetWordsDictonary,seedsDetected_weights,\
orderedSeedWordsList,allSeedsDictionary] = mergeTargets.mergeTargetsFromDetectedSeeds(\
reweightedSeedsFileName, seedsCentralityFile,1500)
print("\tmerging targets completed..")
if pipelineStage == "merge":
## Note: We will not do parallel processing for this
return [
sortedScoreAndIndexList, targetWordsList, targetWordsDictonary
]
#### Step 2: cluster the merged set of targets
[sortedScoreAndIndexList,pairwiseDistancesTargetWords] = clusterTargets.returnClustersFast(\
sortedScoreAndIndexList, targetWordsList, targetWordsDictonary, orderedSeedWordsList, 2500)
print(pairwiseDistancesTargetWords.shape)
print("\tclustering targets completed..")
#### Step 3: create 1-word and 2-word model
pslOne.VERBOSE = False
finalReorderedTargetsFileName = pslOne.optimizeAndInferConceptsModelOneNew(\
allSeedsDictionary,seedsDetected_weights,\
orderedSeedWordsList,reweightedSeedsFileName,sortedScoreAndIndexList, targetWordsList, targetWordsDictonary, \
pairwiseDistancesTargetWords)
[acc,
simWord] = calculateRelativeAccuracy(prefix,
finalReorderedTargetsFileName, 20)
print('\taccuracy %g' % acc)
sumIndividualAccuracy = sumIndividualAccuracy + acc
return sumIndividualAccuracy
def solveIndividualRiddles(detectionFolder,prefix,allSeedsDictionary,inferenceFolder,seedsCentralityFile,
pipelineStage, imageNum, API_USED, vocab=None):
sumIndividualAccuracy = 0;
trainingImage = detectionFolder+prefix+"_"+str(imageNum)+".txt";
WordWeightsOptimization2.VERBOSE = False;
reweightedSeedsFileName = normalizeSeedsAndWeights(allSeedsDictionary,\
detectionFolder,prefix,int(imageNum),inferenceFolder,API_USED);
print("\treweighting seeds completed..");
if reweightedSeedsFileName == None:
if pipelineStage == "clarifai":
return [None, None];
elif pipelineStage == "merge":
return [None,None, None];
else:
return None;
if pipelineStage == "clarifai":
## Note: We will not do parallel processing for this
centroid = conceptnet_util.calculateWord2vecCentroidAndHighestAcc(allSeedsDictionary,reweightedSeedsFileName, vocab);
return [centroid,reweightedSeedsFileName];
#### Step 1: Merge targets from different seeds.
mergeTargets.VERBOSE= False;
[sortedScoreAndIndexList, targetWordsList, targetWordsDictonary,seedsDetected_weights,\
orderedSeedWordsList,allSeedsDictionary] = mergeTargets.mergeTargetsFromDetectedSeeds(\
reweightedSeedsFileName, seedsCentralityFile,1500);
print("\tmerging targets completed..");
if pipelineStage == "merge":
## Note: We will not do parallel processing for this
return [sortedScoreAndIndexList, targetWordsList, targetWordsDictonary];
#### Step 2: cluster the merged set of targets
[sortedScoreAndIndexList,pairwiseDistancesTargetWords] = clusterTargets.returnClustersFast(\
sortedScoreAndIndexList, targetWordsList, targetWordsDictonary, orderedSeedWordsList, 2500);
print(pairwiseDistancesTargetWords.shape);
print("\tclustering targets completed..");
#### Step 3: create 1-word and 2-word model
pslOne.VERBOSE = False;
finalReorderedTargetsFileName = pslOne.optimizeAndInferConceptsModelOneNew(\
allSeedsDictionary,seedsDetected_weights,\
orderedSeedWordsList,reweightedSeedsFileName,sortedScoreAndIndexList, targetWordsList, targetWordsDictonary, \
pairwiseDistancesTargetWords);
[acc,simWord]= calculateRelativeAccuracy(prefix, finalReorderedTargetsFileName, 20);
print('\taccuracy %g' % acc);
sumIndividualAccuracy = sumIndividualAccuracy+acc;
return sumIndividualAccuracy;
def solveIndividualRiddles(detectionFolder, prefix, allSeedsDictionary,
inferenceFolder, seedsCentralityFile, part):
sumIndividualAccuracy = 0
#for part in range(1,5):
trainingImage = detectionFolder + prefix + "_" + str(part) + ".txt"
WordWeightsOptimization2.VERBOSE = False
reorderedSeedsFiles = WordWeightsOptimization2.reorderWeightsBasedOnPopularity(allSeedsDictionary, \
detectionFolder, prefix, int(part), int(part), inferenceFolder)
reweightedSeedsFileName = reorderedSeedsFiles[0]
print("\tmerging targets completed..")
#### Step 1: Merge targets from different seeds.
mergeTargets.VERBOSE = False
[sortedScoreAndIndexList, targetWordsList, targetWordsDictonary,seedsDetected_weights,\
orderedSeedWordsList,allSeedsDictionary] = mergeTargets.mergeTargetsFromDetectedSeeds(\
reweightedSeedsFileName, seedsCentralityFile,1500)
print("\tmerging targets completed..")
#### Step 2: cluster the merged set of targets
[sortedScoreAndIndexList,pairwiseDistancesTargetWords] = clusterTargets.returnClustersFast(\
sortedScoreAndIndexList, targetWordsList, targetWordsDictonary, orderedSeedWordsList, 2500)
print(pairwiseDistancesTargetWords.shape)
print("\tclustering targets completed..")
#### Step 3: create 1-word and 2-word model
pslOne.VERBOSE = False
finalReorderedTargetsFileName = pslOne.optimizeAndInferConceptsModelOneNew(\
allSeedsDictionary,seedsDetected_weights,\
orderedSeedWordsList,reweightedSeedsFileName,sortedScoreAndIndexList, targetWordsList, targetWordsDictonary, \
pairwiseDistancesTargetWords)
[acc,
simWord] = calculateRelativeAccuracy(prefix,
finalReorderedTargetsFileName, 20)
print('\taccuracy %g' % acc)
sumIndividualAccuracy = sumIndividualAccuracy + acc
return sumIndividualAccuracy
break
#try:
print('Iteration for prefix:%g\t%s\n' %
(parameterSearchTries, prefix))
for part in partsList:
trainingImage = detectionFolder + prefix + "_" + part + ".txt"
WordWeightsOptimization2.VERBOSE = False
reorderedSeedsFiles = WordWeightsOptimization2.reorderWeightsBasedOnPopularity(allSeedsDictionary, \
detectionFolder, prefix, int(part), int(part),"intermediateFiles/opt_test/", API_USED)
reweightedSeedsFileName = reorderedSeedsFiles[0]
print("\tmerging targets completed..")
#### Step 1: Merge targets from different seeds.
mergeTargets.VERBOSE = False
[sortedScoreAndIndexList, targetWordsList, targetWordsDictonary,seedsDetected_weights,\
orderedSeedWordsList,allSeedsDictionary] = mergeTargets.mergeTargetsFromDetectedSeeds(\
reweightedSeedsFileName, seedsCentralityFile,1500)
print("\tmerging targets completed..")
#### Step 2: cluster the merged set of targets
[sortedScoreAndIndexList,pairwiseDistancesTargetWords] = clusterTargets.returnClustersFast(\
sortedScoreAndIndexList, targetWordsList, targetWordsDictonary, orderedSeedWordsList, 2500)
print(pairwiseDistancesTargetWords.shape)
print("\tclustering targets completed..")
#### Step 3: create 1-word and 2-word model
pslModelOneNewOptimization_v2.VERBOSE = False
finalReorderedTargetsFileName = pslModelOneNewOptimization_v2.optimizeAndInferConceptsModelOneNew(\
allSeedsDictionary,seedsDetected_weights,\
orderedSeedWordsList,reweightedSeedsFileName,sortedScoreAndIndexList, targetWordsList, targetWordsDictonary, \
pairwiseDistancesTargetWords)
########################################################################
if __name__ == "__main__":
# sys.argv[1] = reweighted set of seeds for an image
# sys.argv[2] = detected seeds to modified map file
# sys.argv[3] = targets for each seed
if len(sys.argv) >= 4:
VERBOSE = True
mergeTargets.VERBOSE = True
for index in range(1, 5):
reweightedSeedsFile = sys.argv[1] + "_" + str(index) + "_c.txt"
#### Step 1: Merge targets from different seeds.
[sortedScoreAndIndexList, targetWordsList, targetWordsDictonary,seedsDetected_weights,\
orderedSeedWordsList,allSeedsDictionary] = mergeTargets.mergeTargetsFromDetectedSeeds(\
reweightedSeedsFile, sys.argv[2], 1500)
print("\tmerging targets completed..")
#### Step 2: cluster the merged set of targets
[clusters,sortedScoreAndIndexList,Z,pairwiseDistancesTargetWords] = clusterTargets.returnClusters(sortedScoreAndIndexList, \
targetWordsList, targetWordsDictonary, orderedSeedWordsList, 2500)
print(pairwiseDistancesTargetWords.shape)
print("\tclustering targets completed..")
#### Step 3: create 1-word and 2-word model
optimizeAndInferConceptsModelOneNew(allSeedsDictionary,seedsDetected_weights,orderedSeedWordsList,reweightedSeedsFile,\
sortedScoreAndIndexList, targetWordsList, targetWordsDictonary, pairwiseDistancesTargetWords)
else:
VERBOSE = True
reweightedSeedsFile = sys.argv[1]
def mergeTargetsAndReturnClusters(reweightedSeedsFileImg,
allSeedsMapFile,
maxTargets=3000):
[sortedScoreAndIndexList, targetWordsList, targetWordsDictionary,seedsDetected_weights,seedWordsList,allSeedsDictionary] = \
mergeTargetsFromDetectedSeeds(reweightedSeedsFileImg,allSeedsMapFile)
return returnClusters(sortedScoreAndIndexList, targetWordsList,
targetWordsDictionary, seedWordsList, maxTargets)
if __name__ == "__main__":
# sys.arv[1] = reweighted set of seeds for an image
# sys.argv[2] = detected seeds to modified map file
# sys.argv[3] = clustering choice
[sortedScoreAndIndexList, targetWordsList, targetWordsDictionary,seedsDetected_weights,seedWordsList,allSeedsDictionary] = \
mergeTargets.mergeTargetsFromDetectedSeeds(sys.argv[1], sys.argv[2], 1, 0.1)
if sys.argv[3] == "hac":
[clusters,modifiedSortedScoreAndIndexList,Z,pairwiseDistancesTargetWords] = returnClusters(sortedScoreAndIndexList, targetWordsList, \
targetWordsDictionary,seedWordsList, 3000)
print clusters.shape
for i in range(clusters.shape[0]):
print targetWordsList[modifiedSortedScoreAndIndexList[i]
[0]], "\t", clusters[i], "\t**"
labelsList = []
for c in range(0, len(modifiedSortedScoreAndIndexList)):
try:
clusterIndex = clusters[c]
indexAndScore = modifiedSortedScoreAndIndexList[c]