def __init__(self, parent = None,name = None,modal = 0,fl = 0):
"""Set up the dialogue.
"""
Source.__init__(self,parent,name,modal,fl)
self.setCaption(self.tr("Edit Source"))
viewList= Control.Global.getListSelected()
so= viewList[0]
self.fromSourceToWidget(so) # Make a copy and fill in fields
self.wNext.setEnabled(0)
self.wPrev.setEnabled(0)
self.wNext.hide()
self.wPrev.hide()
def __init__(self,parent = None,name = None,modal = 0,fl = 0):
"""Set up the dialogue.
"""
Source.__init__(self,parent,name,modal,fl)
self.setCaption(self.tr("New Source"))
self.wNum.setText(self._sourceL.getNewRef()) # get next ref number
self.wNext.setEnabled(0)
self.wPrev.setEnabled(0)
self.wNext.hide()
self.wPrev.hide()
self._cuven= None
self.connect(self.wRule,SIGNAL("activated(int)"),
self.slotRuleActivated)
def calculateNewSources(self):
self.newSources = {}
for s in self.currentSources:
#print("Replacing: " + str(s))
sizeForS = len(self.currentSources.get(s).destinations)
eDis = 0
newDis = 0
xCoord = 0.0
yCoord = 0.0
xCoordw = 0.0
yCoordw = 0.0
demandSum = 0
for ds in self.currentSources.get(s).destinations:
demandSum += ds.demand
eDis += self.distanceMap[self.currentSources[s].destinationId][
ds.id]
xCoordw += float(ds.demand) * ds.x
yCoordw += float(ds.demand) * ds.y
xCoord += ds.x
yCoord += ds.y
if (xCoord != 0.0 and sizeForS != 0):
xCoordw = (xCoordw / demandSum)
yCoordw = (yCoordw / demandSum)
xCoord = xCoord / sizeForS
yCoord = yCoord / sizeForS
#print("weighted =>" + str(xCoordw) + ", " + str(yCoordw))
#print("Non weighted =>" + str(xCoord) + ", " + str(yCoord))
cd = self.findClosestDestination(xCoordw, yCoordw)
for ds in self.currentSources.get(s).destinations:
newDis += self.distanceMap[cd.id][ds.id]
if newDis < eDis:
self.newSources[s] = Source.Source(cd.x, cd.y, cd.id, s)
else:
self.newSources[s] = self.currentSources[s]
def calculateNewSourcesActual(self):
self.newSources = {}
for s in self.currentSources:
# print("Replacing: " + str(s))
sizeForS = len(self.currentSources.get(s).destinations)
xCoord = 0.0
yCoord = 0.0
xCoordw = 0.0
yCoordw = 0.0
demandSum = 0
for ds in self.currentSources.get(s).destinations:
demandSum += ds.demand
xCoordw += float(ds.demand) * ds.x
yCoordw += float(ds.demand) * ds.y
xCoord += ds.x
yCoord += ds.y
if (xCoord != 0.0 and sizeForS != 0):
xCoordw = (xCoordw / demandSum)
yCoordw = (yCoordw / demandSum)
xCoord = xCoord / sizeForS
yCoord = yCoord / sizeForS
# print("weighted =>" + str(xCoordw) + ", " + str(yCoordw))
# print("Non weighted =>" + str(xCoord) + ", " + str(yCoord))
self.newSources[s] = Source.Source(xCoordw, yCoordw, -1, s)
def __init__(self, parent = None,name = None,modal = 0,fl = 0):
"""Set up the dialogue. The source object to delete is stored
in the global listSelected.
"""
Source.__init__(self,parent,name,modal,fl)
self.setCaption(self.tr("Delete Source"))
self._viewList= Control.Global.getListSelected()
so= self._viewList[0]
self.fromSourceToWidget(so)
# set up buttons
self.wSave.setText(self.tr('Delete'))
self.wClear.setEnabled(0)
self.wSave.setEnabled(1)
self.wRound.setEnabled(0)
self.wResc.setEnabled(0)
self.wPrev.setEnabled(0)
self.wNext.setEnabled(0)
self.wPrev.hide()
self.wNext.hide()
def __init__(self, parent = None,name = None,modal = 0,fl = 0):
"""Set up the dialogue. A list of selected source objects
are stored in the global listSelected tuple.
"""
Source.__init__(self,parent,name,modal,fl)
self.setCaption(self.tr("View Source"))
self._viewList= Control.Global.getListSelected()
self._pos= len(self._viewList) -1 # Show the last source
self.fromSourceToWidget(self._viewList[self._pos])
#Disable several buttons
self.wClear.setEnabled(0)
self.wSave.setEnabled(0)
self.wRound.setEnabled(0)
self.wNext.setEnabled(0)
self.wResc.setEnabled(0)
self.numGrep= None#Model.Grep.Grep()
if len(self._viewList)==1:
self.wPrev.setEnabled(0)
self.wNext.setEnabled(0)
def calculateAverageDistanceCost(self):
currentKeys = [
6, 16, 17, 21, 27, 28, 35, 46, 57, 60, 64, 67, 75, 80, 86, 93, 98,
109, 112, 119, 120, 124, 141, 146, 147, 152, 154, 156, 159, 162,
165, 172
]
#currentKeys = [6,16,17,21, 33,36,39,46,51, 60, 67,78,85,93,100,103,108,111,118,122,123,125,137,142,145,150,152,155,159,162,169,171]
i = 1
self.currentSources = {}
for k in currentKeys:
dest = self.destinations.get(k)
#self.tabooList[k] = dest
source = Source.Source(dest.x, dest.y, dest.id, i)
self.currentSources[i] = source
i += 1
self.doCurrentAssignment()
self.calculateCurrentCost()
sourceAverage = 0
demandSum = 0
disAverage = 0
weightedAverage = 0
for d in self.destinations:
dest = self.destinations.get(d)
demandSum += dest.demand
for d in self.destinations:
dest = self.destinations.get(d)
dest.weight = float(dest.demand) / demandSum
ncheck = 0
#print(len(self.currentSources))
for s in self.currentSources:
source = self.currentSources.get(s)
#print("Number of destinations for this source")
#print(len(source.destinations))
ncheck += len(source.destinations)
for dest in source.destinations:
source.totalDemand += dest.demand
weightedAverage += (self.distanceMap[source.destinationId][
dest.id]) * dest.weight
disAverage += self.distanceMap[source.destinationId][dest.id]
for dest in source.destinations:
sourceAverage += (
self.distanceMap[source.destinationId][dest.id] *
dest.demand) / source.totalDemand
n = len(self.destinations)
print("n = " + str(n))
print("ncheck = " + str(ncheck))
print("Distance total " + str(disAverage))
print("Weighted Distance total " + str(weightedAverage))
print("Source Weighted Distance total " + str(sourceAverage))
print("*******\n\n")
print("Distance total " + str(disAverage / n))
print("Weighted Distance total " + str(weightedAverage / n))
print("Source Weighted Distance total " + str(sourceAverage / self.k))
def createParts(self, merge=True):
#print("creating parts " + str(merge))
self.parts = {}
for d in self.destinations:
dest = self.destinations.get(d)
key = int(dest.label)
if self.parts.get(key) is None:
destArray = []
destArray.append(dest)
self.parts[key] = Part(destArray, key)
else:
self.parts.get(key).destinations.append(dest)
print("Total Parts " + str(len(self.parts)))
assignedFacilities = 0
maxDemandPointPart = None
maxdemandPoints = -1
for k in self.parts:
part = self.parts.get(k)
part.k = self.getNumberOfFacilitiesForThisPart(part.destinations)
assignedFacilities += part.k
if part.totalDemandPoints > maxdemandPoints:
maxDemandPointPart = part
maxdemandPoints = part.totalDemandPoints
if assignedFacilities < params['k']:
remaining = params['k'] - assignedFacilities
maxDemandPointPart.k += remaining
maxDemandPointPart.valid = False
self.parts[maxDemandPointPart.id] = maxDemandPointPart
for p in self.parts:
#print(part.k)
part = self.parts.get(p)
part.populateCentroid()
self.currentSources[p] = Source.Source(part.x, part.y, -1, p)
## bring me back
if merge:
part.homogeneous = self.areDemandPointsRandom(
part.destinations)
if merge:
self.mergePartsByDeletingSources()
self.k = len(self.parts)
self.sourceDao.k = self.k
self.partToDestinationMap = {}
for p in self.parts:
for d in self.parts.get(p).destinations:
self.partToDestinationMap[d.id] = p
def pmedian(self):
model = Model()
print("Problem scale " + str(self.n) + ", " + str(self.p))
alpha = [[model.add_var(name='alpha', var_type=BINARY) for j in range(self.n)] for i in range(self.n)]
x = [model.add_var(name='isFacility', var_type=BINARY) for i in range(self.n)]
# constraint 1
for i in range(self.n):
model += xsum(alpha[i][j] for j in range(self.n) ) == 1
# constraint 2
for i in range(self.n):
for j in range(self.n):
model += alpha[i][j] - x[j] <= 0
#constraint 3
model += xsum(x[j] for j in range(self.n)) == self.p
# objective function
model.objective = minimize(xsum(self.demandWeights[i]*self.dij[i][j]*alpha[i][j] for j in range(self.n) for i in range(self.n)))
status = model.optimize(max_seconds=360000)
print("solution status " + str(status) )
xj = []
alphaij = []
for v in model.vars:
if v.name == "isFacility":
xj.append(v.x)
iIndex = 0
sId = 1
for k in xj:
if int(k)==1:
destId = self.indexToId[iIndex]
dest = self.destinations.get(destId)
newsource = Source.Source(dest.x, dest.y, dest.id,sId )
self.currentSources[sId] = newsource
sId+=1
#print("iIndex "+ str(iIndex) + ", corresponding destination " + str(self.indexToId[iIndex]))
iIndex +=1
self.doCurrentAssignment()
cost = self.calculateCurrentCost()
print("cost (obj value) " + str(cost))
return cost
def kMeansStuff(self):
# Do kmeans stuff
self.iterations = 0
printFlag = False
if len(self.currentSources) == 0:
#print("Random selection")
currentKeys = random.sample(list(self.destinations), self.k)
i = 1
self.currentSources = {}
for k in currentKeys:
dest = self.destinations.get(k)
source = Source.Source(dest.x, dest.y, dest.id, i)
#source.id = i
i += 1
self.currentSources[source.id] = source
else:
printFlag = True
print("I already have sources")
#print(" Initial Sources ")
#self.printCurrentSources()
terminationCondition = False
itc = 0
self.doCurrentAssignment()
while not terminationCondition:
if printFlag:
print("itr " + str(itc) + "cost " +
str(self.calculateCurrentCost()))
#print("Iteration # " + str(itc))
self.calculateNewSources()
terminationCondition = self.checkForTermination()
#print(" termination Condition " + str(terminationCondition))
if not terminationCondition:
#print(" Not terminating ")
self.currentSources = dict(self.newSources)
self.newSources.clear()
self.doCurrentAssignment()
itc += 1
def myopicAlgo(self):
currentNumberOfSources = 1
while currentNumberOfSources <= self.k:
print("Adding source #"+ str(currentNumberOfSources))
addingCost = sys.maxsize
for did in self.destinations:
dest = self.destinations.get(did)
if self.checkCurrentSources(did): # destination not already a source
newSource = Source.Source(dest.x, dest.y, dest.id,currentNumberOfSources)
self.tempSources[currentNumberOfSources] = newSource
self.doCurrentAssignment(self.tempSources)
thisSourceCost = self.calculateCurrentCost()
if(addingCost > thisSourceCost):
#print("Candidate "+ str(did) + " cost = " + str(thisSourceCost))
addingCost = thisSourceCost
self.currentSources = dict(self.tempSources)
currentNumberOfSources +=1
self.tempSources = dict(self.currentSources)
self.doCurrentAssignment(self.currentSources)
print('Done with this shittt')
return self.calculateCurrentCost()
def getNewSource(self, source):
xCoord = 0.0
yCoord = 0.0
size = len(source.destinations)
xCoordw = 0.0
yCoordw = 0.0
demandSum = 0
sizeForS = len(source.destinations)
for ds in source.destinations:
demandSum += ds.demand
for dest in source.destinations:
xCoord += dest.x
yCoord += dest.y
xCoordw += float(dest.demand) * dest.x
yCoordw += float(dest.demand) * dest.y
if (xCoord != 0.0 and sizeForS != 0):
xCoordw = (xCoordw / demandSum)
yCoordw = (yCoordw / demandSum)
newMedian = self.findClosestDestination(xCoordw, yCoordw)
if newMedian.id != source.destinationId:
source = Source.Source(newMedian.x, newMedian.y, newMedian.id, source.id)
return source
def getPartsBasedOnValidity(self):
p = 2
pm = None
validityCondition = False
start = timeit.default_timer()
while not validityCondition:
print(
"****************************************************************************************************\n\n"
)
print("p = " + str(p))
pm = PartitioningManager.PartitioningManager(p, self.distType)
pm.k = p
pm.runMaranzanaForDecomposition()
pm.maranzana.createPartitionsFromSources()
validityCondition = not pm.maranzana.checkValidityOfParts()
p = p + 1
p = p - 1
print("p selected for decomposition " + str(p))
maranzanaRuns = runs
mrz = 0
minCost = sys.maxsize
minSourceIds = []
while mrz < maranzanaRuns:
print("########################## mrz Run " + str(mrz) +
"##################################################")
pm = PartitioningManager.PartitioningManager(p, self.distType)
pm.k = p
validityCondition = False
validCounter = 0
while not validityCondition:
print("p = " + str(p) + "validity counter " +
str(validCounter))
pm = PartitioningManager.PartitioningManager(p, self.distType)
pm.k = p
pm.runMaranzanaAfterPIsKnown()
pm.maranzana.createPartitionsFromSources()
validityCondition = not pm.maranzana.checkValidityOfParts()
validCounter += 1
#pm.maranzana.mergeParts()
pm.maranzana.mergePartsByDeletingSources()
#pm.maranzana.sourceDao.saveParts(pm.maranzana.currentDestinationToSourceMapping,pm.maranzana.destinations, len(pm.maranzana.parts))
#exit(0)
sourceIds = []
for par in pm.maranzana.parts:
print("part " + str(par))
part = pm.maranzana.parts.get(par)
pm.gria = GRIA.GRIA(part.k, self.distType)
print(" My k " + str(part.k))
print(" My destinations " + str(len(part.destinations)))
for d in part.destinations:
pm.gria.destinations[d.id] = d
pm.runGria()
for s in pm.gria.minSources:
src = pm.gria.minSources.get(s)
sourceIds.append(src.destinationId)
pm.maranzana = Maranzana.Maranzana(params['k'], self.distType)
pm.maranzana.getAllDestinations()
pm.maranzana.buildDistanceMap()
i = 1
print("Length of sourceids " + str(len(sourceIds)))
for s in sourceIds:
dest = pm.maranzana.destinations.get(s)
source = Source.Source(dest.x, dest.y, dest.id, i)
pm.maranzana.currentSources[i] = source
i += 1
pm.maranzana.doCurrentAssignment()
prevCost = pm.maranzana.calculateCurrentCost()
print("final cost before global maranzana " + str(prevCost))
pm.runMaranzana()
pm.maranzana.doCurrentAssignment()
afterCost = pm.maranzana.calculateCurrentCost()
print("cost after global maranzana " + str(afterCost))
if afterCost < prevCost:
print("Global Maranzana helped")
sourceIds = []
for s in pm.maranzana.currentSources:
src = pm.maranzana.minSources.get(s)
sourceIds.append(src.destinationId)
cost = prevCost
if afterCost < prevCost:
cost = afterCost
if minCost > cost:
minCost = cost
minSourceIds = {}
minSourceIds = sourceIds.copy()
mrz += 1
print("Final solution cost " + str(minCost) + " Number of sources " +
str(len(minSourceIds)))
i = 1
sourceIds = []
for s in minSourceIds:
#dest = pm.maranzana.destinations.get(s)
#source = Source.Source(dest.x, dest.y, dest.id, i)
sourceIds.append(s)
## post gria
pm.gria = GRIA.GRIA(params['k'], self.distType)
pm.gria.getAllDestinations()
pm.gria.buildDistanceMap()
i = 1
for s in sourceIds:
dest = pm.gria.destinations.get(s)
source = Source.Source(dest.x, dest.y, dest.id, i)
pm.gria.currentSources[i] = source
i += 1
pm.gria.doCurrentAssignment(pm.gria.currentSources)
cost = pm.gria.calculateCurrentCost()
print("final cost before gria " + str(cost))
stop = timeit.default_timer()
diff = stop - start
print("time before gria local = " + str(diff / 3600) + " hours")
pm.runGriaGlobalOnly()
pm.gria.doCurrentAssignment(pm.gria.currentSources)
stop = timeit.default_timer()
diff = stop - start
print("time = " + str(diff / 3600) + " hours")
cost = pm.gria.calculateCurrentCost()
print("final cost after local gria " + str(cost))
pm.gria.saveDestToSourceMapping()
stop = timeit.default_timer()
diff = stop - start
print("time = " + str(diff / 3600) + " hours")
def calculateAverageDistanceCost(self):
#currentKeys = [6,13,16,21,28, 32 ,36, 41, 51,63, 68 ,75,78,80,86,87, 93, 103,107,111, 117,122, 123,135,137,144,145,150,152,154,165,168 ] ## from sitation exchange
currentKeys = [6,16,17,21, 33,36,39,46,51, 60, 67,78,85,93,100,103,108,111,118,122,123,125,137,142,145,150,152,155,159,162,169,171]
currentKeys = [6, 16, 17, 21, 27, 28, 34, 46, 51, 57, 60,64, 67, 75, 80, 86, 93, 98, 108, 111, 118, 123, 140, 144, 145, 150, 152, 154, 159, 162, 165, 172] ## Lagrangioan
#currentKeys = []
i = 1
self.currentSources = {}
for k in currentKeys:
dest = self.destinations.get(k)
# self.tabooList[k] = dest
source = Source.Source(dest.x, dest.y, dest.id, i)
self.currentSources[i] = source
i += 1
self.doCurrentAssignment(self.currentSources)
self.calculateCurrentCost()
sourceAverage = 0
demandSum = 0
disAverage = 0
weightedAverage = 0
for d in self.destinations:
dest = self.destinations.get(d)
demandSum += dest.demand
for d in self.destinations:
dest = self.destinations.get(d)
dest.weight = float(dest.demand) / demandSum
ncheck = 0
# print(len(self.currentSources))
tdp = 0
for s in self.currentSources:
thisSource =0
source = self.currentSources.get(s)
print('\n*************\n source ' + str(source.destinationId))
# print("Number of destinations for this source")
# print(len(source.destinations))
ncheck += len(source.destinations)
for dest in source.destinations:
source.totalDemand += dest.demand
weightedAverage += (self.distanceMap[source.destinationId][dest.id]) * dest.weight
disAverage += self.distanceMap[source.destinationId][dest.id]
for dest in source.destinations:
dp = round(self.distanceMap[source.destinationId][dest.id]) * dest.demand
#print(str(dest.id) + ": dist --> " + str(round(self.distanceMap[source.destinationId][dest.id])) + "demamd " + str(dest.demand) + " prod " + str(dp))
tdp += dp
thisSource += (round(self.distanceMap[source.destinationId][dest.id]) * dest.demand)
print("Total demand for this source " + str(source.totalDemand))
print("Cost for this source " + str(thisSource))
thisSource = round(thisSource/source.totalDemand,2)
print("Average for this source " + str(thisSource))
sourceAverage += (thisSource*(source.totalDemand/demandSum))
n = len(self.destinations)
print("int cost " + str(tdp))
#print("n = " + str(n))
#print("ncheck = " + str(ncheck))
print("*******\n\n")
#print("Distance total " + str(disAverage / n))
#print("Weighted Distance total " + str(weightedAverage / n))
print("Source Weighted Distance total " + str(sourceAverage ))
print("26 to 32 theirrs " + str(round(self.distanceMap[26][32])))
print(str(self.destinations.get(26).x) + ", "+ str(self.destinations.get(26).y))
print(str(self.destinations.get(42).x) + ", " + str(self.destinations.get(42).y))
print("26 to 42 mine " + str(round(self.distanceMap[26][42])))
print("26 to 43 mine " + str(round(self.distanceMap[26][43])))
print("Gorttta make sense")
for d in self.currentDestinationToSourceMapping:
print("Node " + str(self.destinations.get(d).id) + "(" + str(self.destinations.get(d).x) + ","+ str(self.destinations.get(d).y) + ")" + "--> "+ str(self.currentDestinationToSourceMapping[d]) + ", cost =" + str(round(self.distanceMap[d][self.currentDestinationToSourceMapping[d]]) *self.destinations.get(d).demand))
def GetCallups( fname, soundalike=True, useUciId=True, useLicense=True, callbackfunc=None, callbackupdate=None ):
if callbackupdate: callbackupdate( _('Reading spreadsheet...') )
reader = GetExcelReader( fname )
sheet_names = [name for name in reader.sheet_names()]
registration_sheet_count = sum( 1 for sheet in sheet_names if sheet == RegistrationSheet )
if registration_sheet_count == 0:
raise ValueError( u'{}: "{}"'.format('Spreadsheet is missing sheet', RegistrationSheet ) )
if registration_sheet_count > 1:
raise ValueError( u'{}: "{}"'.format('Spreadsheet must have exactly one sheet named', RegistrationSheet ) )
if callbackupdate: callbackupdate( u'{}: {}'.format(_('Reading'), RegistrationSheet) )
reader = GetExcelReader( fname )
registration = Source( fname, RegistrationSheet, False )
registrationErrors = registration.read( reader )
if callbackfunc:callbackfunc( [registration], [registrationErrors] )
sources = []
errors = []
for sheet in sheet_names:
if sheet == RegistrationSheet:
continue
if callbackfunc: callbackfunc( sources + [registration], errors + [registrationErrors] )
if callbackupdate: callbackupdate( u'{}: {}'.format(_('Reading'), sheet) )
source = Source( fname, sheet, soundalike=soundalike, useUciId=useUciId, useLicense=useLicense )
errs = source.read( reader )
sources.append( source )
errors.append( errs )
# Add a random sequence as a final sort order.
registration.randomize_positions()
sources.append( registration )
errors.append( registrationErrors )
if callbackfunc: callbackfunc( sources, errors )
for reg in registration.results:
reg.result_vector = [source.find(reg) for source in sources]
callup_order = sorted(
registration.results,
key = lambda reg: tuple(r.get_sort_key() for r in reg.result_vector)
)
# Randomize riders with no criteria.
for i_random, reg in enumerate(callup_order):
if all( r.get_status() == r.NoMatch for r in reg.result_vector[:-1] ):
cu2 = callup_order[i_random:]
random.seed()
random.shuffle(cu2)
callup_order = callup_order[:i_random] + cu2
break
callup_results = []
registration_headers = registration.get_ordered_fields()
# Also add the team code if there is one.
if not 'team_code' in registration_headers:
for iSource, source in enumerate(sources):
if 'team_code' in source.get_ordered_fields():
try:
i_team = registration_headers.index('team')
registration_headers = tuple(
list(registration_headers[:i_team+1]) +
['team_code'] +
list(registration_headers[i_team+1:])
)
except ValueError:
registration_headers = tuple( list(registration_headers) + ['team_code'] )
for reg in callup_order:
try:
reg.team_code = reg.result_vector[iSource].matches[0].team_code
except:
pass
break
callup_headers = list(registration_headers) + [source.sheet_name for source in sources[:-1]]
for reg in callup_order:
row = [getattr(reg, f, u'') for f in registration_headers]
row.extend( reg.result_vector[:-1] )
callup_results.append( row )
return registration_headers, callup_headers, callup_results, sources, errors
def GetCallups(fname,
soundalike=True,
useUciId=True,
useLicense=True,
callbackfunc=None,
callbackupdate=None):
if callbackupdate: callbackupdate(_('Reading spreadsheet...'))
reader = GetExcelReader(fname)
sheet_names = [name for name in reader.sheet_names()]
registration_sheet_count = sum(1 for sheet in sheet_names
if sheet == RegistrationSheet)
if registration_sheet_count == 0:
raise ValueError(u'{}: "{}"'.format('Spreadsheet is missing sheet',
RegistrationSheet))
if registration_sheet_count > 1:
raise ValueError(u'{}: "{}"'.format(
'Spreadsheet must have exactly one sheet named',
RegistrationSheet))
if callbackupdate:
callbackupdate(u'{}: {}'.format(_('Reading'), RegistrationSheet))
reader = GetExcelReader(fname)
registration = Source(fname, RegistrationSheet, False)
registrationErrors = registration.read(reader)
if callbackfunc: callbackfunc([registration], [registrationErrors])
sources = []
errors = []
for sheet in sheet_names:
if sheet == RegistrationSheet:
continue
if callbackfunc:
callbackfunc(sources + [registration],
errors + [registrationErrors])
if callbackupdate:
callbackupdate(u'{}: {}'.format(_('Reading'), sheet))
source = Source(fname,
sheet,
soundalike=soundalike,
useUciId=useUciId,
useLicense=useLicense)
errs = source.read(reader)
sources.append(source)
errors.append(errs)
# Add a random sequence as a final sort order.
registration.randomize_positions()
sources.append(registration)
errors.append(registrationErrors)
if callbackfunc: callbackfunc(sources, errors)
for reg in registration.results:
reg.result_vector = [source.find(reg) for source in sources]
callup_order = sorted(registration.results,
key=lambda reg: tuple(r.get_sort_key()
for r in reg.result_vector))
# Randomize riders with no criteria.
for i_random, reg in enumerate(callup_order):
if all(r.get_status() == r.NoMatch for r in reg.result_vector[:-1]):
cu2 = callup_order[i_random:]
random.seed()
random.shuffle(cu2)
callup_order = callup_order[:i_random] + cu2
break
callup_results = []
registration_headers = registration.get_ordered_fields()
# Also add the team code if there is one.
if not 'team_code' in registration_headers:
for iSource, source in enumerate(sources):
if 'team_code' in source.get_ordered_fields():
try:
i_team = registration_headers.index('team')
registration_headers = tuple(
list(registration_headers[:i_team + 1]) +
['team_code'] +
list(registration_headers[i_team + 1:]))
except ValueError:
registration_headers = tuple(
list(registration_headers) + ['team_code'])
for reg in callup_order:
try:
reg.team_code = reg.result_vector[iSource].matches[
0].team_code
except:
pass
break
callup_headers = list(registration_headers) + [
source.sheet_name for source in sources[:-1]
]
for reg in callup_order:
row = [getattr(reg, f, u'') for f in registration_headers]
row.extend(reg.result_vector[:-1])
callup_results.append(row)
return registration_headers, callup_headers, callup_results, sources, errors
def useEMClusters(self):
finalData = []
#fileName = "n_" + str(params['blockgroups']) + "_" + self.distType+"_k_"+str(params['k'])+"_merge_mip_fi"+".csv"
fileName = self.distType + "_k_" + str(
params['k']) + "_merge_mip_fi_nonparallel" + ".csv"
clusterFileName = "n_" + str(
params['blockgroups']) + "_" + self.distType + "_k_" + str(
params['k']) + "_indexes.csv"
p = 2
scores = {}
ps = []
selectedPs = []
selectedDistancePs = []
ins = 1
pm = None
cnt = 5
clusterIndexes = []
while cnt <= 5:
start = timeit.default_timer()
print(" trial " + str(cnt))
logging.info(" trial " + str(cnt))
while ins <= params['selectionRuns']:
row = [ins]
print(ins)
#clusterIndexes.append(i)
pm = PartitioningManager.PartitioningManager(p, self.distType)
pm.em.getAllDestinations()
while p <= params['maxP']:
pm.em.k = p
scores[p] = {}
ps.append(p)
sc = pm.em.createGmmClusters()
scores[p] = sc
p += 1
results = pm.evaluateP(scores)
selectedPs.append(results['densityEstimates'])
selectedDistancePs.append(results['clustering'])
p = 2
ins += 1
ins = 1
print(selectedPs)
medianP = int(statistics.median(selectedPs))
medianPCluster = int(statistics.median(selectedDistancePs))
medianPCluster = min(medianPCluster, medianP, 10)
medianPCluster_org = medianPCluster
print("winner for distance " + str(medianP))
print("winner " + str(medianP))
#exit(0)
j = 0
minCost = sys.maxsize
minSourceIds = []
minCost = sys.maxsize
minSources = []
dests = {}
pm = PartitioningManager.PartitioningManager(
medianP, self.distType)
pm.em.getAllDestinations()
dests = copy.deepcopy(pm.em.destinations)
ltime = 0
gtime = 0
gcost = 0
gtime = 0
brakFlag = False
while (j < params['instances']):
print(
"******************** whole instance " + str(j) +
" *********************************************************\n\n"
)
logging.info(
"******************** whole instance " + str(j) +
" *********************************************************\n\n"
)
pm = PartitioningManager.PartitioningManager(
medianP, self.distType)
pm.em.getAllDestinations()
#dests = copy.deepcopy(pm.em.destinations)
pm.em.createGmmClusters(True)
pm.em.createParts(True) # Make it true for distance analysis
medianP = len(pm.em.parts)
print("Number of em current sources == " +
str(len(pm.em.currentSources)))
if len(pm.em.currentSources) == 0:
exit(0)
pm.k = medianPCluster
pm.em.k = medianPCluster
pm.em.createGmmClusters(True) ## plot flag is true
pm.em.createParts(False) # Dont merge
#pm.em.sourceDao.savePartBoundaries(pm.em.partToDestinationMap, "nmp_silhouette")
j += 1
sourceIds = pm.assignPartsToProcessesForMIP(pm.em.parts)
pm = PartitioningManager.PartitioningManager(
params['k'], self.distType)
# try multiple instances with this p
pm.fi = FI_Parallel.FastInterchange(params['k'], self.distType)
i = 1
pm.fi.destinations = copy.deepcopy(dests)
if len(sourceIds) != params['k']:
logging.info("Not sure ehst hsppened selected only" +
str(len(sourceIds)) + " sources")
brakFlag = True
continue
for s in sourceIds:
dest = pm.fi.destinations.get(s)
source = Source.Source(dest.x, dest.y, dest.id, i)
pm.fi.currentSources[i] = source
i += 1
#pm.fi
#pm.fi.doCurrentAssignment(pm.gria.currentSources)
pm.fi.doInitialAssignment()
cost = pm.fi.calculateCurrentCost()
if cost < minCost:
minCost = cost
minSources = sourceIds
stop = timeit.default_timer()
gtime = (stop - start) / 3600
gcost = minCost
print("time and cost after gmm " + str(gcost) + ", " +
str(gtime))
logging.info("time and cost after gmm " + str(gcost) + ", " +
str(gtime))
print("Cost for this instance " + str(cost))
logging.info("Cost for this instance " + str(cost))
print("Divinding by distance into " + str(medianPCluster))
logging.info("Divinding by distance into " +
str(medianPCluster))
disData = []
medianPCluster_org = medianPCluster
prevCost = gcost
disCost = 0
while medianPCluster >= 1 and disCost != prevCost:
print("*********medianPCluster = " + str(medianPCluster) +
"\n")
logging.info("*********medianPCluster = " +
str(medianPCluster) + "\n")
prevCost = disCost
pm.em.k = medianPCluster
pm.em.destinations = {}
for s, src in pm.fi.currentSources.items():
pm.em.currentSources[s] = pm.fi.currentSources[s]
pm.em.clusterSources()
sourceIds = pm.createAndRunFIInstances()
pm = PartitioningManager.PartitioningManager(
params['k'], self.distType)
# try multiple instances with this p
pm.fi = FI_Parallel.FastInterchange(
params['k'], self.distType)
pm.fi.destinations = copy.deepcopy(dests)
i = 1
for s in sourceIds:
dest = pm.fi.destinations.get(s)
source = Source.Source(dest.x, dest.y, dest.id, i)
pm.fi.currentSources[i] = source
i += 1
pm.fi.doInitialAssignment()
disCost = pm.fi.calculateCurrentCost()
lcost = disCost
stop = timeit.default_timer()
ltime = (stop - start) / 3600
disData.append([medianPCluster, disCost, ltime])
print("Cost for this instance fpr" + str(medianPCluster) +
"after distance division = " + str(disCost) +
"time = " + str(ltime))
logging.info("Cost for this instance fpr" +
str(medianPCluster) +
"after distance division = " + str(disCost) +
"time = " + str(ltime))
medianPCluster -= 1
logging.info("I got here")
self.saveCostData(
self.distType + str(params['k']) + "_trial" + str(cnt) +
"_distanceClusters.csv", disData)
if brakFlag:
continue
if medianPCluster > 1:
print("time and cost after distance " + str(disCost) + ", " +
str(ltime))
print("Num of srcs " + str(len(minSources)))
pm.fi.fastInterchange(parallelize=False)
stop = timeit.default_timer()
diff = stop - start
print("final time = " + str(diff / 3600) + " hours")
tame = diff / 3600
cost = pm.fi.calculateCurrentCost()
print("final cost after FI " + str(cost))
name = "merge_fi_mip" # or
else:
tame = ltime
cost = disCost
logging.info("appending data")
#name = "noMerge_Distance"
#pm.fi.saveDestToSourceMapping(cnt, name)
#3 "Number of parts, final cost, final time, time after gmm, cost after gmm"
finalData.append([
medianP, medianPCluster_org, cost, tame, ltime, lcost, gtime,
gcost
])
#finalData.append([medianP, cost, tame,ltame,lcost, pm.gria.localSwapCount, pm.gria.globalSwapCount, pm.gria.iterations ])
cnt += 1
#path = "sc2020/"
self.saveCostData(fileName, finalData)
def completeTB(self):
currentKeys = random.sample(list(self.destinations), self.k)
i = 1
self.currentSources = {}
for k in currentKeys:
dest = self.destinations.get(k)
self.tabooList[k] = dest
source = Source.Source(dest.x, dest.y, dest.id, i)
i += 1
self.currentSources[source.id] = source
#print(" Initial Sources ")
#self.printCurrentSources()
complementKeys = []
for d in self.destinations:
if d not in currentKeys:
complementKeys.append(d)
self.doCurrentAssignment()
self.current_r_value = self.calculateCurrentCost()
terminationCondition = False
iterations = 0
while terminationCondition == False and iterations < self.maxIterations:
print("iteration " + str(iterations))
currentSourcesBackUp = dict(self.currentSources)
v1_sources = dict(self.currentSources)
r_min_b = self.current_r_value
bWinner = {}
for comp in complementKeys:
if self.tabooList.get(comp) is None:
dest_b = self.destinations.get(comp)
vertex_k = -1
delta_min = sys.maxsize
for vertex in v1_sources:
vj = v1_sources.get(vertex)
self.currentSources.update({
vertex:
Source.Source(dest_b.x, dest_b.y, dest_b.id,
vertex)
})
self.doCurrentAssignment()
delta_bj = self.calculateCurrentCost(
) - self.current_r_value
self.currentSources = dict(currentSourcesBackUp)
if delta_bj < 0 and delta_bj < delta_min:
delta_min = delta_bj
vertex_k = vertex
if vertex_k != -1:
toReplace = self.currentSources[vertex_k]
self.currentSources.update({
vertex_k:
Source.Source(dest_b.x, dest_b.y, dest_b.id,
vertex)
})
self.doCurrentAssignment()
if self.calculateCurrentCost() < r_min_b:
r_min_b = self.calculateCurrentCost()
bWinner = dict(self.currentSources)
self.currentSources.update({vertex_k: toReplace})
self.doCurrentAssignment()
if r_min_b < self.current_r_value:
print("r current defeated !!!")
print("r_current " + str(self.current_r_value) + " r_min_b " +
str(r_min_b))
self.currentSources = dict(bWinner)
self.current_r_value = r_min_b
currentKeys = []
for s in self.currentSources:
currentKeys.append(
self.currentSources.get(s).destinationId)
self.tabooList.update({
self.currentSources.get(s).destinationId:
self.destinations.get(
self.currentSources.get(s).destinationId)
})
complementKeys = []
for d in self.destinations:
if d not in currentKeys:
complementKeys.append(d)
iterations += 1
else:
self.doCurrentAssignment()
terminationCondition = True
print(" Ending Program==> No improvement possible ")
def fastInterchange(self, parallelize=False):
self.iterations = 0
if len(self.currentSources) == 0:
print("random sources in fi")
currentKeys = random.sample(list(self.destinations), self.k)
i = 1
self.currentSources = {}
for k in currentKeys:
dest = self.destinations.get(k)
self.tabooList[k] = dest
source = Source.Source(dest.x, dest.y, dest.id, i)
i += 1
self.currentSources[source.id] = source
currentKeys = []
for s in self.currentSources:
currentKeys.append(self.currentSources[s].destinationId)
self.doInitialAssignment() ## optimized step in the
self.initializeSecondClosestSource()
fopt = self.calculateCostUpdated()
print("Initial cost " + str(fopt))
terminationCondition = False
complementKeys = []
for d in self.destinations:
if d not in currentKeys:
complementKeys.append(d)
while not terminationCondition:
fopt = self.calculateCostUpdated()
#print("\n~~~~~~~~~~~~~~Iteration: " + str(self.iterations))
wmin = sys.maxsize
goin_min = -1
gout_min = -1
for goin in complementKeys:
if parallelize:
res = self.myMoveEvalParallel(goin)
else:
res = self.myMoveEval(goin)
if wmin > res['w']:
wmin = res['w']
#if parallelize:
#print("I found a reduced wmin" + str(wmin))
goin_min = goin
gout_min = res['goOut']
#print("################################\n wmin = " + str(wmin))
self.iterations += 1
if wmin >= 0:
print("Extermination!!")
#self.doCurrentAssignment()
fCost = self.calculateCostUpdated()
print("final cost = " + str(fCost))
#exit(0)
terminationCondition = True
else:
fopt = fopt + wmin
#print("\n swap change " + str(gout_min) + " to " + str(goin_min) + "\n")
sourceIdToUpdate = self.destIdToSourceId[gout_min]
#print("Updating source id " + str(sourceIdToUpdate) + " to " + str(goin_min))
self.currentSources[sourceIdToUpdate] = Source.Source(
self.destinations.get(goin_min).x,
self.destinations.get(goin_min).y, goin_min,
sourceIdToUpdate)
if not parallelize:
self.updateClosestAndSecondClostest(goin_min, gout_min)
else:
#self.updateClosestAndSecondClostest(goin_min, gout_min)
self.updateClosestAndSecondClostestParallel(
goin_min, gout_min)
#self.doCurrentAssignment()
cost = self.calculateCostUpdated()
self.destIdToSourceId = {}
for s in self.currentSources:
source = self.currentSources.get(s)
self.destIdToSourceId[source.destinationId] = s
ind = 0
for key in complementKeys:
if key == goin_min:
complementKeys[ind] = gout_min
break
ind += 1
def MakeExampleExcel( include_uci_points=True, include_national_points=True, include_previous_result=True ):
Year = datetime.date.today().year
YearAdjust = Year - 2017
fname = os.path.join(Utils.getImageFolder(), 'IndividualRanking.xlsx')
reader = GetExcelReader( fname )
uci_points = Source( fname, 'Individual' )
uci_points.read( reader )
for r in uci_points.results:
r.age += YearAdjust
r.license = random_license()
uci_sample = random.sample( uci_points.results, 20 )
common_first_names = u'Léopold Grégoire Aurélien Rémi Léandre Thibault Kylian Nathan Lucas Enzo Léo Louis Hugo Gabriel Ethan Mathis Jules Raphaël Arthur Théo Noah Timeo Matheo Clément Maxime Yanis Maël'.split()
common_last_names = u'Tisserand Lavergne Guignard Parmentier Evrard Leclerc Martin Bernard Dubois Petit Durand Leroy Moreau Simon Laurent Lefevre Roux Fournier Dupont'.split()
other_sample = []
for i in range(20):
other_sample.append( Result(
first_name=common_first_names[i%len(common_first_names)],
last_name=common_last_names[i%len(common_last_names)],
uci_id=random_uci_id(),
license=random_license(),
)
)
registration = list(uci_sample) + other_sample
bibs = list(range(100,200))
random.shuffle( bibs )
for i, r in enumerate(registration):
r.bib = bibs[i]
fname_excel = os.path.join( Utils.getHomeDir(), 'CS_Test_Input.xlsx' )
wb = xlsxwriter.Workbook( fname_excel )
ws = wb.add_worksheet('Registration')
fit_sheet = FitSheetWrapper( ws )
fields = ['bib', 'first_name', 'last_name', 'uci_id', 'license']
for c, field in enumerate(fields):
fit_sheet.write( 0, c, make_title(field) )
for r, result in enumerate(registration):
for c, field in enumerate(fields):
fit_sheet.write( r+1, c, getattr(result, field) )
if include_uci_points:
ws = wb.add_worksheet('Individual')
fit_sheet = FitSheetWrapper( ws )
for c, header in enumerate(['Rank', 'UCI ID', 'Name', 'Team Code', 'Age', 'Points']):
fit_sheet.write( 0, c, header )
for r, result in enumerate(uci_points.results):
row = r + 1
fit_sheet.write( row, 0, u'{}'.format(row) )
fit_sheet.write( row, 1, result.uci_id if result.uci_id else u'' )
fit_sheet.write( row, 2, u'{} {}'.format(result.last_name.upper(), result.first_name) )
fit_sheet.write( row, 3, result.team_code if result.team_code else u'' )
fit_sheet.write( row, 4, result.age )
fit_sheet.write( row, 5, result.points )
eligible_for_points = other_sample + [rr for rr in uci_points.results if rr.nation_code == 'FRA']
if include_national_points:
ws = wb.add_worksheet('National Points')
fit_sheet = FitSheetWrapper( ws )
for c, header in enumerate(['First Name', 'Last Name', 'License', 'Points']):
fit_sheet.write( 0, c, header )
for r, result in enumerate(random.sample(eligible_for_points, min(len(eligible_for_points),35))):
row = r + 1
fit_sheet.write( row, 0, result.first_name )
fit_sheet.write( row, 1, result.last_name )
fit_sheet.write( row, 2, result.license )
fit_sheet.write( row, 3, random.randint(1, 200) )
if include_previous_result:
ws = wb.add_worksheet( '{} Result'.format(Year-1) )
fit_sheet = FitSheetWrapper( ws )
for c, header in enumerate(['Pos', 'First Name', 'Last Name', 'UCI ID', 'License']):
fit_sheet.write( 0, c, header )
for r, result in enumerate(random.sample(eligible_for_points, min(len(eligible_for_points),35))):
row = r + 1
fit_sheet.write( row, 0, row )
fit_sheet.write( row, 1, result.first_name )
fit_sheet.write( row, 2, result.last_name )
fit_sheet.write( row, 3, result.uci_id )
fit_sheet.write( row, 4, result.license )
wb.close()
return fname_excel
