def resync(self): # Quicker version than the general purpose implementation result = ParameterSource.resync(self) for psource in self._psourceList: result = combineSyncResult(result, psource.resync()) oldMaxParameters = self._maxParameters self._maxParameters = self.initMaxParameters() return (result[0], result[1], oldMaxParameters != self._maxParameters)
def resync(self):
(result_redo, result_disable, result_sizeChange) = ParameterSource.resync(self)
if self.resyncEnabled() and self._dataProvider:
# Get old and new dataset information
old = DataProvider.loadFromFile(self.getDataPath('cache.dat')).getBlocks()
self._dataProvider.clearCache()
new = self._dataProvider.getBlocks()
self._dataProvider.saveToFile(self.getDataPath('cache-new.dat'), new)
# Use old splitting information to synchronize with new dataset infos
jobChanges = self._dataSplitter.resyncMapping(self.getDataPath('map-new.tar'), old, new)
if jobChanges:
# Move current splitting to backup and use the new splitting from now on
def backupRename(old, cur, new):
if self._keepOld:
os.rename(self.getDataPath(cur), self.getDataPath(old))
os.rename(self.getDataPath(new), self.getDataPath(cur))
backupRename( 'map-old-%d.tar' % time.time(), 'map.tar', 'map-new.tar')
backupRename('cache-old-%d.dat' % time.time(), 'cache.dat', 'cache-new.dat')
old_maxN = self._dataSplitter.getMaxJobs()
self._dataSplitter.importPartitions(self.getDataPath('map.tar'))
self._maxN = self._dataSplitter.getMaxJobs()
result_redo.update(jobChanges[0])
result_disable.update(jobChanges[1])
result_sizeChange = result_sizeChange or (old_maxN != self._maxN)
self.resyncFinished()
return (result_redo, result_disable, result_sizeChange)
def resync(self): (result_redo, result_disable, result_sizeChange) = ParameterSource.resync(self) (psource_redo, psource_disable, _) = self._psource.resync() # size change is irrelevant if outside of range for pNum in psource_redo: if (pNum >= self._posStart) and (pNum <= self._posEnd): result_redo.add(pNum - self._posStart) for pNum in psource_disable: if (pNum >= self._posStart) and (pNum <= self._posEnd): result_disable.add(pNum - self._posStart) oldPosEnd = self._posEnd self._posEnd = utils.QM(self._posEndUser is None, self._psource.getMaxParameters() - 1, self._posEndUser) return (result_redo, result_disable, result_sizeChange or (oldPosEnd != self._posEnd))
def resync(self): (result_redo, result_disable, result_sizeChange) = ParameterSource.resync(self) if self.resyncEnabled(): (psource_redo, psource_disable, psource_sizeChange) = self._psource.resync() self._pSpace = self.initPSpace() for pNum, pInfo in enumerate(self._pSpace): subNum, _ = pInfo # ignore lookupIndex if subNum in psource_redo: result_redo.add(pNum) if subNum in psource_disable: result_disable.add(pNum) self.resyncFinished() return (result_redo, result_disable, result_sizeChange or psource_sizeChange)
def resync(self): oldMaxParameters = self._maxParameters # Perform resync of subsources psourceResyncList = lmap(lambda p: p.resync(), self._psourceList) # Update max for _translateNum self._psourceMaxList = lmap(lambda p: p.getMaxParameters(), self._psourceList) self._maxParameters = self.initMaxParameters() # translate affected pNums from subsources (result_redo, result_disable, dummy) = ParameterSource.resync(self) for (idx, psource_resync) in enumerate(psourceResyncList): (psource_redo, psource_disable, dummy) = psource_resync for pNum in psource_redo: result_redo.update(self._translateNum(idx, pNum)) for pNum in psource_disable: result_disable.update(self._translateNum(idx, pNum)) return (result_redo, result_disable, oldMaxParameters != self._maxParameters)
def resync(self):
(result_redo, result_disable,
result_sizeChange) = ParameterSource.resync(self)
if self.resyncEnabled():
(psource_redo, psource_disable,
psource_sizeChange) = self._psource.resync()
self._pSpace = self.initPSpace()
for pNum, pInfo in enumerate(self._pSpace):
subNum, _ = pInfo # ignore lookupIndex
if subNum in psource_redo:
result_redo.add(pNum)
if subNum in psource_disable:
result_disable.add(pNum)
self.resyncFinished()
return (result_redo, result_disable, result_sizeChange
or psource_sizeChange)
def resync(self):
(result_redo, result_disable,
result_sizeChange) = ParameterSource.resync(self)
if self.resyncEnabled() and self._dataProvider:
# Get old and new dataset information
old = DataProvider.loadFromFile(
self.getDataPath('cache.dat')).getBlocks()
self._dataProvider.clearCache()
new = self._dataProvider.getBlocks()
self._dataProvider.saveToFile(self.getDataPath('cache-new.dat'),
new)
# Use old splitting information to synchronize with new dataset infos
jobChanges = self._dataSplitter.resyncMapping(
self.getDataPath('map-new.tar'), old, new)
if jobChanges:
# Move current splitting to backup and use the new splitting from now on
def backupRename(old, cur, new):
if self._keepOld:
os.rename(self.getDataPath(cur), self.getDataPath(old))
os.rename(self.getDataPath(new), self.getDataPath(cur))
backupRename('map-old-%d.tar' % time.time(), 'map.tar',
'map-new.tar')
backupRename('cache-old-%d.dat' % time.time(), 'cache.dat',
'cache-new.dat')
old_maxN = self._dataSplitter.getMaxJobs()
self._dataSplitter.importPartitions(
self.getDataPath('map.tar'))
self._maxN = self._dataSplitter.getMaxJobs()
result_redo.update(jobChanges[0])
result_disable.update(jobChanges[1])
result_sizeChange = result_sizeChange or (old_maxN !=
self._maxN)
self.resyncFinished()
return (result_redo, result_disable, result_sizeChange)
