def enablePartition(self, enabled, clusterName, instanceName, resourceName, partitionNames):
"""
Returns void
Parameters:
enabled: booleanclusterName: StringinstanceName: StringresourceName: StringpartitionNames: List<String>
@Override
"""
# String
path = PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.PARTICIPANT.toString(), instanceName)
# ZkBaseDataAccessor<ZNRecord>
baseAccessor = ZkBaseDataAccessor<ZNRecord>(_zkClient)
if not baseAccessor.exists(path, 0):
raise HelixException("Cluster: " + clusterName + ", instance: " + instanceName + ", instance config does not exist")
# String
idealStatePath = PropertyPathConfig.getPath(PropertyType.IDEALSTATES, clusterName, resourceName)
# ZNRecord
idealStateRecord = None
try:
idealStateRecord = baseAccessor.get(idealStatePath, None, 0)
except ZkNoNodeException, e:
if idealStateRecord == None:
raise HelixException("Cluster: " + clusterName + ", resource: " + resourceName + ", ideal state does not exist")
# IdealState
idealState = IdealState(idealStateRecord)
for # String
partitionName = None
in partitionNames) if (idealState.getIdealStateMode() == IdealStateModeProperty.AUTO && idealState.getPreferenceList(partitionName) == None) or (idealState.getIdealStateMode() == IdealStateModeProperty.CUSTOMIZED && idealState.getInstanceStateMap(partitionName) == None):
logger.warn("Cluster: " + clusterName + ", resource: " + resourceName + ", partition: " + partitionName + ", partition does not exist in ideal state")
def createClient(self, zkServers):
"""
Returns void
Parameters:
zkServers: String
Java modifiers:
private
Throws:
Exception
"""
# String
propertyStorePath = PropertyPathConfig.getPath(
PropertyType.PROPERTYSTORE, self._clusterName)
# PathBasedZkSerializer
zkSerializer = ChainedPathZkSerializer.builder(
ZNRecordStreamingSerializer()).serialize(
propertyStorePath, ByteArraySerializer()).build()
# this does notwork, need ot use the path
# zkSerializer = ByteArraySerializer()
# zkSerializer = BasicZkSerializer(ByteArraySerializer())
self._zkClient = ZkClient(zkServers, self._sessionTimeout,
self.CONNECTIONTIMEOUT, zkSerializer)
# dzhang: not there in latest version
# self._accessor = ZKDataAccessor(self._clusterName, self._zkClient)
# ZkBaseDataAccessor<ZNRecord>
# baseDataAccessor = ZkBaseDataAccessor<ZNRecord>(self._zkClient)
baseDataAccessor = ZkBaseDataAccessor(self._zkClient)
if self._instanceType == InstanceType.PARTICIPANT:
# String
curStatePath = PropertyPathConfig.getPath(
PropertyType.CURRENTSTATES, self._clusterName,
self._instanceName)
# TODO: handle cache later
self._baseDataAccessor = baseDataAccessor
# self._baseDataAccessor = ZkCacheBaseDataAccessor<ZNRecord>(baseDataAccessor, curStatePath)
# self._baseDataAccessor = ZkCacheBaseDataAccessor<ZNRecord>(baseDataAccessor, Arrays.asList(curStatePath))
else:
if self._instanceType == InstanceType.CONTROLLER:
# String
extViewPath = PropertyPathConfig.getPath(
PropertyType.EXTERNALVIEW, self._clusterName)
# TODO: handle cache later
self._baseDataAccessor = baseDataAccessor
# self._baseDataAccessor = ZkCacheBaseDataAccessor<ZNRecord>(baseDataAccessor, extViewPath)
# self._baseDataAccessor = ZkCacheBaseDataAccessor<ZNRecord>(baseDataAccessor, Arrays.asList(extViewPath))
else:
self._baseDataAccessor = baseDataAccessor
self._helixAccessor = ZKHelixDataAccessor(self._clusterName,
self._instanceType,
self._baseDataAccessor)
self._configAccessor = ConfigAccessor(self._zkClient)
# int
retryCount = 0
self._zkClient.subscribeStateChanges(self._zkStateChangeListener)
# set the state?
self._zkStateChangeListener.handleStateChanged(KazooState.CONNECTED)
self._zkStateChangeListener.handleNewSession()
def createClient(self, zkServers):
"""
Returns void
Parameters:
zkServers: String
Java modifiers:
private
Throws:
Exception
"""
# String
propertyStorePath = PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, self._clusterName)
# PathBasedZkSerializer
zkSerializer = ChainedPathZkSerializer.builder(ZNRecordStreamingSerializer()).serialize(propertyStorePath, ByteArraySerializer()).build()
# this does notwork, need ot use the path
# zkSerializer = ByteArraySerializer()
# zkSerializer = BasicZkSerializer(ByteArraySerializer())
self._zkClient = ZkClient(zkServers, self._sessionTimeout, self.CONNECTIONTIMEOUT, zkSerializer)
# dzhang: not there in latest version
# self._accessor = ZKDataAccessor(self._clusterName, self._zkClient)
# ZkBaseDataAccessor<ZNRecord>
# baseDataAccessor = ZkBaseDataAccessor<ZNRecord>(self._zkClient)
baseDataAccessor = ZkBaseDataAccessor(self._zkClient)
if self._instanceType == InstanceType.PARTICIPANT:
# String
curStatePath = PropertyPathConfig.getPath(PropertyType.CURRENTSTATES, self._clusterName, self._instanceName)
# TODO: handle cache later
self._baseDataAccessor = baseDataAccessor
# self._baseDataAccessor = ZkCacheBaseDataAccessor<ZNRecord>(baseDataAccessor, curStatePath)
# self._baseDataAccessor = ZkCacheBaseDataAccessor<ZNRecord>(baseDataAccessor, Arrays.asList(curStatePath))
else:
if self._instanceType == InstanceType.CONTROLLER:
# String
extViewPath = PropertyPathConfig.getPath(PropertyType.EXTERNALVIEW, self._clusterName)
# TODO: handle cache later
self._baseDataAccessor = baseDataAccessor
# self._baseDataAccessor = ZkCacheBaseDataAccessor<ZNRecord>(baseDataAccessor, extViewPath)
# self._baseDataAccessor = ZkCacheBaseDataAccessor<ZNRecord>(baseDataAccessor, Arrays.asList(extViewPath))
else:
self._baseDataAccessor = baseDataAccessor
self._helixAccessor = ZKHelixDataAccessor(self._clusterName, self._instanceType, self._baseDataAccessor)
self._configAccessor = ConfigAccessor(self._zkClient)
# int
retryCount = 0
self._zkClient.subscribeStateChanges(self._zkStateChangeListener)
# set the state?
self._zkStateChangeListener.handleStateChanged(KazooState.CONNECTED)
self._zkStateChangeListener.handleNewSession()
def addLiveInstance(self):
"""
Returns void
Java modifiers:
private
"""
# LiveInstance
liveInstance = LiveInstance(self._instanceName)
liveInstance.setSessionId(self._sessionId)
liveInstance.setHelixVersion(self._version)
liveInstance.setLiveInstance("%s@%s" % (os.getpid(), socket.gethostname()))
# liveInstance.setLiveInstance(ManagementFactory.getRuntimeMXBean().getName())
self.logger.info("Add live instance: InstanceName: " + self._instanceName + " Session id:" + str(self._sessionId))
# Builder
keyBuilder = self._helixAccessor.keyBuilder()
if not self._helixAccessor.createProperty(keyBuilder.liveInstance(self._instanceName), liveInstance):
# String
errorMsg = "Fail to create live instance node after waiting, so quit. instance:" + self._instanceName
self.logger.warn(errorMsg)
# TODO: still raise
# raise HelixException(errorMsg)
# String
currentStatePathParent = PropertyPathConfig.getPath(PropertyType.CURRENTSTATES, self._clusterName, self._instanceName, self.getSessionId())
if not self._zkClient.exists(currentStatePathParent):
self._zkClient.createPersistent(currentStatePathParent)
self.logger.info("Creating current state path " + currentStatePathParent)
def updateProperty(self, type, value, keys):
"""
Returns boolean
Parameters:
type: PropertyTypevalue: ZNRecordkeys: String
@Override
"""
# String
path = PropertyPathConfig.getPath(type, _clusterName, keys)
if type.isUpdateOnlyOnExists():
ZKUtil.updateIfExists(_zkClient, path, value, type.isMergeOnUpdate())
else:
# String
parent = File(path).getParent()
if not _zkClient.exists(parent):
_zkClient.createPersistent(parent, True)
if not type.usePropertyTransferServer():
ZKUtil.createOrUpdate(_zkClient, path, value, type.isPersistent(), type.isMergeOnUpdate())
else:
ZKUtil.asyncCreateOrUpdate(_zkClient, path, value, type.isPersistent(), type.isMergeOnUpdate())
return True
def addLiveInstance(self):
"""
Returns void
Java modifiers:
private
"""
# LiveInstance
liveInstance = LiveInstance(self._instanceName)
liveInstance.setSessionId(self._sessionId)
liveInstance.setHelixVersion(self._version)
liveInstance.setLiveInstance("%s@%s" %
(os.getpid(), socket.gethostname()))
# liveInstance.setLiveInstance(ManagementFactory.getRuntimeMXBean().getName())
self.logger.info("Add live instance: InstanceName: " +
self._instanceName + " Session id:" +
str(self._sessionId))
# Builder
keyBuilder = self._helixAccessor.keyBuilder()
if not self._helixAccessor.createProperty(
keyBuilder.liveInstance(self._instanceName), liveInstance):
# String
errorMsg = "Fail to create live instance node after waiting, so quit. instance:" + self._instanceName
self.logger.warn(errorMsg)
# TODO: still raise
# raise HelixException(errorMsg)
# String
currentStatePathParent = PropertyPathConfig.getPath(
PropertyType.CURRENTSTATES, self._clusterName, self._instanceName,
self.getSessionId())
if not self._zkClient.exists(currentStatePathParent):
self._zkClient.createPersistent(currentStatePathParent)
self.logger.info("Creating current state path " +
currentStatePathParent)
def dropInstance(self, clusterName, instanceConfig):
"""
Returns void
Parameters:
clusterName: StringinstanceConfig: InstanceConfig
@Override
"""
# String
instanceConfigsPath = PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.PARTICIPANT.toString())
# String
nodeId = instanceConfig.getId()
# String
instanceConfigPath = instanceConfigsPath + "/" + nodeId
# String
instancePath = HelixUtil.getInstancePath(clusterName, nodeId)
if not _zkClient.exists(instanceConfigPath):
raise HelixException("Node " + nodeId + " does not exist in config for cluster " + clusterName)
if not _zkClient.exists(instancePath):
raise HelixException("Node " + nodeId + " does not exist in instances for cluster " + clusterName)
ZKUtil.dropChildren(_zkClient, instanceConfigsPath, instanceConfig.getRecord())
_zkClient.deleteRecursive(instancePath)
def addInstance(self, clusterName, instanceConfig):
"""
Returns void
Parameters:
clusterName: StringinstanceConfig: InstanceConfig
@Override
"""
if not ZKUtil.isClusterSetup(clusterName, _zkClient):
raise HelixException("cluster " + clusterName + " is not setup yet")
# String
instanceConfigsPath = PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.PARTICIPANT.toString())
# String
nodeId = instanceConfig.getId()
# String
instanceConfigPath = instanceConfigsPath + "/" + nodeId
if _zkClient.exists(instanceConfigPath):
raise HelixException("Node " + nodeId + " already exists in cluster " + clusterName)
ZKUtil.createChildren(_zkClient, instanceConfigsPath, instanceConfig.getRecord())
_zkClient.createPersistent(HelixUtil.getMessagePath(clusterName, nodeId), True)
_zkClient.createPersistent(HelixUtil.getCurrentStateBasePath(clusterName, nodeId), True)
_zkClient.createPersistent(HelixUtil.getErrorsPath(clusterName, nodeId), True)
_zkClient.createPersistent(HelixUtil.getStatusUpdatesPath(clusterName, nodeId), True)
def setProperty(self, type, value, keys):
"""
Returns boolean
Parameters:
type: PropertyTypevalue: ZNRecordkeys: String
@Override
"""
# String
path = PropertyPathConfig.getPath(type, _clusterName, keys)
# String
parent = File(path).getParent()
if not _zkClient.exists(parent):
_zkClient.createPersistent(parent, True)
if _zkClient.exists(path):
if type.isCreateOnlyIfAbsent():
return False
else:
ZKUtil.createOrUpdate(_zkClient, path, value, type.isPersistent(), False)
else:
try:
if type.isPersistent():
_zkClient.createPersistent(path, value)
else:
_zkClient.createEphemeral(path, value)
except Exception, e:
logger.warn("Exception while creating path:" + path + " Most likely due to race condition(Ignorable).", e)
return False
def isInstanceSetup(self):
"""
Returns boolean
Java modifiers:
private
"""
if self._instanceType == InstanceType.PARTICIPANT or self._instanceType == InstanceType.CONTROLLER_PARTICIPANT:
# boolean
isValid = self._zkClient.exists(PropertyPathConfig.getPath(PropertyType.CONFIGS, self._clusterName, ConfigScopeProperty.toString(ConfigScopeProperty.PARTICIPANT), self._instanceName)) \
and self._zkClient.exists(PropertyPathConfig.getPath(PropertyType.MESSAGES, self._clusterName, self._instanceName)) \
and self._zkClient.exists(PropertyPathConfig.getPath(PropertyType.CURRENTSTATES, self._clusterName, self._instanceName)) \
and self._zkClient.exists(PropertyPathConfig.getPath(PropertyType.STATUSUPDATES, self._clusterName, self._instanceName)) \
and self._zkClient.exists(PropertyPathConfig.getPath(PropertyType.ERRORS, self._clusterName, self._instanceName))
return isValid
return True
def isInstanceSetup(self):
"""
Returns boolean
Java modifiers:
private
"""
if self._instanceType == InstanceType.PARTICIPANT or self._instanceType == InstanceType.CONTROLLER_PARTICIPANT:
# boolean
isValid = self._zkClient.exists(PropertyPathConfig.getPath(PropertyType.CONFIGS, self._clusterName, ConfigScopeProperty.toString(ConfigScopeProperty.PARTICIPANT), self._instanceName)) \
and self._zkClient.exists(PropertyPathConfig.getPath(PropertyType.MESSAGES, self._clusterName, self._instanceName)) \
and self._zkClient.exists(PropertyPathConfig.getPath(PropertyType.CURRENTSTATES, self._clusterName, self._instanceName)) \
and self._zkClient.exists(PropertyPathConfig.getPath(PropertyType.STATUSUPDATES, self._clusterName, self._instanceName)) \
and self._zkClient.exists(PropertyPathConfig.getPath(PropertyType.ERRORS, self._clusterName, self._instanceName))
return isValid
return True
def addCluster(self, clusterName, overwritePrevRecord):
"""
Returns void
Parameters:
clusterName: StringoverwritePrevRecord: boolean
@Override
"""
# String
root = "/" + clusterName
# String
path = None
if _zkClient.exists(root):
logger.warn("Root directory exists.Cleaning the root directory:" + root + " overwritePrevRecord: " + overwritePrevRecord)
if overwritePrevRecord:
_zkClient.deleteRecursive(root)
else:
raise HelixException("Cluster " + clusterName + " already exists")
_zkClient.createPersistent(root)
_zkClient.createPersistent(HelixUtil.getIdealStatePath(clusterName))
path = PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.CLUSTER.toString(), clusterName)
_zkClient.createPersistent(path, True)
_zkClient.writeData(path, ZNRecord(clusterName))
path = PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.PARTICIPANT.toString())
_zkClient.createPersistent(path)
path = PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.RESOURCE.toString())
_zkClient.createPersistent(path)
path = PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, clusterName)
_zkClient.createPersistent(path)
_zkClient.createPersistent(HelixUtil.getLiveInstancesPath(clusterName))
_zkClient.createPersistent(HelixUtil.getMemberInstancesPath(clusterName))
_zkClient.createPersistent(HelixUtil.getExternalViewPath(clusterName))
_zkClient.createPersistent(HelixUtil.getStateModelDefinitionPath(clusterName))
_zkClient.createPersistent(HelixUtil.getControllerPath(clusterName))
path = PropertyPathConfig.getPath(PropertyType.HISTORY, clusterName)
# ZNRecord
emptyHistory = ZNRecord(PropertyType.HISTORY.toString())
# List<String>
emptyList = ArrayList<String>()
emptyHistory.setListField(clusterName, emptyList)
_zkClient.createPersistent(path, emptyHistory)
path = PropertyPathConfig.getPath(PropertyType.MESSAGES_CONTROLLER, clusterName)
_zkClient.createPersistent(path)
path = PropertyPathConfig.getPath(PropertyType.STATUSUPDATES_CONTROLLER, clusterName)
_zkClient.createPersistent(path)
path = PropertyPathConfig.getPath(PropertyType.ERRORS_CONTROLLER, clusterName)
_zkClient.createPersistent(path)
def removeProperty(self, type, keys):
"""
Returns boolean
Parameters:
type: PropertyTypekeys: String
@Override
"""
# String
path = PropertyPathConfig.getPath(type, _clusterName, keys)
return _zkClient.delete(path)
def getChildNames(self, type, keys):
"""
Returns List<String>
Parameters:
type: PropertyTypekeys: String
@Override
"""
# String
path = PropertyPathConfig.getPath(type, _clusterName, keys)
if _zkClient.exists(path):
return _zkClient.getChildren(path)
else:
return Collections.emptyList()
def getHelixPropertyStore(self):
"""
Returns ZkHelixPropertyStore<ZNRecord>
@Override
Java modifiers:
synchronized
"""
self.checkConnected()
if self._helixPropertyStore == None:
# String
path = PropertyPathConfig.getPath(PropertyType.HELIX_PROPERTYSTORE, self._clusterName)
_helixPropertyStore = ZkHelixPropertyStore<ZNRecord>(ZkBaseDataAccessor<ZNRecord>(self._zkClient), path, None)
return _helixPropertyStore
def getHelixPropertyStore(self):
"""
Returns ZkHelixPropertyStore<ZNRecord>
@Override
Java modifiers:
synchronized
"""
self.checkConnected()
if self._helixPropertyStore == None:
# String
path = PropertyPathConfig.getPath(PropertyType.HELIX_PROPERTYSTORE,
self._clusterName)
_helixPropertyStore = ZkHelixPropertyStore < ZNRecord > (
ZkBaseDataAccessor < ZNRecord > (self._zkClient), path, None)
return _helixPropertyStore
def addIdealStateChangeListener(self, listener):
"""
Returns void
Parameters:
listener: IdealStateChangeListener
@Override
Throws:
Exception
"""
self.logger.info("ClusterManager.addIdealStateChangeListener()")
self.checkConnected()
# String
path = PropertyPathConfig.getPath(PropertyType.IDEALSTATES, self._clusterName)
# CallbackHandler
callbackHandler = self.createCallBackHandler(path, listener, [ZKEventType.NodeDataChanged, ZKEventType.NodeDeleted, ZKEventType.NodeCreated], ChangeType.IDEAL_STATE)
self.addListener(callbackHandler)
def getPath(self):
"""
Returns String
"""
# String
clusterName = self._params[0]
# String[]
# subKeys = Arrays.copyOfRange(self._params, 1, self._params.length)
subKeys = self._params[1:]
# String
path = PropertyPathConfig.getPath(self._type, clusterName, *subKeys)
if path == None:
# self.LOG.error("Invalid property key with type:" + str(self._type)+ "subKeys:" + str(Arrays.toString(self._params)))
self.LOG.error("Invalid property key with type:" + str(self._type)+ "subKeys:" + str(str(self._params)))
return path
def dropStat(self, clusterName, statName):
"""
Returns void
Parameters:
clusterName: StringstatName: String
@Override
"""
if not ZKUtil.isClusterSetup(clusterName, _zkClient):
raise HelixException("cluster " + clusterName + " is not setup yet")
# String
persistentStatsPath = PropertyPathConfig.getPath(PropertyType.PERSISTENTSTATS, clusterName)
# ZkBaseDataAccessor<ZNRecord>
baseAccessor = ZkBaseDataAccessor<ZNRecord>(_zkClient)
baseAccessor.update(persistentStatsPath, DataUpdater<ZNRecord>() {
def update(self, statsRec):
"""
Returns ZNRecord
Parameters:
statsRec: ZNRecord
@Override
"""
if statsRec == None:
raise HelixException("No stats record in ZK, nothing to drop")
# Map<String, Map<String, String>>
currStatMap = statsRec.getMapFields()
# Map<String, Map<String, String>>
newStatMap = StatsHolder.parseStat(statName)
for # String
newStat = None
in newStatMap.keySet()) if currStatMap.containsKey(newStat):
currStatMap.remove(newStat)
statsRec.setMapFields(currStatMap)
return statsRec
def addAlert(self, clusterName, alertName):
"""
Returns void
Parameters:
clusterName: StringalertName: String
@Override
"""
if not ZKUtil.isClusterSetup(clusterName, _zkClient):
raise HelixException("cluster " + clusterName + " is not setup yet")
# ZkBaseDataAccessor<ZNRecord>
baseAccessor = ZkBaseDataAccessor<ZNRecord>(_zkClient)
# String
alertsPath = PropertyPathConfig.getPath(PropertyType.ALERTS, clusterName)
baseAccessor.update(alertsPath, DataUpdater<ZNRecord>() {
def update(self, alertsRec):
"""
Returns ZNRecord
Parameters:
alertsRec: ZNRecord
@Override
"""
if alertsRec == None:
alertsRec = ZNRecord(Alerts.nodeName)
# Map<String, Map<String, String>>
currAlertMap = alertsRec.getMapFields()
# StringBuilder
newStatName = StringBuilder()
# Map<String, String>
newAlertMap = HashMap<String, String>()
AlertsHolder.parseAlert(alertName, newStatName, newAlertMap)
addStat(clusterName, newStatName.toString())
currAlertMap.put(alertName, newAlertMap)
alertsRec.setMapFields(currAlertMap)
return alertsRec
def getInstanceConfig(self, clusterName, instanceName):
"""
Returns InstanceConfig
Parameters:
clusterName: StringinstanceName: String
@Override
"""
# String
instanceConfigPath = PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.PARTICIPANT.toString(), instanceName)
if not _zkClient.exists(instanceConfigPath):
raise HelixException("instance" + instanceName + " does not exist in cluster " + clusterName)
# ZKHelixDataAccessor
accessor = ZKHelixDataAccessor(clusterName, ZkBaseDataAccessor<ZNRecord>(_zkClient))
# Builder
keyBuilder = accessor.keyBuilder()
return accessor.getProperty(keyBuilder.instanceConfig(instanceName))
def getPath(self):
"""
Returns String
"""
# String
clusterName = self._params[0]
# String[]
# subKeys = Arrays.copyOfRange(self._params, 1, self._params.length)
subKeys = self._params[1:]
# String
path = PropertyPathConfig.getPath(self._type, clusterName, *subKeys)
if path == None:
# self.LOG.error("Invalid property key with type:" + str(self._type)+ "subKeys:" + str(Arrays.toString(self._params)))
self.LOG.error("Invalid property key with type:" +
str(self._type) + "subKeys:" +
str(str(self._params)))
return path
def dropAlert(self, clusterName, alertName):
"""
Returns void
Parameters:
clusterName: StringalertName: String
@Override
"""
if not ZKUtil.isClusterSetup(clusterName, _zkClient):
raise HelixException("cluster " + clusterName + " is not setup yet")
# String
alertsPath = PropertyPathConfig.getPath(PropertyType.ALERTS, clusterName)
# ZkBaseDataAccessor<ZNRecord>
baseAccessor = ZkBaseDataAccessor<ZNRecord>(_zkClient)
if not baseAccessor.exists(alertsPath, 0):
raise HelixException("No alerts node in ZK, nothing to drop")
baseAccessor.update(alertsPath, DataUpdater<ZNRecord>() {
def update(self, alertsRec):
"""
Returns ZNRecord
Parameters:
alertsRec: ZNRecord
@Override
"""
if alertsRec == None:
raise HelixException("No alerts record in ZK, nothing to drop")
# Map<String, Map<String, String>>
currAlertMap = alertsRec.getMapFields()
currAlertMap.remove(alertName)
alertsRec.setMapFields(currAlertMap)
return alertsRec
def getChildValues(self, type, keys):
"""
Returns List<ZNRecord>
Parameters:
type: PropertyTypekeys: String
@Override
"""
# String
path = PropertyPathConfig.getPath(type, _clusterName, keys)
if _zkClient.exists(path):
if not type.isCached():
return ZKUtil.getChildren(_zkClient, path)
else:
# Map<String, ZNRecord>
newChilds = refreshChildValuesCache(type, keys)
return ArrayList<ZNRecord>(newChilds.values())
return Collections.emptyList()
def addIdealStateChangeListener(self, listener):
"""
Returns void
Parameters:
listener: IdealStateChangeListener
@Override
Throws:
Exception
"""
self.logger.info("ClusterManager.addIdealStateChangeListener()")
self.checkConnected()
# String
path = PropertyPathConfig.getPath(PropertyType.IDEALSTATES,
self._clusterName)
# CallbackHandler
callbackHandler = self.createCallBackHandler(path, listener, [
ZKEventType.NodeDataChanged, ZKEventType.NodeDeleted,
ZKEventType.NodeCreated
], ChangeType.IDEAL_STATE)
self.addListener(callbackHandler)
def enableInstance(self, clusterName, instanceName, enabled):
"""
Returns void
Parameters:
clusterName: StringinstanceName: Stringenabled: boolean
@Override
"""
# String
path = PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.PARTICIPANT.toString(), instanceName)
# ZkBaseDataAccessor<ZNRecord>
baseAccessor = ZkBaseDataAccessor<ZNRecord>(_zkClient)
if not baseAccessor.exists(path, 0):
raise HelixException("Cluster " + clusterName + ", instance: " + instanceName + ", instance config does not exist")
baseAccessor.update(path, DataUpdater<ZNRecord>() {
def update(self, currentData):
"""
Returns ZNRecord
Parameters:
currentData: ZNRecord
@Override
"""
if currentData == None:
raise HelixException("Cluster: " + clusterName + ", instance: " + instanceName + ", participant config is null")
# InstanceConfig
config = InstanceConfig(currentData)
config.setInstanceEnabled(enabled)
return config.getRecord()
}, AccessOption.PERSISTENT)
def refreshChildValuesCache(self, type, keys):
"""
Returns Map<String, ZNRecord>
Parameters:
type: PropertyTypekeys: String
Java modifiers:
private
"""
if not type.isCached():
raise IllegalArgumentException("Type:" + type + " is NOT cached")
# String
path = PropertyPathConfig.getPath(type, _clusterName, keys)
# Map<String, ZNRecord>
newChilds = refreshChildValues(path, _cache.get(path))
if newChilds != None && newChilds.size() > 0:
_cache.put(path, newChilds)
return newChilds
else:
_cache.remove(path)
return Collections.emptyMap()
def getProperty(self, type, keys):
"""
Returns ZNRecord
Parameters:
type: PropertyTypekeys: String
@Override
"""
# String
path = PropertyPathConfig.getPath(type, _clusterName, keys)
if not type.isCached():
return _zkClient.readData(path, True)
else:
# int
len = keys.length
if len == 0:
return _zkClient.readData(path, True)
else:
# String[]
subkeys = Arrays.copyOfRange(keys, 0, len - 1)
# Map<String, ZNRecord>
newChilds = refreshChildValuesCache(type, subkeys)
return newChilds.get(keys[len - 1])
def invoke(self, changeContext):
"""
Returns void
Parameters:
changeContext: NotificationContext
Throws:
Exception
"""
# synchronized (self._manager) # Builder
keyBuilder = self._accessor.keyBuilder()
# long
start = time.time()
# start = System.currentTimeMillis()
# if self.logger.isInfoEnabled():
self.logger.info(str(threading.current_thread()) + " START:INVOKE " + self._path + " listener:" + self._listener.__class__.__name__)
if self._changeType == ChangeType.IDEAL_STATE:
# IdealStateChangeListener
idealStateChangeListener = self._listener
# idealStateChangeListener = (IdealStateChangeListener) self._listener
self.subscribeForChanges(changeContext, self._path, True, True)
# List<IdealState>
idealStates = self._accessor.getChildValues(keyBuilder.idealStates())
idealStateChangeListener.onIdealStateChange(idealStates, changeContext)
else:
if self._changeType == ChangeType.CONFIG:
# ConfigChangeListener
configChangeListener = self._listener
# configChangeListener = (ConfigChangeListener) self._listener
self.subscribeForChanges(changeContext, self._path, True, True)
# List<InstanceConfig>
configs = self._accessor.getChildValues(keyBuilder.instanceConfigs())
configChangeListener.onConfigChange(configs, changeContext)
else:
if self._changeType == ChangeType.LIVE_INSTANCE:
# LiveInstanceChangeListener
liveInstanceChangeListener = self._listener
# liveInstanceChangeListener = (LiveInstanceChangeListener) self._listener
self.subscribeForChanges(changeContext, self._path, True, True)
# List<LiveInstance>
liveInstances = self._accessor.getChildValues(keyBuilder.liveInstances())
liveInstanceChangeListener.onLiveInstanceChange(liveInstances, changeContext)
else:
if self._changeType == ChangeType.CURRENT_STATE:
# CurrentStateChangeListener
currentStateChangeListener = None
currentStateChangeListener = self._listener
# currentStateChangeListener = (CurrentStateChangeListener) self._listener
self.subscribeForChanges(changeContext, self._path, True, True)
# String
instanceName = PropertyPathConfig.getInstanceNameFromPath(self._path)
# String[]
pathParts = self._path.split("/")
# List<CurrentState>
currentStates = self._accessor.getChildValues(keyBuilder.currentStates(instanceName, pathParts[pathParts.length - 1]))
currentStateChangeListener.onStateChange(instanceName, currentStates, changeContext)
else:
if self._changeType == ChangeType.MESSAGE:
# MessageListener
messageListener = self._listener
# messageListener = (MessageListener) self._listener
self.subscribeForChanges(changeContext, self._path, True, False)
# String
instanceName = PropertyPathConfig.getInstanceNameFromPath(self._path)
# List<Message>
messages = self._accessor.getChildValues(keyBuilder.messages(instanceName))
messageListener.onMessage(instanceName, messages, changeContext)
else:
if self._changeType == ChangeType.MESSAGES_CONTROLLER:
# MessageListener
messageListener = self._listener
# messageListener = (MessageListener) self._listener
self.subscribeForChanges(changeContext, self._path, True, False)
# List<Message>
messages = self._accessor.getChildValues(keyBuilder.controllerMessages())
messageListener.onMessage(self._manager.getInstanceName(), messages, changeContext)
else:
if self._changeType == ChangeType.EXTERNAL_VIEW:
# ExternalViewChangeListener
externalViewListener = self._listener
# externalViewListener = (ExternalViewChangeListener) self._listener
self.subscribeForChanges(changeContext, self._path, True, True)
# List<ExternalView>
externalViewList = self._accessor.getChildValues(keyBuilder.externalViews())
externalViewListener.onExternalViewChange(externalViewList, changeContext)
else:
if self._changeType == ChangeType.CONTROLLER:
# ControllerChangeListener
controllerChangelistener = self._listener
# controllerChangelistener = (ControllerChangeListener) self._listener
self.subscribeForChanges(changeContext, self._path, True, False)
controllerChangelistener.onControllerChange(changeContext)
else:
if self._changeType == ChangeType.HEALTH:
# HealthStateChangeListener
healthStateChangeListener = self._listener
# healthStateChangeListener = (HealthStateChangeListener) self._listener
self.subscribeForChanges(changeContext, self._path, True, True)
# String
instanceName = PropertyPathConfig.getInstanceNameFromPath(self._path)
# List<HealthStat>
healthReportList = self._accessor.getChildValues(keyBuilder.healthReports(instanceName))
healthStateChangeListener.onHealthChange(instanceName, healthReportList, changeContext)
# long
# end = System.currentTimeMillis()
end = time.time()
# if self.logger.isInfoEnabled():
self.logger.info(str(threading.current_thread()) + " END:INVOKE " + self._path + " listener:" + self._listener.__class__.__name__ + " Took: " + str(end - start))
def isClusterSetup(clusterName, zkClient):
"""
Returns boolean
Parameters:
clusterName: StringzkClient: ZkClient
Java modifiers:
static
"""
if clusterName == None or zkClient == None:
return False
# boolean
isValid = zkClient.exists(PropertyPathConfig.getPath(PropertyType.IDEALSTATES, clusterName)) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.toString(ConfigScopeProperty.CLUSTER), clusterName)) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.toString(ConfigScopeProperty.PARTICIPANT))) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.toString(ConfigScopeProperty.RESOURCE))) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, clusterName)) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.LIVEINSTANCES, clusterName)) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.INSTANCES, clusterName)) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.EXTERNALVIEW, clusterName)) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.CONTROLLER, clusterName)) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.STATEMODELDEFS, clusterName)) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.MESSAGES_CONTROLLER, clusterName)) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.ERRORS_CONTROLLER, clusterName)) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.STATUSUPDATES_CONTROLLER, clusterName)) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.HISTORY, clusterName))
return isValid
mode = IdealStateModeProperty.valueOf(idealStateMode)
except Exception, e:
logger.error(""+ str(e))
# IdealState
idealState = IdealState(resourceName)
idealState.setNumPartitions(partitions)
idealState.setStateModelDefRef(stateModelRef)
idealState.setIdealStateMode(mode.toString())
idealState.setReplicas("" + 0)
idealState.setStateModelFactoryName(HelixConstants.DEFAULT_STATE_MODEL_FACTORY)
if bucketSize > 0:
idealState.setBucketSize(bucketSize)
# String
stateModelDefPath = PropertyPathConfig.getPath(PropertyType.STATEMODELDEFS, clusterName, stateModelRef)
if not _zkClient.exists(stateModelDefPath):
raise HelixException("State model " + stateModelRef + " not found in the cluster STATEMODELDEFS path")
# String
idealStatePath = HelixUtil.getIdealStatePath(clusterName)
# String
dbIdealStatePath = idealStatePath + "/" + resourceName
if _zkClient.exists(dbIdealStatePath):
raise HelixException("Skip the operation. DB ideal state directory exists:" + dbIdealStatePath)
ZKUtil.createChildren(_zkClient, idealStatePath, idealState.getRecord())