def enablePartition(self, enabled, clusterName, instanceName, resourceName, partitionNames):
"""
Returns void
Parameters:
enabled: booleanclusterName: StringinstanceName: StringresourceName: StringpartitionNames: List<String>
@Override
"""
# String
path = PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.PARTICIPANT.toString(), instanceName)
# ZkBaseDataAccessor<ZNRecord>
baseAccessor = ZkBaseDataAccessor<ZNRecord>(_zkClient)
if not baseAccessor.exists(path, 0):
raise HelixException("Cluster: " + clusterName + ", instance: " + instanceName + ", instance config does not exist")
# String
idealStatePath = PropertyPathConfig.getPath(PropertyType.IDEALSTATES, clusterName, resourceName)
# ZNRecord
idealStateRecord = None
try:
idealStateRecord = baseAccessor.get(idealStatePath, None, 0)
except ZkNoNodeException, e:
if idealStateRecord == None:
raise HelixException("Cluster: " + clusterName + ", resource: " + resourceName + ", ideal state does not exist")
# IdealState
idealState = IdealState(idealStateRecord)
for # String
partitionName = None
in partitionNames) if (idealState.getIdealStateMode() == IdealStateModeProperty.AUTO && idealState.getPreferenceList(partitionName) == None) or (idealState.getIdealStateMode() == IdealStateModeProperty.CUSTOMIZED && idealState.getInstanceStateMap(partitionName) == None):
logger.warn("Cluster: " + clusterName + ", resource: " + resourceName + ", partition: " + partitionName + ", partition does not exist in ideal state")
def rebalance(self, clusterName, resourceName, replica, keyPrefix):
"""
Returns void
Parameters:
clusterName: StringresourceName: Stringreplica: intkeyPrefix: String
"""
# List<String>
InstanceNames = getInstancesInCluster(clusterName)
Collections.sort(InstanceNames)
# IdealState
idealState = getResourceIdealState(clusterName, resourceName)
if idealState == None:
raise HelixException("Resource: " + resourceName + " has NOT been added yet")
idealState.setReplicas(Integer.toString(replica))
# int
partitions = idealState.getNumPartitions()
# String
stateModelName = idealState.getStateModelDefRef()
# StateModelDefinition
stateModDef = getStateModelDef(clusterName, stateModelName)
if stateModDef == None:
raise HelixException("cannot find state model: " + stateModelName)
# List<String>
statePriorityList = stateModDef.getStatesPriorityList()
# String
masterStateValue = None
# String
slaveStateValue = None
replica--
for # String
state = None
in statePriorityList) # String
count = stateModDef.getNumInstancesPerState(state)
if (count == "1"):
if masterStateValue != None:
raise HelixException("Invalid or unsupported state model definition")
masterStateValue = state
else:
if count.equalsIgnoreCase("R"):
if slaveStateValue != None:
raise HelixException("Invalid or unsupported state model definition")
slaveStateValue = state
else:
if count.equalsIgnoreCase("N"):
if not (masterStateValue == None && slaveStateValue == None):
raise HelixException("Invalid or unsupported state model definition")
replica = InstanceNames.size() - 1
masterStateValue = slaveStateValue = state
def addStateModelDef(self, clusterName, stateModelDef, stateModel):
"""
Returns void
Parameters:
clusterName: StringstateModelDef: StringstateModel: StateModelDefinition
@Override
"""
if not ZKUtil.isClusterSetup(clusterName, _zkClient):
raise HelixException("cluster " + clusterName + " is not setup yet")
# String
stateModelDefPath = HelixUtil.getStateModelDefinitionPath(clusterName)
# String
stateModelPath = stateModelDefPath + "/" + stateModelDef
if _zkClient.exists(stateModelPath):
logger.warn("Skip the operation.State Model directory exists:" + stateModelPath)
raise HelixException("State model path " + stateModelPath + " already exists.")
# ZKHelixDataAccessor
accessor = ZKHelixDataAccessor(clusterName, ZkBaseDataAccessor<ZNRecord>(_zkClient))
# Builder
keyBuilder = accessor.keyBuilder()
accessor.setProperty(keyBuilder.stateModelDef(stateModel.getId()), stateModel)
def addInstance(self, clusterName, instanceConfig):
"""
Returns void
Parameters:
clusterName: StringinstanceConfig: InstanceConfig
@Override
"""
if not ZKUtil.isClusterSetup(clusterName, _zkClient):
raise HelixException("cluster " + clusterName + " is not setup yet")
# String
instanceConfigsPath = PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.PARTICIPANT.toString())
# String
nodeId = instanceConfig.getId()
# String
instanceConfigPath = instanceConfigsPath + "/" + nodeId
if _zkClient.exists(instanceConfigPath):
raise HelixException("Node " + nodeId + " already exists in cluster " + clusterName)
ZKUtil.createChildren(_zkClient, instanceConfigsPath, instanceConfig.getRecord())
_zkClient.createPersistent(HelixUtil.getMessagePath(clusterName, nodeId), True)
_zkClient.createPersistent(HelixUtil.getCurrentStateBasePath(clusterName, nodeId), True)
_zkClient.createPersistent(HelixUtil.getErrorsPath(clusterName, nodeId), True)
_zkClient.createPersistent(HelixUtil.getStatusUpdatesPath(clusterName, nodeId), True)
def dropCluster(self, clusterName):
"""
Returns void
Parameters:
clusterName: String
@Override
"""
logger.info("Deleting cluster " + clusterName)
# ZKHelixDataAccessor
accessor = ZKHelixDataAccessor(clusterName, ZkBaseDataAccessor<ZNRecord>(_zkClient))
# Builder
keyBuilder = accessor.keyBuilder()
# String
root = "/" + clusterName
if accessor.getChildNames(keyBuilder.liveInstances()).size() > 0:
raise HelixException("There are still live instances in the cluster, shut them down first.")
if accessor.getProperty(keyBuilder.controllerLeader()) != None:
raise HelixException("There are still LEADER in the cluster, shut them down first.")
_zkClient.deleteRecursive(root)
def dropInstance(self, clusterName, instanceConfig):
"""
Returns void
Parameters:
clusterName: StringinstanceConfig: InstanceConfig
@Override
"""
# String
instanceConfigsPath = PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.PARTICIPANT.toString())
# String
nodeId = instanceConfig.getId()
# String
instanceConfigPath = instanceConfigsPath + "/" + nodeId
# String
instancePath = HelixUtil.getInstancePath(clusterName, nodeId)
if not _zkClient.exists(instanceConfigPath):
raise HelixException("Node " + nodeId + " does not exist in config for cluster " + clusterName)
if not _zkClient.exists(instancePath):
raise HelixException("Node " + nodeId + " does not exist in instances for cluster " + clusterName)
ZKUtil.dropChildren(_zkClient, instanceConfigsPath, instanceConfig.getRecord())
_zkClient.deleteRecursive(instancePath)
def verifyMessage(self, message):
"""
Returns void
Parameters:
message: Message
"""
if not message.getMsgType().toString().upper() == MessageType.TASK_REPLY.toString().upper():
# String
errorMsg = "Unexpected msg type for message " + message.getMsgId() + " type:" + message.getMsgType() + " Expected : " + MessageType.TASK_REPLY
self._logger.error(errorMsg)
raise HelixException(errorMsg)
# String
correlationId = message.getCorrelationId()
if correlationId == None:
# String
errorMsg = "Message " + message.getMsgId() + " does not have correlation id"
self._logger.error(errorMsg)
raise HelixException(errorMsg)
if not self._callbackMap.containsKey(correlationId):
# String
errorMsg = "Message " + message.getMsgId() + " does not have correponding callback. Probably timed out already. Correlation id: " + correlationId
self._logger.error(errorMsg)
raise HelixException(errorMsg)
self._logger.info("Verified reply message " + message.getMsgId() + " correlation:" + correlationId)
def resetPartition(self, clusterName, instanceName, resourceName, partitionNames):
"""
Returns void
Parameters:
clusterName: StringinstanceName: StringresourceName: StringpartitionNames: List<String>
@Override
"""
# ZKHelixDataAccessor
accessor = ZKHelixDataAccessor(clusterName, ZkBaseDataAccessor<ZNRecord>(_zkClient))
# Builder
keyBuilder = accessor.keyBuilder()
# LiveInstance
liveInstance = accessor.getProperty(keyBuilder.liveInstance(instanceName))
if liveInstance == None:
raise HelixException("Can't reset state for " + resourceName + "/" + partitionNames + " on " + instanceName + ", because " + instanceName + " is not alive")
# IdealState
idealState = accessor.getProperty(keyBuilder.idealStates(resourceName))
if idealState == None:
raise HelixException("Can't reset state for " + resourceName + "/" + partitionNames + " on " + instanceName + ", because " + resourceName + " is not added")
# Set<String>
resetPartitionNames = HashSet<String>(partitionNames)
if idealState.getIdealStateMode() == IdealStateModeProperty.CUSTOMIZED:
# Set<String>
partitions = HashSet<String>(idealState.getRecord().getMapFields().keySet())
if not partitions.containsAll(resetPartitionNames):
raise HelixException("Can't reset state for " + resourceName + "/" + partitionNames + " on " + instanceName + ", because not all " + partitionNames + " exist")
else:
# Set<String>
partitions = HashSet<String>(idealState.getRecord().getListFields().keySet())
if not partitions.containsAll(resetPartitionNames):
raise HelixException("Can't reset state for " + resourceName + "/" + partitionNames + " on " + instanceName + ", because not all " + partitionNames + " exist")
# String
sessionId = liveInstance.getSessionId()
# CurrentState
curState = accessor.getProperty(keyBuilder.currentState(instanceName, sessionId, resourceName))
for # String
partitionName = None
in resetPartitionNames) if not (curState.getState(partitionName) == "ERROR"):
raise HelixException("Can't reset state for " + resourceName + "/" + partitionNames + " on " + instanceName + ", because not all " + partitionNames + " are in ERROR state")
def registerStateModelFactory(
self,
stateModelName,
factory,
factoryName=HelixConstants.DEFAULT_STATE_MODEL_FACTORY):
"""
Returns boolean
Parameters:
stateModelName: Stringfactory: StateModelFactory<? extends StateModel>factoryName: String
@Override
"""
if stateModelName == None or factory == None or factoryName == None:
raise HelixException(
"stateModelDef|stateModelFactory|factoryName cannot be null")
self.logger.info("Register state model factory for state model " +
stateModelName + " using factory name " +
factoryName + " with " + str(factory))
if not self._stateModelFactoryMap.__contains__(stateModelName):
self._stateModelFactoryMap.__setitem__(stateModelName, {})
# self._stateModelFactoryMap.__setitem__(stateModelName, ConcurrentHashMap<String, StateModelFactory<? extends StateModel>>())
if self._stateModelFactoryMap.get(stateModelName).__contains__(
factoryName):
self.logger.warn("stateModelFactory for " + stateModelName +
" using factoryName " + factoryName +
" has already been registered.")
return False
self._stateModelFactoryMap.get(stateModelName).__setitem__(
factoryName, factory)
self.sendNopMessage()
return True
def setProperty(self, key, value):
"""
Returns boolean
Parameters:
key: PropertyKeyvalue: T
@Override
Parameterized: <T extends HelixProperty>
"""
# PropertyType
propertyType = key.getType()
if not value.isValid():
raise HelixException("The ZNRecord for " + propertyType +
" is not valid.")
# String
path = key.getPath()
# int
options = self.constructOptions(propertyType)
# if propertyType.usePropertyTransferServer():
# if self._zkPropertyTransferSvcUrl != None and self._zkPropertyTransferClient != None:
# # ZNRecordUpdate
# update = ZNRecordUpdate(path, OpCode.SET, value.getRecord())
# self._zkPropertyTransferClient.enqueueZNRecordUpdate(update, self._zkPropertyTransferSvcUrl)
# return True
# boolean
success = False
if propertyType in (PropertyType.IDEALSTATES,
PropertyType.EXTERNALVIEW):
# if value.getBucketSize() > 0:
# # ZNRecord
# metaRecord = ZNRecord(value.getId())
# metaRecord.setSimpleFields(value.getRecord().getSimpleFields())
# success = self._baseDataAccessor.set(path, metaRecord, options)
# if success:
# # ZNRecordBucketizer
# bucketizer = ZNRecordBucketizer(value.getBucketSize())
# # Map<String, ZNRecord>
# map = bucketizer.bucketize(value.getRecord())
# # List<String>
# paths = []
## paths = ArrayList<String>()
# # List<ZNRecord>
# bucketizedRecords = []
## bucketizedRecords = ArrayList<ZNRecord>()
# for bucketName in map:
# paths.add(path + "/" + bucketName)
# bucketizedRecords.add(map.get(bucketName))
# self._baseDataAccessor.setChildren(paths, bucketizedRecords, options)
# else:
# success = self._baseDataAccessor.set(path, value.getRecord(), options)
# dzhang: skip the bucketizer
success = self._baseDataAccessor.set(path, value.getRecord(),
options)
else:
success = self._baseDataAccessor.set(path, value.getRecord(),
options)
return success
def serialize(self, data):
"""
Returns byte[]
Parameters:
data: T
@Override
Throws:
PropertyStoreException
"""
# ObjectMapper
mapper = ObjectMapper()
# SerializationConfig
serializationConfig = mapper.getSerializationConfig()
serializationConfig.set(SerializationConfig.Feature.INDENT_OUTPUT, True)
serializationConfig.set(SerializationConfig.Feature.AUTO_DETECT_FIELDS, True)
serializationConfig.set(SerializationConfig.Feature.CAN_OVERRIDE_ACCESS_MODIFIERS, True)
# StringWriter
sw = StringWriter()
try:
mapper.writeValue(sw, data)
if sw.toString().getBytes().length > ZNRecord.SIZE_LIMIT:
raise HelixException("Data size larger than 1M. Write empty string to zk.")
return sw.toString().getBytes()
except Exception, e:
LOG.error("Error during serialization of data (first 1k): " + str(str(sw.toString().substring(0)+ str(1024)))+ str(e))
def handleNewSessionAsParticipant(self):
"""
Returns void
Java modifiers:
private
"""
# Builder
keyBuilder = self._helixAccessor.keyBuilder()
if self._helixAccessor.getProperty(
keyBuilder.liveInstance(self._instanceName)) != None:
self.logger.warn(
"Found another instance with same instanceName: " +
self._instanceName + " in cluster " + self._clusterName)
try:
time.sleep((self._sessionTimeout + 5000) / 1000.0)
except Exception, e:
self.logger.warn(
"Sleep interrupted while waiting for previous liveinstance to go away.",
e)
if self._helixAccessor.getProperty(
keyBuilder.liveInstance(self._instanceName)) != None:
# String
errorMessage = "instance " + self._instanceName + " already has a liveinstance in cluster " + self._clusterName
self.logger.error(errorMessage)
raise HelixException(errorMessage)
def createReplyMessage(srcMessage, instanceName, taskResultMap):
"""
Returns Message
Parameters:
srcMessage: MessageinstanceName: StringtaskResultMap: Map<String, String>
Java modifiers:
static
"""
if srcMessage.getCorrelationId() == None:
raise HelixException("Message " + srcMessage.getMsgId() + " does not contain correlation id")
# Message
replyMessage = Message(MessageType.TASK_REPLY, str(uuid.uuid4()))
replyMessage.setCorrelationId(srcMessage.getCorrelationId())
replyMessage.setResultMap(taskResultMap)
replyMessage.setTgtSessionId("*")
replyMessage.setMsgState(MessageState.NEW)
replyMessage.setSrcName(instanceName)
if srcMessage.getSrcInstanceType() == InstanceType.CONTROLLER:
replyMessage.setTgtName("Controller")
else:
replyMessage.setTgtName(srcMessage.getMsgSrc())
return replyMessage
def get_list(self, scope, keyList):
if scope is None or scope.getScope() is None:
self.LOG.error("Scope can't be null")
return None
# String
value = None
# String
clusterName = scope.getClusterName()
if not ZKUtil.isClusterSetup(clusterName, self.zkClient):
raise HelixException("cluster " + clusterName +
" is not setup yet")
# String
scopeStr = scope.getScopeStr()
retDict = {}
# String[]
# splits = scopeStr.split("\|")
splits = re.split("[\\|]", scopeStr)
# ZNRecord
record = self.zkClient.readData(splits[0], True)
if record is not None:
for key in keyList:
if len(splits) == 1:
value = record.getSimpleField(key)
else:
if splits.length == 2:
if record.getMapField(splits[1]) is not None:
value = record.getMapField(splits[1]).get(key)
retDict[key] = value
return retDict
def registerMessageHandlerFactory(self,
type,
factory,
threadpoolSize=DEFAULT_PARALLEL_TASKS):
"""
Returns void
Parameters:
type: Stringfactory: MessageHandlerFactorythreadpoolSize: int
"""
if not self._handlerFactoryMap.__contains__(type):
if not type.upper() == factory.getMessageType().upper():
raise HelixException("Message factory type mismatch. Type: " +
type + " factory : " +
factory.getMessageType())
self._handlerFactoryMap.__setitem__(type, factory)
self._threadpoolMap.__setitem__(
type, futures.ThreadPoolExecutor(threadpoolSize))
# self._threadpoolMap.__setitem__(type, Executors.newFixedThreadPool(threadpoolSize))
self.LOG.info("Adding msg factory for type " + type +
" threadpool size " + str(threadpoolSize))
else:
self.LOG.error("Ignoring duplicate msg handler factory for type " +
str(type))
def getClusterManagmentTool(self):
"""
Returns HelixAdmin
@Override
Java modifiers:
synchronized
"""
raise HelixException("Not implemented")
def checkConnected(self):
"""
Returns void
Java modifiers:
private
"""
if not self.isConnected():
raise HelixException(
"ClusterManager not connected. Call clusterManager.connect()")
def onControllerChange(self, changeContext):
"""
Returns void
Parameters:
changeContext: NotificationContext
@Override
"""
raise HelixException("Not implemented")
def dropAlert(self, clusterName, alertName):
"""
Returns void
Parameters:
clusterName: StringalertName: String
@Override
"""
if not ZKUtil.isClusterSetup(clusterName, _zkClient):
raise HelixException("cluster " + clusterName + " is not setup yet")
# String
alertsPath = PropertyPathConfig.getPath(PropertyType.ALERTS, clusterName)
# ZkBaseDataAccessor<ZNRecord>
baseAccessor = ZkBaseDataAccessor<ZNRecord>(_zkClient)
if not baseAccessor.exists(alertsPath, 0):
raise HelixException("No alerts node in ZK, nothing to drop")
baseAccessor.update(alertsPath, DataUpdater<ZNRecord>() {
def update(self, alertsRec):
"""
Returns ZNRecord
Parameters:
alertsRec: ZNRecord
@Override
"""
if alertsRec == None:
raise HelixException("No alerts record in ZK, nothing to drop")
# Map<String, Map<String, String>>
currAlertMap = alertsRec.getMapFields()
currAlertMap.remove(alertName)
alertsRec.setMapFields(currAlertMap)
return alertsRec
def addCluster(self, clusterName, overwritePrevRecord):
"""
Returns void
Parameters:
clusterName: StringoverwritePrevRecord: boolean
@Override
"""
# String
root = "/" + clusterName
# String
path = None
if _zkClient.exists(root):
logger.warn("Root directory exists.Cleaning the root directory:" + root + " overwritePrevRecord: " + overwritePrevRecord)
if overwritePrevRecord:
_zkClient.deleteRecursive(root)
else:
raise HelixException("Cluster " + clusterName + " already exists")
_zkClient.createPersistent(root)
_zkClient.createPersistent(HelixUtil.getIdealStatePath(clusterName))
path = PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.CLUSTER.toString(), clusterName)
_zkClient.createPersistent(path, True)
_zkClient.writeData(path, ZNRecord(clusterName))
path = PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.PARTICIPANT.toString())
_zkClient.createPersistent(path)
path = PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.RESOURCE.toString())
_zkClient.createPersistent(path)
path = PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, clusterName)
_zkClient.createPersistent(path)
_zkClient.createPersistent(HelixUtil.getLiveInstancesPath(clusterName))
_zkClient.createPersistent(HelixUtil.getMemberInstancesPath(clusterName))
_zkClient.createPersistent(HelixUtil.getExternalViewPath(clusterName))
_zkClient.createPersistent(HelixUtil.getStateModelDefinitionPath(clusterName))
_zkClient.createPersistent(HelixUtil.getControllerPath(clusterName))
path = PropertyPathConfig.getPath(PropertyType.HISTORY, clusterName)
# ZNRecord
emptyHistory = ZNRecord(PropertyType.HISTORY.toString())
# List<String>
emptyList = ArrayList<String>()
emptyHistory.setListField(clusterName, emptyList)
_zkClient.createPersistent(path, emptyHistory)
path = PropertyPathConfig.getPath(PropertyType.MESSAGES_CONTROLLER, clusterName)
_zkClient.createPersistent(path)
path = PropertyPathConfig.getPath(PropertyType.STATUSUPDATES_CONTROLLER, clusterName)
_zkClient.createPersistent(path)
path = PropertyPathConfig.getPath(PropertyType.ERRORS_CONTROLLER, clusterName)
_zkClient.createPersistent(path)
def addClusterToGrandCluster(self, clusterName, grandCluster):
"""
Returns void
Parameters:
clusterName: StringgrandCluster: String
@Override
"""
if not ZKUtil.isClusterSetup(grandCluster, _zkClient):
raise HelixException("Grand cluster " + grandCluster + " is not setup yet")
if not ZKUtil.isClusterSetup(clusterName, _zkClient):
raise HelixException("Cluster " + clusterName + " is not setup yet")
# IdealState
idealState = IdealState(clusterName)
idealState.setNumPartitions(1)
idealState.setStateModelDefRef("LeaderStandby")
# List<String>
controllers = getInstancesInCluster(grandCluster)
if controllers.size() == 0:
raise HelixException("Grand cluster " + grandCluster + " has no instances")
idealState.setReplicas(Integer.toString(controllers.size()))
Collections.shuffle(controllers)
idealState.getRecord().setListField(clusterName, controllers)
idealState.setPartitionState(clusterName, controllers.get(0), "LEADER")
for (# int
i = 1; i < controllers.size(); i++) idealState.setPartitionState(clusterName, controllers.get(i), "STANDBY")
# ZKHelixDataAccessor
accessor = ZKHelixDataAccessor(grandCluster, ZkBaseDataAccessor<ZNRecord>(_zkClient))
# Builder
keyBuilder = accessor.keyBuilder()
accessor.setProperty(keyBuilder.idealStates(idealState.getResourceName()), idealState)
def enableInstance(self, clusterName, instanceName, enabled):
"""
Returns void
Parameters:
clusterName: StringinstanceName: Stringenabled: boolean
@Override
"""
# String
path = PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.PARTICIPANT.toString(), instanceName)
# ZkBaseDataAccessor<ZNRecord>
baseAccessor = ZkBaseDataAccessor<ZNRecord>(_zkClient)
if not baseAccessor.exists(path, 0):
raise HelixException("Cluster " + clusterName + ", instance: " + instanceName + ", instance config does not exist")
baseAccessor.update(path, DataUpdater<ZNRecord>() {
def update(self, currentData):
"""
Returns ZNRecord
Parameters:
currentData: ZNRecord
@Override
"""
if currentData == None:
raise HelixException("Cluster: " + clusterName + ", instance: " + instanceName + ", participant config is null")
# InstanceConfig
config = InstanceConfig(currentData)
config.setInstanceEnabled(enabled)
return config.getRecord()
}, AccessOption.PERSISTENT)
def __init__(self, clusterName, instanceName, instanceType,
zkConnectString):
self.logger.info("Create a zk-based cluster manager. clusterName:" +
clusterName + ", instanceName:" + instanceName +
", type:" + InstanceType.toString(instanceType) +
", zkSvr:" + zkConnectString)
# print("Create a zk-based cluster manager. clusterName:" + clusterName + ", instanceName:" + instanceName + ", type:" + InstanceType.toString(instanceType) + ", zkSvr:" + zkConnectString)
# int
sessionTimeoutInt = -1
sessionTimeoutInt = self.DEFAULT_SESSION_TIMEOUT
# try:
# sessionTimeoutInt = int(env["zk.session.timeout", "" + self.DEFAULT_SESSION_TIMEOUT))
# except ValueError, e:
# self.logger.warn("Exception while parsing session timeout: " + System.getProperty("zk.session.timeout", "" + self.DEFAULT_SESSION_TIMEOUT))
if sessionTimeoutInt > 0:
self._sessionTimeout = sessionTimeoutInt
else:
self._sessionTimeout = self.DEFAULT_SESSION_TIMEOUT
if instanceName == None:
# try:
instanceName = socket.gethostbyname(
socket.gethostname()) + "-" + instanceType.toString()
# except UnknownHostException, e:
# self.logger.info("Unable to get host name. Will set it to UNKNOWN, mostly ignorable", e)
# instanceName = "UNKNOWN"
self._clusterName = clusterName
self._instanceName = instanceName
self._instanceType = instanceType
self._zkConnectString = zkConnectString
self._zkStateChangeListener = ZkStateChangeListener(self)
self._timer = None
# self._handlers = ArrayList<CallbackHandler>()
self._handlers = []
self._messagingService = DefaultMessagingService(self)
# TODO: fix ther version
#self._version = PropertiesReader("cluster-manager-version.properties").getProperty("clustermanager.version")
self._version = '0.5.1'
self._stateMachEngine = HelixStateMachineEngine(self)
# self._controllerTimerTasks = ArrayList<HelixTimerTask>()
self._controllerTimerTasks = []
if self._instanceType == InstanceType.CONTROLLER:
raise HelixException(
"ERROR. Cannot handle CONTROLLER type for now!")
# self._controllerTimerTasks.add(HealthStatsAggregationTask(self))
self._preConnectCallbacks = []
def createCallBackHandler(self, path, listener, eventTypes, changeType):
"""
Returns CallbackHandler
Parameters:
path: Stringlistener: ObjecteventTypes: EventType[]changeType: ChangeType
Java modifiers:
private
"""
if listener == None:
raise HelixException("Listener cannot be null")
return CallbackHandler(self, self._zkClient, path, listener,
eventTypes, changeType)
def serialize(self, data):
"""
Returns byte[]
Parameters:
data: Object
@Override
"""
if not (type(data) == ZNRecord):
self.logger.error("Input object must be of type ZNRecord but it is " + str(data)+ ". Will not write to zk")
raise HelixException("Input object is not of type ZNRecord (was " + data + ")")
return str(data)
def setProperty(self, type, value, keys):
"""
Returns boolean
Parameters:
type: PropertyTypevalue: HelixPropertykeys: String
@Override
"""
if not value.isValid():
raise HelixException("The ZNRecord for " + type + " is not valid.")
return setProperty(type, value.getRecord(), keys)
def createHandler(self, message, context):
"""
Returns MessageHandler
Parameters:
message: Messagecontext: NotificationContext
@Override
"""
# String
type = message.getMsgType()
if not (type == getMessageType()):
raise HelixException("Unexpected msg type for message " + message.getMsgId() + " type:" + message.getMsgType())
return DefaultSchedulerMessageHandler(message, context, _manager)
def prepareMessageExecution(self, manager, message):
"""
Returns void
Parameters:
manager: HelixManagermessage: Message
Java modifiers:
private
Throws:
HelixException
HelixStateMismatchException
"""
if not message.isValid():
# String
# errorMessage = "Invalid Message, ensure that message: " + message + " has all the required fields: " + str(Message.Attributes.values())
errorMessage = "Invalid Message, ensure that message: " + message + " has all the required fields: " + str(
Attributes.values())
self._statusUpdateUtil.logError(message,
HelixStateTransitionHandler,
errorMessage,
manager.getHelixDataAccessor())
self.logger.error(errorMessage)
raise HelixException(errorMessage)
# HelixDataAccessor
accessor = manager.getHelixDataAccessor()
# String
partitionName = message.getPartitionName()
# String
fromState = message.getFromState()
# String
state = self._currentStateDelta.getState(partitionName)
if fromState != None and (
fromState == "*") and not fromState.upper() == state.upper():
# String
errorMessage = "Current state of stateModel does not match the fromState in Message" + ", Current State:" + str(
state) + ", message expected:" + str(
fromState) + ", partition: " + str(
partitionName) + ", from: " + message.getMsgSrc(
) + ", to: " + message.getTgtName()
self._statusUpdateUtil.logError(message,
HelixStateTransitionHandler,
errorMessage, accessor)
self.logger.error(errorMessage)
raise HelixStateMismatchException(errorMessage)
def update(self, alertsRec):
"""
Returns ZNRecord
Parameters:
alertsRec: ZNRecord
@Override
"""
if alertsRec == None:
raise HelixException("No alerts record in ZK, nothing to drop")
# Map<String, Map<String, String>>
currAlertMap = alertsRec.getMapFields()
currAlertMap.remove(alertName)
alertsRec.setMapFields(currAlertMap)
return alertsRec
def dropStat(self, clusterName, statName):
"""
Returns void
Parameters:
clusterName: StringstatName: String
@Override
"""
if not ZKUtil.isClusterSetup(clusterName, _zkClient):
raise HelixException("cluster " + clusterName + " is not setup yet")
# String
persistentStatsPath = PropertyPathConfig.getPath(PropertyType.PERSISTENTSTATS, clusterName)
# ZkBaseDataAccessor<ZNRecord>
baseAccessor = ZkBaseDataAccessor<ZNRecord>(_zkClient)
baseAccessor.update(persistentStatsPath, DataUpdater<ZNRecord>() {
def update(self, statsRec):
"""
Returns ZNRecord
Parameters:
statsRec: ZNRecord
@Override
"""
if statsRec == None:
raise HelixException("No stats record in ZK, nothing to drop")
# Map<String, Map<String, String>>
currStatMap = statsRec.getMapFields()
# Map<String, Map<String, String>>
newStatMap = StatsHolder.parseStat(statName)
for # String
newStat = None
in newStatMap.keySet()) if currStatMap.containsKey(newStat):
currStatMap.remove(newStat)
statsRec.setMapFields(currStatMap)
return statsRec