class ZNRecordStreamingSerializer():
"""
Java modifiers:
private static
Type:
Logger
"""
LOG = get_logger(__name__)
@staticmethod
def getListFieldBound(record):
"""
Returns int
Parameters:
record: ZNRecord
Java modifiers:
private static
"""
# int
max = sys.maxint
if record.getSimpleFields().__contains__(ZNRecord.LIST_FIELD_BOUND):
# String
maxStr = record.getSimpleField(ZNRecord.LIST_FIELD_BOUND)
try:
max = int(maxStr)
except ValueError, e:
ZNRecordStreamingSerializer.LOG.error(
"IllegalNumberFormat for list field bound: " + str(maxStr))
return max
class ZNRecordSerializer(ZkSerializer):
"""
Java modifiers:
private static
Type:
Logger
"""
logger = get_logger(__name__)
def getListFieldBound(self, record):
"""
Returns int
Parameters:
record: ZNRecord
Java modifiers:
private static
"""
# int
# max = Integer.MAX_VALUE
max = sys.maxint
if record.getSimpleFields().containsKey(ZNRecord.LIST_FIELD_BOUND):
# String
maxStr = record.getSimpleField(ZNRecord.LIST_FIELD_BOUND)
try:
max = int(maxStr)
except ValueError, e:
self.logger.error("IllegalNumberFormat for list field bound: " + str(maxStr))
return max
class ZNRecordJsonSerializer(PropertySerializer):
"""
Java modifiers:
private static
Type:
Logger
"""
LOG = get_logger(__name__)
def serialize(self, data):
"""
Returns byte[]
Parameters:
data: ZNRecord
@Override
Throws:
PropertyStoreException
"""
return self._serializer.serialize(data)
def deserialize(self, bytes):
"""
Returns ZNRecord
Parameters:
bytes: byte[]
@Override
Throws:
PropertyStoreException
"""
# return (ZNRecord) self._serializer.deserialize(bytes)
return self._serializer.deserialize(bytes)
class CriteriaEvaluator:
"""
Java modifiers:
private static
Type:
Logger
"""
logger = get_logger(__name__)
def evaluateCriteria(self, recipientCriteria, manager):
"""
Returns List<Map<String, String>>
Parameters:
recipientCriteria: Criteriamanager: HelixManager
"""
# List<Map<String, String>>
selected = []
# # TODO: Skip the query for now
# selected = ArrayList<Map<String, String>>()
# # String
# queryFields = ternary(not (recipientCriteria.getInstanceName() == ""), " " + ZNRecordRow.MAP_SUBKEY, " ''") + "," + (ternary(not (recipientCriteria.getResource() == ""), " " + ZNRecordRow.ZNRECORD_ID, " ''")) + "," + (ternary(not (recipientCriteria.getPartition() == ""), " " + ZNRecordRow.MAP_KEY, " ''")) + "," + (ternary(not (recipientCriteria.getPartitionState() == ""), " " + ZNRecordRow.MAP_VALUE, " '' "))
# # String
# matchCondition = ZNRecordRow.MAP_SUBKEY + " LIKE '" + (ternary(not (recipientCriteria.getInstanceName() == ""), (recipientCriteria.getInstanceName() + "'"), "%' ")) + " AND " + ZNRecordRow.ZNRECORD_ID + " LIKE '" + (ternary(not (recipientCriteria.getResource() == ""), (recipientCriteria.getResource() + "'"), "%' ")) + " AND " + ZNRecordRow.MAP_KEY + " LIKE '" + (ternary(not (recipientCriteria.getPartition() == ""), (recipientCriteria.getPartition() + "'"), "%' ")) + " AND " + ZNRecordRow.MAP_VALUE + " LIKE '" + (ternary(not (recipientCriteria.getPartitionState() == ""), (recipientCriteria.getPartitionState() + "'"), "%' ")) + " AND " + ZNRecordRow.MAP_SUBKEY + " IN ((SELECT [*]id FROM :LIVEINSTANCES))"
# # String
# queryTarget = recipientCriteria.getDataSource().toString() + ClusterJosqlQueryProcessor.FLATTABLE
# # String
# josql = "SELECT DISTINCT " + queryFields + " FROM " + queryTarget + " WHERE " + matchCondition
# # ClusterJosqlQueryProcessor
# # TODO: Skip the query for now
## p = ClusterJosqlQueryProcessor(manager)
# # List<Object>
# result = []
## result = ArrayList<Object>()
# try:
# self.logger.info("JOSQL query: " + josql)
# # TODO: Skip the query for now
## result = p.runJoSqlQuery(josql, None, None)
# except Exception, e:
# self.logger.error(""+ str(e))
# return selected
#
# for o in result: # Map<String, String>
## resultRow = HashMap<String, String>()
# resultRow = {}
# # List<Object>
# row = o
## row = (List<Object>) o
# resultRow.__setitem__("instanceName", (row[0]))
# resultRow.__setitem__("resourceName", (row[1]))
# resultRow.__setitem__("partitionName",(row[2]))
# resultRow.__setitem__("partitionState", (row[3]))
# selected.append(resultRow)
#
# self.logger.info("JOSQL query return " + selected.size() + " rows")
return selected
class ZNRecordBucketizer:
"""
Java modifiers:
private static
Type:
Logger
"""
LOG = get_logger(__name__)
"""
Parameters:
int bucketSize
"""
def __init__(self, bucketSize):
if bucketSize <= 0:
self.LOG.debug("bucketSize <= 0 (was " + str(bucketSize) +
"). Set to 0 to use non-bucketized HelixProperty.")
bucketSize = 0
self._bucketSize = bucketSize
def getBucketName(self, key):
"""
Returns String
Parameters:
key: String
"""
if self._bucketSize == 0:
return None
# int
idx = key.lastIndexOf('_')
if idx < 0:
raise IllegalArgumentException(
"Could NOT find partition# in " + key +
". partitionName should be in format of resourceName_partition#"
)
try:
# int
partitionNb = int(key.substring(idx + 1))
# int
bucketNb = partitionNb / self._bucketSize
# int
startPartition = bucketNb * self._bucketSize
# int
endPartition = bucketNb * self._bucketSize + (self._bucketSize - 1)
return key.substring(
0, idx) + "_p" + startPartition + "-p" + endPartition
except ValueError, e:
raise IllegalArgumentException("Could NOT parse partition# (" +
key.substring(idx + 1) + ") in " +
key)
class SystemUtil(object):
OS_NAME = platform.system()
logger = get_logger(__name__)
@staticmethod
def getProcessState(processId):
"""
Returns ProcessStateCode
Parameters:
processId: String
Java modifiers:
static
Throws:
Exception
"""
if (SystemUtil.OS_NAME == "Mac OS X") or (SystemUtil.OS_NAME
== "Linux"):
# ExternalCommand
# cmd = ExternalCommand.start("ps", processId)
# cmd.waitFor()
output = os.popen("ps %s" % processId).read()
# String
# lines = cmd.getStringOutput().split("\n")
lines = output.split("\n")
if len(lines) != 2:
SystemUtil.logger.info("process: " + processId + " not exist")
return None
attributes = re.split("\s+", lines[0].strip())
values = re.split("\s+", lines[1].strip())
# Character
processStateCodeChar = None
for i in range(len(attributes)):
attribute = attributes[i]
if ("STAT" == attribute) or ("S" == attribute):
processStateCodeChar = values[i][0]
break
return ProcessStateCode.valueOf(processStateCodeChar)
else:
raise ("Not supported OS: " + SystemUtil.OS_NAME)
@staticmethod
def getPidFromFile(file):
try:
with open(file) as pidFile:
line = pidFile.readline()
return line
except IOError, e:
SystemUtil.logger.error("fail to read pid from pidFile: " + file,
e)
finally:
class HealthStat(HelixProperty):
"""
Java modifiers:
private final static
Type:
Logger
"""
_logger = get_logger(__name__)
"""
Parameters:
String id
"""
def __init__(self, id):
super(id)
"""
Parameters:
ZNRecord record
"""
def __init__(self, record):
super(record)
if self.getCreateTimeStamp() == 0:
self._record.setSimpleField(Attributes.CREATE_TIMESTAMP.toString(),
"" + time.time())
def getLastModifiedTimeStamp(self):
"""
Returns long
"""
return self._record.getModifiedTime()
def getCreateTimeStamp(self):
"""
Returns long
"""
if self._record.getSimpleField(
Attributes.CREATE_TIMESTAMP.toString()) == None:
return 0
try:
return long(
self._record.getSimpleField(
Attributes.toString(Attributes.CREATE_TIMESTAMP)))
except ValueError, e:
return 0
class GroupCommit:
"""
Java modifiers:
private static
Type:
Logger
"""
# LOG = Logger.getLogger(GroupCommit.class)
LOG = get_logger(__name__)
# Java modifiers:
# private static
# class Queue:
class Entry:
"""
Parameters:
String key
ZNRecord record
"""
def __init__(self, key, record):
self._key = key
self._record = record
"""
"""
def __init__(self):
self._queues = []
for i in range(100):
self._queues.append(Queue())
def getQueue(self, key):
"""
Returns Queue
Parameters:
key: String
Java modifiers:
private
"""
return self._queues[hash(key) % self._queues.__len__()]
def commit(self, accessor, options, key, record):
accessor.set(key, record, options)
class ProcessMonitorThread(threading.Thread):
"""
Java modifiers:
private final static
Type:
Logger
"""
logger = get_logger(__name__)
"""
Java modifiers:
private final static
Type:
int
"""
MONITOR_PERIOD_BASE = 1000
"""
Parameters:
String pid
"""
def __init__(self, pid):
self._pid = pid
def run(self):
"""
Returns void
@Override
"""
try:
# ProcessStateCode
processState = SystemUtil.getProcessState(self._pid)
while processState != None:
if processState == ProcessStateCode.Z:
self.logger.error("process: " + self._pid +
" is in zombie state")
break
time.sleep((random.randint(
0, ProcessMonitorThread.MONITOR_PERIOD_BASE) +
ProcessMonitorThread.MONITOR_PERIOD_BASE) / 1000.0)
processState = SystemUtil.getProcessState(self._pid)
except Exception, e:
ProcessMonitorThread.logger.error(
"fail to monitor process: " + self._pid, e)
class ClusterConstraints(HelixProperty):
"""
Java modifiers:
private static
Type:
Logger
"""
LOG = get_logger(__name__)
class ConstraintItem:
"""
Parameters:
Map<String, String> attributes
"""
def __init__(self, attributes):
# self._attributes = TreeMap<ConstraintAttribute, String>()
self._attributes = {}
self._constraintValue = None
if attributes != None:
for key in attributes.keys:
try:
# ConstraintAttribute
attr = getattr(ConstraintAttribute, key)
if attr == ConstraintAttribute.CONSTRAINT_VALUE:
# String
value = attributes.get(key)
try:
ConstraintValue.valueOf(value)
except Exception, e:
try:
int(value)
except ValueError, ne:
self.LOG.error("Invalid constraintValue " +
str(key) + ":" + str(value))
continue
self._constraintValue = attributes.get(key)
else:
self._attributes.__setitem__(
attr, attributes.get(key))
except Exception, e:
self.LOG.error("Invalid constraintAttribute " +
str(key) + ":" +
str(attributes.get(key)))
continue
class HelixAgentMain(object):
"""
Java modifiers:
private static
Type:
Logger
"""
logger = get_logger(__name__)
"""
Java modifiers:
final static
Type:
String
"""
zkAddr = "zkSvr"
"""
Java modifiers:
final static
Type:
String
"""
cluster = "cluster"
"""
Java modifiers:
final static
Type:
String
"""
help = "help"
"""
Java modifiers:
final static
Type:
String
"""
instanceName = "instanceName"
"""
Java modifiers:
final static
Type:
String
"""
stateModel = "stateModel"
class ExternalCommand(object):
logger = get_logger(__name__)
@staticmethod
def executeWithTimeout(workingDir, cmdValue, args):
return ExternalCommand.executeWithTimeout(workingDir, cmdValue, 0,
args)
@staticmethod
def executeWithTimeout(workingDir, cmdValue, timeoutValue, args):
cmd = "%s %s" % (cmdValue, " ".join(args))
cmd_list = [cmdValue]
if args:
cmd_list.extend(cmd_list)
cwd = os.getcwd()
os.chdir(workingDir)
# return os.system(cmds)
#print "cmds = %s" % cmds
subProc = subprocess.Popen(cmd_list,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds=True,
shell=True)
slept = 0
sleepInterval = 1
while (timeoutValue == 0 or slept < timeoutValue):
if subProc.poll() is not None:
break
time.sleep(sleepInterval)
slept += sleepInterval
if slept >= timeoutValue and timeoutValue > 0:
subProc.kill()
time.sleep(1)
raise TimeoutException("cmd = %s" % cmd)
elif subProc.returncode != 0:
print "OUTPUT=\n" + "".join(subProc.stdout.readlines())
return subProc.returncode
class HelixStateTransitionHandler(MessageHandler):
"""
Java modifiers:
private static
Type:
Logger
"""
logger = get_logger(__name__)
"""
Parameters:
StateModel stateModel
Message message
NotificationContext context
CurrentState currentStateDelta
HelixTaskExecutor executor
"""
def __init__(self, stateModel, message, context, currentStateDelta,
executor):
super(HelixStateTransitionHandler, self).__init__(message, context)
self._stateModel = stateModel
self._statusUpdateUtil = StatusUpdateUtil()
self._transitionMethodFinder = StateModelParser()
self._currentStateDelta = currentStateDelta
self._executor = executor
def prepareMessageExecution(self, manager, message):
"""
Returns void
Parameters:
manager: HelixManagermessage: Message
Java modifiers:
private
Throws:
HelixException
HelixStateMismatchException
"""
if not message.isValid():
# String
# errorMessage = "Invalid Message, ensure that message: " + message + " has all the required fields: " + str(Message.Attributes.values())
errorMessage = "Invalid Message, ensure that message: " + message + " has all the required fields: " + str(
Attributes.values())
self._statusUpdateUtil.logError(message,
HelixStateTransitionHandler,
errorMessage,
manager.getHelixDataAccessor())
self.logger.error(errorMessage)
raise HelixException(errorMessage)
# HelixDataAccessor
accessor = manager.getHelixDataAccessor()
# String
partitionName = message.getPartitionName()
# String
fromState = message.getFromState()
# String
state = self._currentStateDelta.getState(partitionName)
if fromState != None and (
fromState == "*") and not fromState.upper() == state.upper():
# String
errorMessage = "Current state of stateModel does not match the fromState in Message" + ", Current State:" + str(
state) + ", message expected:" + str(
fromState) + ", partition: " + str(
partitionName) + ", from: " + message.getMsgSrc(
) + ", to: " + message.getTgtName()
self._statusUpdateUtil.logError(message,
HelixStateTransitionHandler,
errorMessage, accessor)
self.logger.error(errorMessage)
raise HelixStateMismatchException(errorMessage)
def postExecutionMessage(self, manager, message, context, taskResult,
exception):
"""
Returns void
Parameters:
manager: HelixManagermessage: Messagecontext: NotificationContexttaskResult: HelixTaskResultexception: Exception
"""
# String
partitionKey = message.getPartitionName()
# String
resource = message.getResourceName()
# String
sessionId = message.getTgtSessionId()
# String
instanceName = manager.getInstanceName()
# HelixDataAccessor
accessor = manager.getHelixDataAccessor()
# Builder
keyBuilder = accessor.keyBuilder()
# int
bucketSize = message.getBucketSize()
# ZNRecordBucketizer
bucketizer = ZNRecordBucketizer(bucketSize)
# TODO
# synchronized (manager)
if not (message.getTgtSessionId() == manager.getSessionId()):
self.logger.warn(
"Session id has changed. Skip postExecutionMessage. Old session "
+ message.getExecutionSessionId() + " , new session : " +
manager.getSessionId())
return
if taskResult.isSucess():
# String
toState = message.getToState()
self._currentStateDelta.setState(partitionKey, toState)
if toState.upper() == "DROPPED":
# ZNRecordDelta
delta = ZNRecordDelta(self._currentStateDelta.getRecord(),
MergeOperation.SUBTRACT)
delta._record.getSimpleFields().clear()
# List<ZNRecordDelta>
deltaList = []
deltaList.append(delta)
self._currentStateDelta.setDeltaList(deltaList)
else:
self._stateModel.updateState(toState)
else:
if type(exception) == HelixStateMismatchException:
self.logger.warn(
"Force CurrentState on Zk to be stateModel's CurrentState. partitionKey: "
+ partitionKey + ", currentState: " +
self._stateModel.getCurrentState() + ", message: " +
message)
self._currentStateDelta.setState(
partitionKey, self._stateModel.getCurrentState())
else:
# StateTransitionError
error = StateTransitionError(ErrorType.INTERNAL,
ErrorCode.ERROR, exception)
# TODO:
# if type(exception) == InterruptedException:
# if _isTimeout:
# error = StateTransitionError(ErrorType.INTERNAL, ErrorCode.TIMEOUT, exception)
# else:
# self.logger.error("State transition interrupted but not timeout. Not updating state. Partition : " + str(message.getPartitionName())+ " MsgId : " + str(message.getMsgId()))
# return
self._stateModel.rollbackOnError(message, context, error)
self._currentStateDelta.setState(partitionKey, "ERROR")
self._stateModel.updateState("ERROR")
# synchronize manager
try:
# PropertyKey
key = keyBuilder.currentState(
instanceName, sessionId, resource,
bucketizer.getBucketName(partitionKey))
if not self._message.getGroupMessageMode():
accessor.updateProperty(key, self._currentStateDelta)
else:
self._executor._groupMsgHandler.addCurStateUpdate(
self._message, key, self._currentStateDelta)
except Exception, e:
self.logger.error("Error when updating the state " + str(e))
# StateTransitionError
error = StateTransitionError(ErrorType.FRAMEWORK, ErrorCode.ERROR,
e)
self._stateModel.rollbackOnError(message, context, error)
self._statusUpdateUtil.logError(message,
HelixStateTransitionHandler, e,
"Error when update the state ",
accessor)
class ZNRecord:
"""
Java modifiers:
static
Type:
Logger
"""
# _logger = Logger.getLogger(__name__)
_logger = get_logger(__name__)
# console = logging.StreamHandler()
# formatter = logging.Formatter('%(asctime)-15s [%(process)d] [%(levelname)s] %(message)s')
# console.setFormatter(formatter)
# console.setLevel(logging.INFO)
# _logger.addHandler(console)
_logger.setLevel(logging.INFO)
"""
# Annotation: @JsonIgnore(True)
Java modifiers:
final static
Type:
String
"""
LIST_FIELD_BOUND = "listField.bound"
"""
# Annotation: @JsonIgnore(True)
Java modifiers:
final static
Type:
int
"""
SIZE_LIMIT = 1000 * 1024
def __init__(self, *args):
if len(args) == 1 and (isinstance(args[0], str) or isinstance(args[0], unicode)):
self.__init_id__(args[0])
elif len(args) == 1 and isinstance(args[0], ZNRecord):
self.__init_record__(args[0])
elif len(args) == 2 and isinstance(args[0],ZNRecord) and (isinstance(args[1], str) or isinstance(args[1], unicode)):
self.__init_record_id__(self, *args)
elif len(args) == 2 and isinstance(args[0],ZNRecord) and isinstance(args[1], int):
self.__init_record_version__(self, *args)
else:
raise IllegalArgumentException("Input arguments not supported. args = %s" % list(args))
"""
@JsonCreator
Parameters:
String id
"""
def __init_id__(self, id):
self.id = id
# simpleFields = TreeMap<String, String>()
# mapFields = TreeMap<String, Map<String, String>>()
# listFields = TreeMap<String, List<String>>()
self.simpleFields = {}
self.mapFields = {}
self.listFields = {}
self._version = 0 # TODO: What should be the default?
self._creationTime = None
self._modifiedTime = None
self._deltaList = []
"""
Parameters:
ZNRecord record
"""
def __init_record__(self, record):
self.__init_record_id__(record, record.getId())
"""
Parameters:
ZNRecord record
String id
"""
def __init_record_id__(self, record, id):
self.__init_id__(id)
self.simpleFields= copy.deepcopy(record.getSimpleFields())
self.mapFields= copy.deepcopy(record.getMapFields())
self.listFields = copy.deepcopy(record.getListFields())
self._version = record.getVersion()
self._creationTime = record.getCreationTime()
self._modifiedTime = record.getModifiedTime()
"""
Parameters:
ZNRecord record
int version
"""
def __init_record_version__(self, record, version):
self.__init__(record)
self._version = version
def setDeltaList(self, deltaList):
"""
Returns void
Parameters:
deltaList: List<ZNRecordDelta>
# Annotation: @JsonIgnore(True)
"""
self._deltaList = deltaList
def getDeltaList(self):
"""
Returns List<ZNRecordDelta>
# Annotation: @JsonIgnore(True)
"""
return self._deltaList
def getSimpleFields(self):
"""
Returns Map<String, String>
@JsonProperty
"""
return self.simpleFields
def setSimpleFields(self, simpleFields):
"""
Returns void
Parameters:
simpleFields: Map<String, String>
@JsonProperty
"""
self.simpleFields = simpleFields
def getMapFields(self):
"""
Returns Map<String, Map<String, String>>
@JsonProperty
"""
return self.mapFields
def setMapFields(self, mapFields):
"""
Returns void
Parameters:
mapFields: Map<String, Map<String, String>>
@JsonProperty
"""
self.mapFields = mapFields
def getListFields(self):
"""
Returns Map<String, List<String>>
@JsonProperty
"""
return self.listFields
def setListFields(self, listFields):
"""
Returns void
Parameters:
listFields: Map<String, List<String>>
@JsonProperty
"""
self.listFields = listFields
def setSimpleField(self, k, v):
"""
Returns void
Parameters:
k: Stringv: String
@JsonProperty
"""
self.simpleFields.__setitem__(k, v)
def getId(self):
"""
Returns String
@JsonProperty
"""
return self.id
def setMapField(self, k, v):
"""
Returns void
Parameters:
k: Stringv: Map<String, String>
"""
self.mapFields.__setitem__(k, v)
def setListField(self, k, v):
"""
Returns void
Parameters:
k: Stringv: List<String>
"""
self.listFields.__setitem__(k, v)
def getSimpleField(self, k):
"""
Returns String
Parameters:
k: String
"""
# return str(self.simpleFields.get(k)) # get rid of the unicode, this will cause the None to return as 'None'
return self.simpleFields.get(k) # get rid of the unicode
def getMapField(self, k):
"""
Returns Map<String, String>
Parameters:
k: String
"""
return self.mapFields.get(k)
def getListField(self, k):
"""
Returns List<String>
Parameters:
k: String
"""
return self.listFields.get(k)
def __str__(self):
"""
Returns String
@Override
"""
# StringBuffer
return self.id + ", " + str(self.simpleFields) + str(self.mapFields) + str(self.listFields)
# sb = []
# sb.append(id + ", ")
# if self.simpleFields != None:
# sb.append(simpleFields)
#
# if mapFields != None:
# sb.append(mapFields)
#
# if listFields != None:
# sb.append(listFields)
#
# return sb.toString()
def merge_record(self, record):
"""
Returns void
Parameters:
record: ZNRecord
"""
if record == None:
return
if len(record.getDeltaList()) > 0:
self._logger.info("Merging with delta list, recordId = " + self.id + " other:" + record.getId())
self.merge(record.getDeltaList())
return
#simpleFields.putAll(record.simpleFields)
self.simpleFields.update(record.simpleFields)
for key in record.mapFields.keys():
map = record.mapFields.get(key)
if map != None:
map.update(record.mapFields[key])
# map.putAll(record.mapFields.get(key))
else:
self.mapFields.__setitem__(key, record.mapFields.get(key))
for key in record.listFields.keys(): # List<String>
list = self.listFields.get(key)
if list != None:
list.extend(record.listFields.get(key))
else:
self.listFields.__setitem__(key, record.listFields.get(key))
def merge_delta(self, delta):
"""
Returns void
Parameters:
delta: ZNRecordDelta
"""
if delta.getMergeOperation() == MergeOperation.ADD:
self.merge(delta.getRecord())
else:
if delta.getMergeOperation() == MergeOperation.SUBTRACT:
self.subtract(delta.getRecord())
def merge_delta_list(self, deltaList):
"""
Returns void
Parameters:
deltaList: List<ZNRecordDelta>
"""
for delta in deltaList:
self.merge(delta)
def equals(self, obj):
"""
Returns boolean
Parameters:
obj: Object
@Override
"""
if not (type(obj) == ZNRecord):
return False
# ZNRecord
that = obj
# that = (ZNRecord) obj
if self.getSimpleFields().size() != that.getSimpleFields().size():
return False
if self.getMapFields().size() != that.getMapFields().size():
return False
if self.getListFields().size() != that.getListFields().size():
return False
if not (self.getSimpleFields() == that.getSimpleFields()):
return False
if not (self.getMapFields() == that.getMapFields()):
return False
if not (self.getListFields() == that.getListFields()):
return False
return True
def subtract(self, value):
"""
Returns void
Parameters:
value: ZNRecord
"""
for key in value.getSimpleFields().keys():
if key in self.simpleFields:
self.simpleFields.pop(key)
for key in value.getListFields().keys():
if key in self.listFields.containsKey:
self.listFields.pop(key)
for key in value.getMapFields().keys():
if key in self.mapFields:
self.mapFields.pop(key)
def getVersion(self):
"""
Returns int
# Annotation: @JsonIgnore(True)
"""
return self._version
def setVersion(self, version):
"""
Returns void
Parameters:
version: int
# Annotation: @JsonIgnore(True)
"""
self._version = version
def getCreationTime(self):
"""
Returns long
# Annotation: @JsonIgnore(True)
"""
return self._creationTime
def setCreationTime(self, creationTime):
"""
Returns void
Parameters:
creationTime: long
# Annotation: @JsonIgnore(True)
"""
_creationTime = creationTime
def getModifiedTime(self):
"""
Returns long
# Annotation: @JsonIgnore(True)
"""
return self._modifiedTime
def setModifiedTime(self, modifiedTime):
"""
Returns void
Parameters:
modifiedTime: long
# Annotation: @JsonIgnore(True)
"""
_modifiedTime = modifiedTime
def merge(self, *args):
if len(args) == 1 and isinstance(args[0], ZNRecord):
self.merge_record(*args)
elif len(args) == 1 and isinstance(args[0], ZNRecordDelta):
self.merge_delta(*args)
elif len(args) == 1 and isinstance(args[0], list):
self.merge_delta_list(*args)
else:
raise IllegalArgumentException("Input arguments not supported. args = %s" % args)
class ConfigAccessor:
"""
Java modifiers:
private static
Type:
Logger
"""
LOG = get_logger(__name__)
"""
Java modifiers:
private final static
Type:
StringTemplate
"""
template = StringTemplate()
template.addEntry(ConfigScopeProperty.CLUSTER, 1,
"/{clusterName}/CONFIGS/CLUSTER")
template.addEntry(
ConfigScopeProperty.CLUSTER, 2,
"/{clusterName}/CONFIGS/CLUSTER/{clusterName}|SIMPLEKEYS")
template.addEntry(ConfigScopeProperty.PARTICIPANT, 1,
"/{clusterName}/CONFIGS/PARTICIPANT")
template.addEntry(
ConfigScopeProperty.PARTICIPANT, 2,
"/{clusterName}/CONFIGS/PARTICIPANT/{participantName}|SIMPLEKEYS")
template.addEntry(ConfigScopeProperty.RESOURCE, 1,
"/{clusterName}/CONFIGS/RESOURCE")
template.addEntry(
ConfigScopeProperty.RESOURCE, 2,
"/{clusterName}/CONFIGS/RESOURCE/{resourceName}|SIMPLEKEYS")
template.addEntry(
ConfigScopeProperty.PARTITION, 2,
"/{clusterName}/CONFIGS/RESOURCE/{resourceName}|MAPKEYS")
template.addEntry(
ConfigScopeProperty.PARTITION, 3,
"/{clusterName}/CONFIGS/RESOURCE/{resourceName}|MAPMAPKEYS|{partitionName}"
)
"""
Parameters:
ZkClient zkClient
"""
def __init__(self, zkClient):
self.zkClient = zkClient
def get(self, *args):
if len(args) == 2 and (isinstance(args[1], str)
or isinstance(args[1], unicode)):
return self.get_single(*args)
elif len(args) == 2 and isinstance(args[1], list):
return self.get_list(*args)
else:
raise IllegalArgumentException(
"Input arguments not supported. args = %s" % list(args))
def get_single(self, scope, key):
"""
Returns String
Parameters:
scope: ConfigScopekey: String
"""
ret = self.get_list(scope, [key])
# not empty dict
if ret:
return ret[key]
def get_list(self, scope, keyList):
if scope is None or scope.getScope() is None:
self.LOG.error("Scope can't be null")
return None
# String
value = None
# String
clusterName = scope.getClusterName()
if not ZKUtil.isClusterSetup(clusterName, self.zkClient):
raise HelixException("cluster " + clusterName +
" is not setup yet")
# String
scopeStr = scope.getScopeStr()
retDict = {}
# String[]
# splits = scopeStr.split("\|")
splits = re.split("[\\|]", scopeStr)
# ZNRecord
record = self.zkClient.readData(splits[0], True)
if record is not None:
for key in keyList:
if len(splits) == 1:
value = record.getSimpleField(key)
else:
if splits.length == 2:
if record.getMapField(splits[1]) is not None:
value = record.getMapField(splits[1]).get(key)
retDict[key] = value
return retDict
# return value
def set(self, *args):
if len(args) == 2 and (isinstance(args[1], str)
or isinstance(args[1], unicode)):
self.set_single(*args)
elif len(args) == 2 and isinstance(args[1], dict):
self.set_list(*args)
else:
raise IllegalArgumentException(
"Input arguments not supported. args = %s" % list(args))
def set_single(self, scope, key, value):
'''
Returns void
Parameters:
scope: ConfigScopekey: Stringvalue: String
'''
self.set_list(scope, {key: value})
def set_list(self, scope, key_value_dict):
"""
Returns void
Parameters:
scope: ConfigScopekey: Stringvalue: String
"""
if scope is None or scope.getScope() is None:
self.LOG.error("Scope can't be null")
return
# String
clusterName = scope.getClusterName()
if not ZKUtil.isClusterSetup(clusterName, self.zkClient):
raise HelixException("cluster " + clusterName +
" is not setup yet")
# String
scopeStr = scope.getScopeStr()
# String[]
splits = re.split("[\\|]", scopeStr)
# String
#znodeId = splits[0].substring(splits[0].lastIndexOf('/') + 1)
znodeId = splits[0].split('/')[-1]
# ZNRecord
update = ZNRecord(znodeId)
for key, value in key_value_dict.iteritems():
if len(splits) == 1:
update.setSimpleField(key, value)
else:
if len(splits) == 2:
if update.getMapField(splits[1]) == None:
update.setMapField(splits[1], {})
update.getMapField(splits[1])[key] = value
ZKUtil.createOrUpdate(self.zkClient, splits[0], update, True, True)
return
#from org.apache.helix.manager.zk import *
#from java.util.concurrent.atomic import AtomicBoolean
#from org.apache.log4j import Logger
#from org.apache.zookeeper.AsyncCallback import DataCallback
#from org.apache.zookeeper.AsyncCallback import StatCallback
#from org.apache.zookeeper.AsyncCallback import StringCallback
#from org.apache.zookeeper.AsyncCallback import VoidCallback
#from org.apache.zookeeper.KeeperException import Code
#from org.apache.zookeeper.data import Stat
import threading
from kazoo.exceptions import NoNodeError, NodeExistsError, BadVersionError
from org.apache.helix.util.logger import get_logger
from kazoo.interfaces import IAsyncResult
LOG = get_logger(__name__)
#class ZkAsyncCallbacks:
#
# """
# Java modifiers:
# private static
# Type:
# Logger
# """
class DefaultCallback(object):
def __init__(self):
self._asyncResult = None
class ConfigScopeBuilder:
"""
Java modifiers:
private static
Type:
Logger
"""
LOG = get_logger(__name__)
"""
Java modifiers:
private static
Type:
StringTemplate
"""
template = StringTemplate()
template.addEntry(ConfigScopeProperty.CLUSTER, 1, "CLUSTER={clusterName}")
template.addEntry(ConfigScopeProperty.RESOURCE, 2, "CLUSTER={clusterName},RESOURCE={resourceName}")
template.addEntry(ConfigScopeProperty.PARTITION, 3, "CLUSTER={clusterName},RESOURCE={resourceName},PARTITION={partitionName}")
template.addEntry(ConfigScopeProperty.PARTICIPANT, 2, "CLUSTER={clusterName},PARTICIPANT={participantName}")
def getScopeMap(self):
"""
Returns Map<ConfigScopeProperty, String>
"""
return self._scopeMap
"""
"""
def __init__(self):
self._scopeMap = {}
# self._scopeMap = HashMap<ConfigScopeProperty, String>()
def forCluster(self, clusterName):
"""
Returns ConfigScopeBuilder
Parameters:
clusterName: String
"""
self._scopeMap.__setitem__(ConfigScopeProperty.CLUSTER, clusterName)
return self
def forParticipant(self, participantName):
"""
Returns ConfigScopeBuilder
Parameters:
participantName: String
"""
self._scopeMap.__setitem__(ConfigScopeProperty.PARTICIPANT, participantName)
return self
def forResource(self, resourceName):
"""
Returns ConfigScopeBuilder
Parameters:
resourceName: String
"""
self._scopeMap.__setitem__(ConfigScopeProperty.RESOURCE, resourceName)
return self
def forPartition(self, partitionName):
"""
Returns ConfigScopeBuilder
Parameters:
partitionName: String
"""
self._scopeMap.__setitem__(ConfigScopeProperty.PARTITION, partitionName)
return self
def build(self, *args):
"""
Returns ConfigScope
"""
if len(args)==0:
return ConfigScope(self)
elif len(args)==3:
return self.build_3(*args)
elif len(args)==1:
return self.build_1(*args)
else:
raise IllegalArgumentException("Wrong args: %s" % args)
def build_3(self, scope, clusterName, scopeKeys):
"""
Returns ConfigScope
Parameters:
scope: ConfigScopePropertyclusterName: StringscopeKeys: String
"""
if scopeKeys == None:
scopeKeys = {}
# scopeKeys = new String[] {}
# String[]
# args = new String[1 + scopeKeys.length]
# args[0] = clusterName
# System.arraycopy(scopeKeys, 0, args, 1, scopeKeys.length)
args=[clusterName]
args.extend(scopeKeys[1:])
# String
scopePairs = ConfigScopeBuilder.template.instantiate(scope, args)
return self.build(scopePairs)
def build_1(self, scopePairs):
"""
Returns ConfigScope
Parameters:
scopePairs: String
"""
# String[]
# scopes = scopePairs.split("[\\s,]+")
# TODO: re split
scopes = scopePairs.split("[\s,]+")
for scope in scopes:
try:
# int
idx = scope.indexOf('=')
if idx == -1:
self.LOG.error("Invalid scope string: " + str(scope))
continue
# String
scopeStr = scope.substring(0, idx)
# String
value = scope.substring(idx + 1)
# ConfigScopeProperty
scopeProperty = ConfigScopeProperty.valueOf(scopeStr)
self._scopeMap.put(scopeProperty, value)
except Exception, e:
self.LOG.error("Invalid scope string: " + str(scope))
continue
return self.build()
class StateModel(object):
"""
Java modifiers:
final static
Type:
String
"""
DEFAULT_INITIAL_STATE = "OFFLINE"
logger = get_logger(__name__)
def __init__(self):
self._currentState = StateModel.DEFAULT_INITIAL_STATE;
def getCurrentState(self):
"""
Returns String
"""
return self._currentState
def defaultTransitionHandler(self):
"""
Returns void
"""
self.logger.error("Default default handler. The idea is to invoke this if no transition method is found. Yet to be implemented")
def updateState(self, newState):
"""
Returns boolean
Parameters:
newState: String
"""
_currentState = newState
return True
def rollbackOnError(self, message, context, error):
"""
Returns void
Parameters:
message: Messagecontext: NotificationContexterror: StateTransitionError
"""
self.logger.error("Default rollback method invoked on error. Error Code:" + str(error.getCode()))
def reset(self):
"""
Returns void
"""
self.logger.warn("Default reset method invoked. Either because the process longer own this resource or session timedout")
class PersistentStats(HelixProperty):
"""
Java modifiers:
private final static
Type:
Logger
"""
_logger = get_logger(__name__)
"""
Java modifiers:
final static
Type:
String
"""
nodeName = "PersistentStats"
"""
Parameters:
String id
"""
def __init__(self, id):
super(id)
"""
Parameters:
ZNRecord record
"""
def __init__(self, record):
super(record)
def setSessionId(self, sessionId):
"""
Returns void
Parameters:
sessionId: String
"""
self._record.setSimpleField(
PersistentStatsProperty.SESSION_ID.toString(), sessionId)
def getSessionId(self):
"""
Returns String
"""
return self._record.getSimpleField(
PersistentStatsProperty.SESSION_ID.toString())
def getInstanceName(self):
"""
Returns String
"""
return self._record.getId()
def getMapFields(self):
"""
Returns Map<String, Map<String, String>>
"""
return self._record.getMapFields()
def getStatFields(self, statName):
"""
Returns Map<String, String>
Parameters:
statName: String
"""
return self._record.getMapField(statName)
def isValid(self):
"""
Returns boolean
@Override
"""
return True
class LiveInstance(HelixProperty):
"""
Java modifiers:
private final static
Type:
Logger
"""
_logger = get_logger(__name__)
"""
Parameters:
String id
"""
def __init__(self, *args):
super(LiveInstance, self).__init__(*args)
# def __init__(self, id):
# super(id)
#
# """
#
# Parameters:
# ZNRecord record
# """
# def __init__(self, record):
# super(record)
def setSessionId(self, sessionId):
"""
Returns void
Parameters:
sessionId: String
"""
self._record.setSimpleField(
LiveInstanceProperty.toString(LiveInstanceProperty.SESSION_ID),
sessionId)
def getSessionId(self):
"""
Returns String
"""
return self._record.getSimpleField(
LiveInstanceProperty.toString(LiveInstanceProperty.SESSION_ID))
def getInstanceName(self):
"""
Returns String
"""
return self._record.getId()
def getHelixVersion(self):
"""
Returns String
"""
return self._record.getSimpleField(
LiveInstanceProperty.toString(LiveInstanceProperty.HELIX_VERSION))
def setHelixVersion(self, helixVersion):
"""
Returns void
Parameters:
helixVersion: String
"""
self._record.setSimpleField(
LiveInstanceProperty.toString(LiveInstanceProperty.HELIX_VERSION),
helixVersion)
def getLiveInstance(self):
"""
Returns String
"""
return self._record.getSimpleField(
LiveInstanceProperty.toString(LiveInstanceProperty.LIVE_INSTANCE))
def setLiveInstance(self, leader):
"""
Returns void
Parameters:
leader: String
"""
self._record.setSimpleField(
LiveInstanceProperty.toString(LiveInstanceProperty.LIVE_INSTANCE),
leader)
def getModifiedTime(self):
"""
Returns long
"""
return self._record.getModifiedTime()
def getWebserviceUrl(self):
"""
Returns String
"""
return self._record.getSimpleField(
LiveInstanceProperty.toString(
LiveInstanceProperty.ZKPROPERTYTRANSFERURL))
def setWebserviceUrl(self, url):
"""
Returns void
Parameters:
url: String
"""
self._record.setSimpleField(
LiveInstanceProperty.toString(
LiveInstanceProperty.ZKPROPERTYTRANSFERURL), url)
def isValid(self):
"""
Returns boolean
@Override
"""
if self.getSessionId() == None:
self._logger.error("liveInstance does not have session id. id:" +
str(self._record.getId()))
return False
if self.getHelixVersion() == None:
self._logger.error("liveInstance does not have CLM verion. id:" +
str(self._record.getId()))
return False
return True
class ConfigScope:
"""
Java modifiers:
private static
Type:
Logger
"""
# LOG = Logger.getLogger(ConfigScope.class)
LOG = get_logger(__name__)
"""
Java modifiers:
private final static
Type:
List<ConfigScopeProperty>
"""
# scopePriority = ArrayList<ConfigScopeProperty>()
scopePriority = []
"""
Java modifiers:
private final static
Type:
Map<ConfigScopeProperty, Map<ConfigScopeProperty, ConfigScopeProperty>>
"""
scopeTransition = {}
# scopeTransition = HashMap<ConfigScopeProperty, Map<ConfigScopeProperty, ConfigScopeProperty>>()
"""
Java modifiers:
private final static
Type:
StringTemplate
"""
template = StringTemplate()
scopePriority.append(ConfigScopeProperty.CLUSTER)
scopePriority.append(ConfigScopeProperty.PARTICIPANT)
scopePriority.append(ConfigScopeProperty.RESOURCE)
scopePriority.append(ConfigScopeProperty.PARTITION)
# scopeTransition.__setitem__(ConfigScopeProperty.CLUSTER, HashMap<ConfigScopeProperty, ConfigScopeProperty>())
scopeTransition.__setitem__(ConfigScopeProperty.CLUSTER, {})
scopeTransition.get(ConfigScopeProperty.CLUSTER).__setitem__(
ConfigScopeProperty.PARTICIPANT, ConfigScopeProperty.PARTICIPANT)
scopeTransition.get(ConfigScopeProperty.CLUSTER).__setitem__(
ConfigScopeProperty.RESOURCE, ConfigScopeProperty.RESOURCE)
scopeTransition.__setitem__(ConfigScopeProperty.RESOURCE, {})
# scopeTransition.put(ConfigScopeProperty.RESOURCE, HashMap<ConfigScopeProperty, ConfigScopeProperty>())
scopeTransition.get(ConfigScopeProperty.RESOURCE).__setitem__(
ConfigScopeProperty.PARTITION, ConfigScopeProperty.PARTITION)
template.addEntry(ConfigScopeProperty.CLUSTER, 2,
"/{clusterName}/CONFIGS/CLUSTER/{clusterName}")
template.addEntry(ConfigScopeProperty.PARTICIPANT, 2,
"/{clusterName}/CONFIGS/PARTICIPANT/{participantName}")
template.addEntry(ConfigScopeProperty.RESOURCE, 2,
"/{clusterName}/CONFIGS/RESOURCE/{resourceName}")
template.addEntry(
ConfigScopeProperty.PARTITION, 3,
"/{clusterName}/CONFIGS/RESOURCE/{resourceName}|{partitionName}")
"""
Parameters:
ConfigScopeBuilder configScopeBuilder
"""
def __init__(self, configScopeBuilder):
# Map<ConfigScopeProperty, String>
scopeMap = configScopeBuilder.getScopeMap()
# List<String>
keys = []
# keys = ArrayList<String>()
# ConfigScopeProperty
curScope = None
for scope in self.scopePriority:
if scopeMap.__contains__(scope):
if curScope == None and scope == ConfigScopeProperty.CLUSTER:
keys.append(scopeMap.get(scope))
curScope = ConfigScopeProperty.CLUSTER
else:
if curScope == None:
raise IllegalArgumentException(
"Missing CLUSTER scope. Can't build scope using " +
configScopeBuilder)
else:
if not self.scopeTransition.__contains__(
curScope) or not self.scopeTransition.get(
curScope).__contains__(scope):
raise IllegalArgumentException(
"Can't build scope using " +
configScopeBuilder)
keys.append(scopeMap.get(scope))
curScope = self.scopeTransition.get(curScope).get(
scope)
if curScope == ConfigScopeProperty.CLUSTER:
keys.append(scopeMap.get(ConfigScopeProperty.CLUSTER))
# String
scopeStr = self.template.instantiate(curScope, keys)
# scopeStr = self.template.instantiate(curScope, keys.toArray(new String[0]))
self._clusterName = keys[0]
self._scopeStr = scopeStr
self._scope = curScope
def getScope(self):
"""
Returns ConfigScopeProperty
"""
return self._scope
def getClusterName(self):
"""
Returns String
"""
return self._clusterName
def getScopeStr(self):
"""
Returns String
"""
return self._scopeStr
# def toString(self):
# """
# Returns String
# @Override
#
#
# """
# return super(null, self).toString() + ": " + _scopeStr
def __repr__(self):
"""
Returns String
@Override
"""
return ": " + self._scopeStr
#from org.apache.helix.manager.zk import *
#from java.util.concurrent.atomic import AtomicBoolean
#from org.apache.log4j import Logger
#from org.apache.zookeeper.AsyncCallback import DataCallback
#from org.apache.zookeeper.AsyncCallback import StatCallback
#from org.apache.zookeeper.AsyncCallback import StringCallback
#from org.apache.zookeeper.AsyncCallback import VoidCallback
#from org.apache.zookeeper.KeeperException import Code
#from org.apache.zookeeper.data import Stat
import threading
from kazoo.exceptions import NoNodeError, NodeExistsError, BadVersionError
from org.apache.helix.util.logger import get_logger
from kazoo.interfaces import IAsyncResult
LOG = get_logger(__name__)
#class ZkAsyncCallbacks:
#
# """
# Java modifiers:
# private static
# Type:
# Logger
# """
class DefaultCallback(object):
def __init__(self):
self._asyncResult = None
self._cv = threading.Condition()
class AsyncCallbackService(MessageHandlerFactory):
"""
Java modifiers:
private static
Type:
Logger
"""
_logger = get_logger(__name__)
"""
"""
def __init__(self): pass
def registerAsyncCallback(self, correlationId, callback):
"""
Returns void
Parameters:
correlationId: Stringcallback: AsyncCallback
"""
if self._callbackMap.containsKey(correlationId):
self._logger.warn("correlation id " + correlationId + " already registered")
self._logger.info("registering correlation id " + correlationId)
self._callbackMap.put(correlationId, callback)
def verifyMessage(self, message):
"""
Returns void
Parameters:
message: Message
"""
if not message.getMsgType().toString().upper() == MessageType.TASK_REPLY.toString().upper():
# String
errorMsg = "Unexpected msg type for message " + message.getMsgId() + " type:" + message.getMsgType() + " Expected : " + MessageType.TASK_REPLY
self._logger.error(errorMsg)
raise HelixException(errorMsg)
# String
correlationId = message.getCorrelationId()
if correlationId == None:
# String
errorMsg = "Message " + message.getMsgId() + " does not have correlation id"
self._logger.error(errorMsg)
raise HelixException(errorMsg)
if not self._callbackMap.containsKey(correlationId):
# String
errorMsg = "Message " + message.getMsgId() + " does not have correponding callback. Probably timed out already. Correlation id: " + correlationId
self._logger.error(errorMsg)
raise HelixException(errorMsg)
self._logger.info("Verified reply message " + message.getMsgId() + " correlation:" + correlationId)
def createHandler(self, message, context):
"""
Returns MessageHandler
Parameters:
message: Messagecontext: NotificationContext
@Override
"""
self.verifyMessage(message)
return AsyncCallbackMessageHandler(message.getCorrelationId(), message, context)
def getMessageType(self):
"""
Returns String
@Override
"""
return MessageType.toString(MessageType.TASK_REPLY)
def reset(self):
"""
class HelixTask():
#class HelixTask(Callable<HelixTaskResult>):
"""
Java modifiers:
private static
Type:
Logger
"""
logger = get_logger(__name__)
# class TimeoutCancelTask(TimerTask):
#
# """
#
# Parameters:
# HelixTaskExecutor executor
# Message message
# NotificationContext context
# """
# def __init__(self, executor, message, context):
# self._executor = executor
# self._message = message
# self._context = context
#
#
# def run(self):
# """
# Returns void
# @Override
#
#
# """
# self._isTimeout = True
# self.logger.warn("Message time out, canceling. id:" + self._message.getMsgId() + " timeout : " + self._message.getExecutionTimeout())
# self._handler.onTimeout()
# self._executor.cancelTask(self._message, self._context)
#
"""
Parameters:
Message message
NotificationContext notificationContext
MessageHandler handler
HelixTaskExecutor executor
Throws:
Exception
"""
def __init__(self, message, notificationContext, handler, executor):
self._notificationContext = notificationContext
self._message = message
self._handler = handler
self._manager = notificationContext.getManager()
self._statusUpdateUtil = StatusUpdateUtil()
self._executor = executor
self._isTimeout = False
def call(self):
"""
Returns HelixTaskResult
@Override
"""
# Timer
# TODO: enable timer
timer = None
# if self._message.getExecutionTimeout() > 0:
# timer = Timer(True)
# timer.schedule(TimeoutCancelTask(self._executor, self._message, self._notificationContext), self._message.getExecutionTimeout())
# self.logger.info("Message starts with timeout " + self._message.getExecutionTimeout() + " MsgId:" + self._message.getMsgId())
# else:
# self.logger.info("Message does not have timeout. MsgId:" + self._message.getMsgId() + "/" + self._message.getPartitionName())
# HelixTaskResult
taskResult = HelixTaskResult()
# Exception
exception = None
# ErrorType
type = ErrorType.INTERNAL
# ErrorCode
code = ErrorCode.ERROR
# long
start = time.time()
self.logger.info("msg:" + str(self._message.getMsgId()) + " handling task begin, at: " + str(start))
# HelixDataAccessor
accessor = self._manager.getHelixDataAccessor()
self._statusUpdateUtil.logInfo(self._message, HelixTask, "Message handling task begin execute", accessor)
self._message.setExecuteStartTimeStamp(time.time())
try:
taskResult = self._handler.handleMessage()
exception = taskResult.getException()
# except Exception, e:
except KeyboardInterrupt, e:
self._statusUpdateUtil.logError(self._message, HelixTask, e, "State transition interrupted, timeout:" + str(self._isTimeout), accessor)
self.logger.info("Message " + self._message.getMsgId() + " is interrupted")
taskResult.setInterrupted(True)
taskResult.setException(e)
exception = e
# except Exception, e:
# # String
# errorMessage = "Exception while executing a message. " + e + " msgId: " + self._message.getMsgId() + " type: " + self._message.getMsgType()
# self.logger.error(errorMessage+ str(e))
# self._statusUpdateUtil.logError(self._message, HelixTask, e, errorMessage, accessor)
# taskResult.setSuccess(False)
# taskResult.setException(e)
# taskResult.setMessage(e.getMessage())
# exception = e
if timer != None:
timer.cancel()
if taskResult.isSucess():
self._statusUpdateUtil.logInfo(self._message, self._handler.__class__, "Message handling task completed successfully", accessor)
self.logger.info("Message " + self._message.getMsgId() + " completed.")
else:
if taskResult.isInterrupted():
self.logger.info("Message " + self._message.getMsgId() + " is interrupted")
code = ternary(self._isTimeout, ErrorCode.TIMEOUT, ErrorCode.CANCEL)
if self._isTimeout:
# int
retryCount = self._message.getRetryCount()
self.logger.info("Message timeout, retry count: " + retryCount + " MSGID:" + self._message.getMsgId())
self._statusUpdateUtil.logInfo(self._message, self._handler.__class__, "Message handling task timeout, retryCount:" + retryCount, accessor)
if retryCount > 0:
self._message.setRetryCount(retryCount - 1)
self._executor.scheduleTask(self._message, self._handler, self._notificationContext)
return taskResult
else:
# String
errorMsg = "Message execution failed. msgId: " + self._message.getMsgId() + taskResult.getMessage()
if exception is not None:
errorMsg += exception
self.logger.error(errorMsg+ str(exception))
self._statusUpdateUtil.logError(self._message, self._handler.getClass(), errorMsg, accessor)
try:
if not self._message.getGroupMessageMode():
self.removeMessageFromZk(accessor, self._message)
self.reportMessageStat(self._manager, self._message, taskResult)
self.sendReply(accessor, self._message, taskResult)
else:
# GroupMessageInfo
info = self._executor._groupMsgHandler.onCompleteSubMessage(self._message)
if info is not None:
# Map<PropertyKey, CurrentState>
curStateMap = info.merge()
for key in curStateMap.keys():
accessor.updateProperty(key, curStateMap.get(key))
self.removeMessageFromZk(accessor, self._message)
self.reportMessageStat(self._manager, self._message, taskResult)
self.sendReply(accessor, self._message, taskResult)
self._executor.reportCompletion(self._message)
# except KeyboardInterrupt, e:
#TODO: should we print the error
except Exception as e:
# String
errorMessage = "Exception after executing a message, msgId: " + self._message.getMsgId() + e
self.logger.error(errorMessage+ str(e))
self._statusUpdateUtil.logError(self._message, HelixTask, errorMessage, accessor)
exception = e
type = ErrorType.FRAMEWORK
code = ErrorCode.ERROR
finally:
# long
end = time.time()
# end = System.currentTimeMillis()
self.logger.info("msg:" + self._message.getMsgId() + " handling task completed, results:" + str(taskResult.isSucess()) + ", at: " + str(end) + ", took:" + str(end - start))
if exception is not None:
self._handler.onError(exception, code, type)
return taskResult
class HelixCustomCodeRunner:
"""
Java modifiers:
private final static
Type:
String
"""
LEADER_STANDBY = "LeaderStandby"
"""
Java modifiers:
private static
Type:
Logger
"""
LOG = get_logger(__name__)
"""
Java modifiers:
private static
Type:
String
"""
PARTICIPANT_LEADER = "PARTICIPANT_LEADER"
"""
Parameters:
HelixManager manager
String zkAddr
"""
def __init__(self, manager, zkAddr):
self._manager = manager
self._zkAddr = zkAddr
def invoke(self, callback):
"""
Returns HelixCustomCodeRunner
Parameters:
callback: CustomCodeCallbackHandler
"""
self._callback = callback
return self
def on(self, notificationTypes):
"""
Returns HelixCustomCodeRunner
Parameters:
notificationTypes: ChangeType
"""
self._notificationTypes = notificationTypes
return self
def usingLeaderStandbyModel(self, id):
"""
Returns HelixCustomCodeRunner
Parameters:
id: String
"""
self._resourceName = HelixCustomCodeRunner.PARTICIPANT_LEADER + "_" + id
return self
def start(self):
"""
Returns void
Throws:
Exception
"""
if self._callback == None or self._notificationTypes == None or self._notificationTypes.size(
) == 0 or self._resourceName == None:
raise IllegalArgumentException(
"Require callback | notificationTypes | resourceName")
self.LOG.info("Register participantLeader on " +
self._notificationTypes + " using " + self._resourceName)
_stateModelFty = GenericLeaderStandbyStateModelFactory(
self._callback, self._notificationTypes)
# StateMachineEngine
stateMach = self._manager.getStateMachineEngine()
stateMach.registerStateModelFactory(
HelixCustomCodeRunner.LEADER_STANDBY, _stateModelFty,
self._resourceName)
# ZkClient
zkClient = None
try:
zkClient = ZkClient(self._zkAddr,
ZkClient.DEFAULT_CONNECTION_TIMEOUT)
zkClient.setZkSerializer(ZNRecordSerializer())
# HelixDataAccessor
accessor = ZKHelixDataAccessor(self._manager.getClusterName(),
ZkBaseDataAccessor(zkClient))
# Builder
keyBuilder = accessor.keyBuilder()
# IdealState
idealState = IdealState(self._resourceName)
idealState.setIdealStateMode(
IdealStateModeProperty.AUTO.toString())
idealState.setReplicas(StateModelToken.ANY_LIVEINSTANCE.toString())
idealState.setNumPartitions(1)
idealState.setStateModelDefRef(
HelixCustomCodeRunner.LEADER_STANDBY)
idealState.setStateModelFactoryName(self._resourceName)
# List<String>
prefList = [
StateModelToken.toString(StateModelToken.ANY_LIVEINSTANCE)
]
idealState.getRecord().setListField(self._resourceName + "_0",
prefList)
# List<String>
idealStates = accessor.getChildNames(keyBuilder.idealStates())
while (idealStates == None
or not idealStates.__contains__(self._resourceName)):
accessor.setProperty(
keyBuilder.idealStates(self._resourceName), idealState)
idealStates = accessor.getChildNames(keyBuilder.idealStates())
self.LOG.info("Set idealState for participantLeader:" +
self._resourceName + ", idealState:" + idealState)
finally:
if zkClient != None and zkClient.getConnection() != None:
zkClient.close()
def stop(self):
"""
Returns void
"""
self.LOG.info("Removing stateModelFactory for " + self._resourceName)
self._manager.getStateMachineEngine().removeStateModelFactory(
HelixCustomCodeRunner.LEADER_STANDBY, self._stateModelFty,
self._resourceName)
class PropertyKey:
"""
Java modifiers:
private static
Type:
Logger
"""
# LOG = Logger.getLogger(PropertyKey.class)
LOG = get_logger(__name__)
LOG.setLevel(logging.INFO)
"""
Parameters:
PropertyType type
Class<? extends HelixProperty> typeClazz
String... params
"""
def __init__(self, type, typeClazz, *params):
self._type = type
# if params == None or params.length == 0 or Arrays.asList(params).contains(None):
if not params:
raise IllegalArgumentException("params cannot be null")
self._params = params
self._typeClazz = typeClazz
# def hashCode(self):
# """
# Returns int
# @Override
#
#
# """
# return super(null, self).hashCode()
def getPath(self):
"""
Returns String
"""
# String
clusterName = self._params[0]
# String[]
# subKeys = Arrays.copyOfRange(self._params, 1, self._params.length)
subKeys = self._params[1:]
# String
path = PropertyPathConfig.getPath(self._type, clusterName, *subKeys)
if path == None:
# self.LOG.error("Invalid property key with type:" + str(self._type)+ "subKeys:" + str(Arrays.toString(self._params)))
self.LOG.error("Invalid property key with type:" +
str(self._type) + "subKeys:" +
str(str(self._params)))
return path
# Java modifiers:
# static
def getType(self):
"""
Returns PropertyType
"""
return self._type
def getParams(self):
"""
Returns String[]
"""
return self._params
def getTypeClass(self):
"""
Returns Class<? extends HelixProperty>
"""
return self._typeClazz
def main(args):
"""
Returns void
Parameters:
args: String[]
Java modifiers:
static
"""
pass
# # ZkClient
# zkClient = ZkClient("localhost:2181")
# zkClient.waitUntilConnected(10, TimeUnit.SECONDS)
# # BaseDataAccessor
# baseDataAccessor = ZkBaseDataAccessor(zkClient)
# # HelixDataAccessor
# accessor = ZKHelixDataAccessor("test-cluster", baseDataAccessor)
# # Builder
# builder = PropertyKey.Builder("test-cluster")
# # HelixProperty
# value = IdealState("test-resource")
# accessor.createProperty(builder.idealStates("test-resource"), value)
class Builder:
"""
Parameters:
String clusterName
"""
def __init__(self, clusterName):
self._clusterName = clusterName
def idealStates(self):
"""
Returns PropertyKey
"""
return PropertyKey(PropertyType.IDEALSTATES, IdealState,
self._clusterName)
def idealStates(self, resourceName):
"""
Returns PropertyKey
Parameters:
resourceName: String
"""
return PropertyKey(PropertyType.IDEALSTATES, IdealState,
self._clusterName, resourceName)
def stateModelDefs(self):
"""
Returns PropertyKey
"""
return PropertyKey(PropertyType.STATEMODELDEFS,
StateModelDefinition, self._clusterName)
def stateModelDef(self, stateModelName):
"""
Returns PropertyKey
Parameters:
stateModelName: String
"""
return PropertyKey(PropertyType.STATEMODELDEFS,
StateModelDefinition, self._clusterName,
stateModelName)
def clusterConfig(self):
"""
Returns PropertyKey
"""
return PropertyKey(
PropertyType.CONFIGS, None, self._clusterName,
ConfigScopeProperty.toString(ConfigScopeProperty.CLUSTER))
def instanceConfigs(self):
"""
Returns PropertyKey
"""
return PropertyKey(
PropertyType.CONFIGS, InstanceConfig, self._clusterName,
ConfigScopeProperty.toString(ConfigScopeProperty.PARTICIPANT))
def instanceConfig(self, instanceName):
"""
Returns PropertyKey
Parameters:
instanceName: String
"""
return PropertyKey(
PropertyType.CONFIGS, InstanceConfig, self._clusterName,
ConfigScopeProperty.toString(ConfigScopeProperty.PARTICIPANT),
instanceName)
def resourceConfig(self, resourceName):
"""
Returns PropertyKey
Parameters:
resourceName: String
"""
return PropertyKey(
PropertyType.CONFIGS, None, self._clusterName,
ConfigScopeProperty.toString(ConfigScopeProperty.RESOURCE),
resourceName)
def resourceConfig(self, instanceName, resourceName):
"""
Returns PropertyKey
Parameters:
instanceName: StringresourceName: String
"""
return PropertyKey(
PropertyType.CONFIGS, None, self._clusterName,
ConfigScopeProperty.toString(ConfigScopeProperty.RESOURCE),
resourceName)
def partitionConfig(self, resourceName, partitionName):
"""
Returns PropertyKey
Parameters:
resourceName: StringpartitionName: String
"""
return PropertyKey(
PropertyType.CONFIGS, None, self._clusterName,
ConfigScopeProperty.toString(ConfigScopeProperty.RESOURCE),
resourceName)
def partitionConfig(self, instanceName, resourceName, partitionName):
"""
Returns PropertyKey
Parameters:
instanceName: StringresourceName: StringpartitionName: String
"""
return PropertyKey(
PropertyType.CONFIGS, None, self._clusterName,
ConfigScopeProperty.toString(ConfigScopeProperty.RESOURCE),
resourceName)
def constraints(self):
"""
Returns PropertyKey
"""
return PropertyKey(
PropertyType.CONFIGS, ClusterConstraints, self._clusterName,
ConfigScopeProperty.toString(ConfigScopeProperty.CONSTRAINT))
def constraint(self, constraintType):
"""
Returns PropertyKey
Parameters:
constraintType: String
"""
return PropertyKey(
PropertyType.CONFIGS, ClusterConstraints, self._clusterName,
ConfigScopeProperty.toString(ConfigScopeProperty.CONSTRAINT),
constraintType)
def liveInstances(self):
"""
Returns PropertyKey
"""
return PropertyKey(PropertyType.LIVEINSTANCES, LiveInstance,
self._clusterName)
def liveInstance(self, instanceName):
"""
Returns PropertyKey
Parameters:
instanceName: String
"""
return PropertyKey(PropertyType.LIVEINSTANCES, LiveInstance,
self._clusterName, instanceName)
def instances(self):
"""
Returns PropertyKey
"""
return PropertyKey(PropertyType.CONFIGS, None, self._clusterName)
def messages(self, instanceName):
"""
Returns PropertyKey
Parameters:
instanceName: String
"""
return PropertyKey(PropertyType.MESSAGES, Message,
self._clusterName, instanceName)
def message(self, instanceName, messageId):
"""
Returns PropertyKey
Parameters:
instanceName: StringmessageId: String
"""
return PropertyKey(PropertyType.MESSAGES, Message,
self._clusterName, instanceName, messageId)
def sessions(self, instanceName):
"""
Returns PropertyKey
Parameters:
instanceName: String
"""
return PropertyKey(PropertyType.CURRENTSTATES, CurrentState,
self._clusterName, instanceName)
def currentStates(self, instanceName, sessionId):
"""
Returns PropertyKey
Parameters:
instanceName: StringsessionId: String
"""
return PropertyKey(PropertyType.CURRENTSTATES, CurrentState,
self._clusterName, instanceName, sessionId)
# def currentState(self, instanceName, sessionId, resourceName):
# """
# Returns PropertyKey
# Parameters:
# instanceName: StringsessionId: StringresourceName: String
#
#
# """
# return PropertyKey(PropertyType.CURRENTSTATES, CurrentState, self._clusterName, instanceName, sessionId, resourceName)
def currentState(self,
instanceName,
sessionId,
resourceName,
bucketName=None):
"""
Returns PropertyKey
Parameters:
instanceName: StringsessionId: StringresourceName: StringbucketName: String
"""
if bucketName == None:
return PropertyKey(PropertyType.CURRENTSTATES, CurrentState,
self._clusterName, instanceName, sessionId,
resourceName)
else:
return PropertyKey(PropertyType.CURRENTSTATES, CurrentState,
self._clusterName, instanceName, sessionId,
resourceName, bucketName)
def stateTransitionStatus(self, *args):
if len(args) == 4: return self.stateTransitionStatus_4(*args)
elif len(args) == 3: return self.stateTransitionStatus_3(*args)
elif len(args) == 2: return self.stateTransitionStatus_2(*args)
else:
raise IllegalArgumentException(
"Input arguments not supported. args = %s" % args)
def stateTransitionStatus_4(self, instanceName, sessionId,
resourceName, partitionName):
"""
Returns PropertyKey
Parameters:
instanceName: StringsessionId: StringresourceName: StringpartitionName: String
"""
return PropertyKey(PropertyType.STATUSUPDATES, StatusUpdate,
self._clusterName, instanceName, sessionId,
resourceName, partitionName)
def stateTransitionStatus_3(self, instanceName, sessionId,
resourceName):
"""
Returns PropertyKey
Parameters:
instanceName: StringsessionId: StringresourceName: String
"""
return PropertyKey(PropertyType.STATUSUPDATES, StatusUpdate,
self._clusterName, instanceName, sessionId,
resourceName)
def stateTransitionStatus_2(self, instanceName, sessionId):
"""
Returns PropertyKey
Parameters:
instanceName: StringsessionId: String
"""
return PropertyKey(PropertyType.STATUSUPDATES, StatusUpdate,
self._clusterName, instanceName, sessionId)
def taskStatus(self, instanceName, sessionId, msgType, msgId):
"""
Returns PropertyKey
Parameters:
instanceName: StringsessionId: StringmsgType: StringmsgId: String
"""
return PropertyKey(PropertyType.STATUSUPDATES, StatusUpdate,
self._clusterName, instanceName, sessionId,
msgType, msgId)
def stateTransitionError(self, instanceName, sessionId, resourceName,
partitionName):
"""
Returns PropertyKey
Parameters:
instanceName: StringsessionId: StringresourceName: StringpartitionName: String
"""
return PropertyKey(PropertyType.ERRORS, Error, self._clusterName,
instanceName, sessionId, resourceName,
partitionName)
def stateTransitionErrors(self, instanceName, sessionId, resourceName):
"""
Returns PropertyKey
Parameters:
instanceName: StringsessionId: StringresourceName: String
"""
return PropertyKey(PropertyType.ERRORS, Error, self._clusterName,
instanceName, sessionId, resourceName)
def taskError(self, instanceName, sessionId, msgType, msgId):
"""
Returns PropertyKey
Parameters:
instanceName: StringsessionId: StringmsgType: StringmsgId: String
"""
return PropertyKey(PropertyType.ERRORS, None, self._clusterName,
instanceName, sessionId, msgType, msgId)
def externalViews(self):
"""
Returns PropertyKey
"""
return PropertyKey(PropertyType.EXTERNALVIEW, ExternalView,
self._clusterName)
def externalView(self, resourceName):
"""
Returns PropertyKey
Parameters:
resourceName: String
"""
return PropertyKey(PropertyType.EXTERNALVIEW, ExternalView,
self._clusterName, resourceName)
def controller(self):
"""
Returns PropertyKey
"""
return PropertyKey(PropertyType.CONTROLLER, None,
self._clusterName)
def controllerTaskErrors(self):
"""
Returns PropertyKey
"""
return PropertyKey(PropertyType.ERRORS_CONTROLLER, StatusUpdate,
self._clusterName)
def controllerTaskError(self, messageType, messageId):
"""
Returns PropertyKey
Parameters:
messageType: StringmessageId: String
"""
return PropertyKey(PropertyType.ERRORS_CONTROLLER, Error,
self._clusterName, messageType, messageId)
def controllerTaskStatuses(self, messageType):
"""
Returns PropertyKey
Parameters:
messageType: String
"""
return PropertyKey(PropertyType.STATUSUPDATES_CONTROLLER,
StatusUpdate, self._clusterName, messageType)
def controllerTaskStatus(self, messageType, messageId):
"""
Returns PropertyKey
Parameters:
messageType: StringmessageId: String
"""
return PropertyKey(PropertyType.STATUSUPDATES_CONTROLLER,
StatusUpdate, self._clusterName, messageType,
messageId)
def controllerMessages(self):
"""
Returns PropertyKey
"""
return PropertyKey(PropertyType.MESSAGES_CONTROLLER, Message,
self._clusterName)
def controllerMessage(self, msgId):
"""
Returns PropertyKey
Parameters:
msgId: String
"""
return PropertyKey(PropertyType.MESSAGES_CONTROLLER, Message,
self._clusterName, msgId)
def controllerLeaderHistory(self):
"""
Returns PropertyKey
"""
return PropertyKey(PropertyType.HISTORY, LeaderHistory,
self._clusterName)
def controllerLeader(self):
"""
Returns PropertyKey
"""
return PropertyKey(PropertyType.LEADER, LiveInstance,
self._clusterName)
def pause(self):
"""
Returns PropertyKey
"""
return PropertyKey(PropertyType.PAUSE, PauseSignal,
self._clusterName)
def persistantStat(self):
"""
Returns PropertyKey
"""
return PropertyKey(PropertyType.PERSISTENTSTATS, PersistentStats,
self._clusterName)
def alerts(self):
"""
Returns PropertyKey
"""
return PropertyKey(PropertyType.ALERTS, Alerts, self._clusterName)
def alertStatus(self):
"""
Returns PropertyKey
"""
return PropertyKey(PropertyType.ALERT_STATUS, AlertStatus,
self._clusterName)
def alertHistory(self):
"""
Returns PropertyKey
"""
return PropertyKey(PropertyType.ALERT_HISTORY, AlertHistory,
self._clusterName)
def healthReport(self, instanceName, id):
"""
Returns PropertyKey
Parameters:
instanceName: Stringid: String
"""
raise BaseException("Not implemented")
# return PropertyKey(PropertyType.HEALTHREPORT, HealthStat, self._clusterName, instanceName, id)
def healthReports(self, instanceName):
"""
Returns PropertyKey
Parameters:
instanceName: String
"""
raise BaseException("Not implemented")
class ZkBaseDataAccessor(BaseDataAccessor):
"""
Java modifiers:
private static
Type:
Logger
"""
LOG = get_logger(__name__)
"""
Parameters:
ZkClient zkClient
"""
def __init__(self, zkClient):
self._zkClient = zkClient
def create(self, *args):
if len(args) == 3 and isinstance(args[1], ZNRecord):
return self.create_path(args[0], args[1], None,
args[2]) == RetCode.OK
elif len(args) == 4 and (isinstance(args[1], ZNRecord)
or args[1] == None):
return self.create_path(*args) == RetCode.OK
elif len(args) == 5 and isinstance(args[3], list):
return self.create_paths(*args)
else:
import pdb
pdb.set_trace()
raise IllegalArgumentException("Wrong args: %s" % str(args))
# def create(self, path, record, options):
# """
# Returns boolean
# Parameters:
# path: Stringrecord: Toptions: int
# @Override
#
#
# """
# return self.create(path, record, None, options) == RetCode.OK
def create_path(self, path, record, pathCreated, options):
"""
Returns RetCode
Parameters:
path: Stringrecord: TpathCreated: List<String>options: int
"""
# CreateMode
mode = AccessOption.getMode(options)
if mode == None:
self.LOG.error("Invalid create mode. options: " + str(options))
return RetCode.ERROR
# boolean
retry = True
while retry:
retry = False
try:
self._zkClient.create(path, record, mode)
if pathCreated != None:
pathCreated.add(path)
return RetCode.OK
except NoNodeException, e:
# String
parentPath = os.path.abspath(os.path.join(
path, os.path.pardir))
# parentPath = File(path).getParent()
try:
# RetCode
rc = self.create(parentPath, None, pathCreated,
AccessOption.PERSISTENT)
if rc == RetCode.OK or rc == RetCode.NODE_EXISTS:
retry = True
# TODO:
# except Exception, e1:
except KeyboardInterrupt, e1:
self.LOG.error("Exception while creating path: " +
str(parentPath) + str(e1))
return RetCode.ERROR
except NodeExistsException, e:
self.LOG.warn("Node already exists. path: " + path)
return RetCode.NODE_EXISTS
class CurrentState(HelixProperty):
"""
Java modifiers:
private static
Type:
Logger
"""
LOG = get_logger(__name__)
def __init__(self, *args):
super(CurrentState,self).__init__(*args)
# if len(args)==1 and (isinstance(args[0],str) or isinstance((args[0]))
# else:
# raise IllegalArgumentException("Input arguments not supported. args = %s" % args)
#
# """
#
# Parameters:
# String resourceName
# """
# def __init__(self, resourceName):
# super(resourceName)
#
#
# """
#
# Parameters:
# ZNRecord record
# """
# def __init__(self, record):
# super(record)
def getResourceName(self):
"""
Returns String
"""
return self._record.getId()
def getPartitionStateMap(self):
"""
Returns Map<String, String>
"""
# Map<String, String>
map = {}
# Map<String, Map<String, String>>
mapFields = self._record.getMapFields()
for partitionName in mapFields.keys(): # Map<String, String>
tempMap = mapFields.get(partitionName)
if tempMap != None:
map.__setitem__(partitionName, tempMap.get(CurrentStateProperty.toString(CurrentStateProperty.CURRENT_STATE)))
return map
def getSessionId(self):
"""
Returns String
"""
return self._record.getSimpleField(CurrentStateProperty.toString(CurrentStateProperty.SESSION_ID))
def setSessionId(self, sessionId):
"""
Returns void
Parameters:
sessionId: String
"""
self._record.setSimpleField(CurrentStateProperty.toString(CurrentStateProperty.SESSION_ID), sessionId)
def getState(self, partitionName):
"""
Returns String
Parameters:
partitionName: String
"""
# Map<String, Map<String, String>>
mapFields = self._record.getMapFields()
# Map<String, String>
mapField = mapFields.get(partitionName)
if mapField != None:
return mapField.get(CurrentStateProperty.toString(CurrentStateProperty.CURRENT_STATE))
return None
def setStateModelDefRef(self, stateModelName):
"""
Returns void
Parameters:
stateModelName: String
"""
self._record.setSimpleField(CurrentStateProperty.toString(CurrentStateProperty.STATE_MODEL_DEF), stateModelName)
def getStateModelDefRef(self):
"""
Returns String
"""
return self._record.getSimpleField(CurrentStateProperty.toString(CurrentStateProperty.STATE_MODEL_DEF))
def setState(self, partitionName, state):
"""
Returns void
Parameters:
partitionName: Stringstate: String
"""
# Map<String, Map<String, String>>
mapFields = self._record.getMapFields()
if mapFields.get(partitionName) == None:
mapFields.__setitem__(partitionName, {})
# mapFields.put(partitionName, TreeMap<String, String>())
mapFields.get(partitionName).__setitem__(CurrentStateProperty.toString(CurrentStateProperty.CURRENT_STATE), state)
def setStateModelFactoryName(self, factoryName):
"""
Returns void
Parameters:
factoryName: String
"""
self._record.setSimpleField(CurrentStateProperty.toString(CurrentStateProperty.STATE_MODEL_FACTORY_NAME), factoryName)
def getStateModelFactoryName(self):
"""
Returns String
"""
return self._record.getSimpleField(CurrentStateProperty.toString(CurrentStateProperty.STATE_MODEL_FACTORY_NAME))
def isValid(self):
"""
Returns boolean
@Override
"""
if self.getStateModelDefRef() == None:
self.LOG.error("Current state does not contain state model ref. id:" + str(self.getResourceName()))
return False
if self.getSessionId() == None:
self.LOG.error("CurrentState does not contain session id, id : " + str(self.getResourceName()))
return False
return True
class IdealState(HelixProperty):
"""
Java modifiers:
final static
Type:
String
"""
QUERY_LIST = "PREFERENCE_LIST_QUERYS"
"""
Java modifiers:
private final static
Type:
Logger
"""
logger = get_logger(__name__)
def __init__(self, *args):
super(IdealState, self).__init__(*args)
# """
#
# Parameters:
# String resourceName
# """
# def __init__(self, resourceName):
# super(resourceName)
#
#
# """
#
# Parameters:
# ZNRecord record
# """
# def __init__(self, record):
# super(record)
def getResourceName(self):
"""
Returns String
"""
return self._record.getId()
def setIdealStateMode(self, mode):
"""
Returns void
Parameters:
mode: String
"""
self._record.setSimpleField(
IdealStateProperty.IDEAL_STATE_MODE.toString(), mode)
def getIdealStateMode(self):
"""
Returns IdealStateModeProperty
"""
# String
mode = self._record.getSimpleField(
IdealStateProperty.IDEAL_STATE_MODE.toString())
try:
return IdealStateModeProperty.valueOf(mode)
except Exception, e:
return IdealStateModeProperty.AUTO
class ZKUtil:
"""
Java modifiers:
private static
Type:
Logger
"""
logger = get_logger(__name__)
"""
Java modifiers:
private static
Type:
int
"""
RETRYLIMIT = 3
"""
Java modifiers:
private
"""
def __init__(self):
pass
@staticmethod
def isClusterSetup(clusterName, zkClient):
"""
Returns boolean
Parameters:
clusterName: StringzkClient: ZkClient
Java modifiers:
static
"""
if clusterName == None or zkClient == None:
return False
# boolean
isValid = zkClient.exists(PropertyPathConfig.getPath(PropertyType.IDEALSTATES, clusterName)) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.toString(ConfigScopeProperty.CLUSTER), clusterName)) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.toString(ConfigScopeProperty.PARTICIPANT))) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.toString(ConfigScopeProperty.RESOURCE))) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, clusterName)) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.LIVEINSTANCES, clusterName)) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.INSTANCES, clusterName)) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.EXTERNALVIEW, clusterName)) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.CONTROLLER, clusterName)) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.STATEMODELDEFS, clusterName)) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.MESSAGES_CONTROLLER, clusterName)) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.ERRORS_CONTROLLER, clusterName)) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.STATUSUPDATES_CONTROLLER, clusterName)) \
and zkClient.exists(PropertyPathConfig.getPath(PropertyType.HISTORY, clusterName))
return isValid
@staticmethod
def createOrUpdate(client, path, record, persistent, mergeOnUpdate):
"""
Returns void
Parameters:
client: ZkClientpath: Stringrecord: ZNRecordpersistent: booleanmergeOnUpdate: boolean
Java modifiers:
static
"""
# int
retryCount = 0
while retryCount < ZKUtil.RETRYLIMIT:
try:
if not client.exists(path):
client.createPersistent(path, True)
stat = HelixZNodeStat()
currentRecord = client.readDataAndStat(path, stat, True)
if currentRecord is not None and mergeOnUpdate:
currentRecord.merge(record)
client.writeData(path, currentRecord, stat.getVersion())
break
except Exception as e:
retryCount = retryCount + 1
ZKUtil.logger.warn("Exception trying to update " + path +
" Exception:" + str(e) + ". Will retry.")
class DefaultMessagingService(ClusterMessagingService):
"""
Java modifiers:
private static
Type:
Logger
"""
_logger = get_logger(__name__)
"""
Parameters:
HelixManager manager
"""
def __init__(self, manager):
self._manager = manager
self._evaluator = CriteriaEvaluator()
self._taskExecutor = HelixTaskExecutor()
self._asyncCallbackService = AsyncCallbackService()
self._taskExecutor.registerMessageHandlerFactory(
MessageType.toString(MessageType.TASK_REPLY),
self._asyncCallbackService)
self._messageHandlerFactoriestobeAdded = {
} #ConcurrentHashMap<String, MessageHandlerFactory>
# def send(self, recipientCriteria, messageTemplate):
# """
# Returns int
# Parameters:
# recipientCriteria: CriteriamessageTemplate: Message
# @Override
#
#
# """
# return self.send(recipientCriteria, messageTemplate, None, -1)
#
#
# def send(self, recipientCriteria, message, callbackOnReply, timeOut):
# """
# Returns int
# Parameters:
# recipientCriteria: Criteriamessage: MessagecallbackOnReply: AsyncCallbacktimeOut: int
# @Override
#
#
# """
# return self.send(recipientCriteria, message, callbackOnReply, timeOut, 0)
#
#
def send(self,
recipientCriteria,
message,
callbackOnReply=None,
timeOut=-1,
retryCount=0):
"""
Returns int
Parameters:
recipientCriteria: Criteriamessage: MessagecallbackOnReply: AsyncCallbacktimeOut: intretryCount: int
@Override
"""
# Map<InstanceType, List<Message>>
generateMessage = self.generateMessage(recipientCriteria, message)
# int
totalMessageCount = 0
for messages in generateMessage.values():
totalMessageCount += messages.size()
self._logger.info("Send " + totalMessageCount +
" messages with criteria " + recipientCriteria)
if totalMessageCount == 0:
return 0
# String
correlationId = None
if callbackOnReply != None:
# int
totalTimeout = timeOut * (retryCount + 1)
if totalTimeout < 0:
totalTimeout = -1
callbackOnReply.setTimeout(totalTimeout)
# correlationId = UUID.randomUUID().toString()
correlationId = str(uuid.uuid4())
for messages in generateMessage.values():
callbackOnReply.setMessagesSent(messages)
self._asyncCallbackService.registerAsyncCallback(
correlationId, callbackOnReply)
for receiverType in generateMessage.keys(): # List<Message>
list = generateMessage.get(receiverType)
for tempMessage in list:
tempMessage.setRetryCount(retryCount)
tempMessage.setExecutionTimeout(timeOut)
tempMessage.setSrcInstanceType(self._manager.getInstanceType())
if correlationId != None:
tempMessage.setCorrelationId(correlationId)
# HelixDataAccessor
accessor = self._manager.getHelixDataAccessor()
# Builder
keyBuilder = accessor.keyBuilder()
if receiverType == InstanceType.CONTROLLER:
accessor.setProperty(
keyBuilder.controllerMessage(tempMessage.getId()),
tempMessage)
if receiverType == InstanceType.PARTICIPANT:
accessor.setProperty(
keyBuilder.message(tempMessage.getTgtName(),
tempMessage.getId()), tempMessage)
if callbackOnReply != None:
callbackOnReply.startTimer()
return totalMessageCount
def generateMessage(self, recipientCriteria, message):
"""
Returns Map<InstanceType, List<Message>>
Parameters:
recipientCriteria: Criteriamessage: Message
Java modifiers:
private
"""
# Map<InstanceType, List<Message>>
messagesToSendMap = {}
# messagesToSendMap = HashMap<InstanceType, List<Message>>()
# InstanceType
instanceType = recipientCriteria.getRecipientInstanceType()
if instanceType == InstanceType.CONTROLLER:
# List<Message>
messages = self.generateMessagesForController(message)
messagesToSendMap.put(InstanceType.CONTROLLER, messages)
else:
if instanceType == InstanceType.PARTICIPANT:
# List<Message>
messages = []
# messages = ArrayList<Message>()
# List<Map<String, String>>
matchedList = self._evaluator.evaluateCriteria(
recipientCriteria, self._manager)
if not matchedList.isEmpty():
# Map<String, String>
sessionIdMap = {}
# sessionIdMap = HashMap<String, String>()
if recipientCriteria.isSessionSpecific():
# HelixDataAccessor
accessor = self._manager.getHelixDataAccessor()
# Builder
keyBuilder = accessor.keyBuilder()
# List<LiveInstance>
liveInstances = accessor.getChildValues(
keyBuilder.liveInstances())
for liveInstance in liveInstances:
sessionIdMap.__setitem__(
liveInstance.getInstanceName(),
liveInstance.getSessionId())
for map in matchedList:
id = str(uuid.uuid4())
# Message
newMessage = Message(message.getRecord(), id)
# String
srcInstanceName = self._manager.getInstanceName()
# String
tgtInstanceName = map.get("instanceName")
if recipientCriteria.isSelfExcluded(
) and srcInstanceName.upper() == tgtInstanceName.upper(
):
continue
newMessage.setSrcName(srcInstanceName)
newMessage.setTgtName(tgtInstanceName)
newMessage.setResourceName(map.get("resourceName"))
newMessage.setPartitionName(map.get("partitionName"))
if recipientCriteria.isSessionSpecific():
newMessage.setTgtSessionId(
sessionIdMap.get(tgtInstanceName))
messages.append(newMessage)
messagesToSendMap.__setitem__(InstanceType.PARTICIPANT,
messages)
return messagesToSendMap
def generateMessagesForController(self, message):
"""
Returns List<Message>
Parameters:
message: Message
Java modifiers:
private
"""
# List<Message>
# messages = ArrayList<Message>()
messages = []
# String
id = str(uuid.uuid4())
# Message
newMessage = Message(message.getRecord(), id)
newMessage.setMsgId(id)
newMessage.setSrcName(self._manager.getInstanceName())
newMessage.setTgtName("Controller")
messages.add(newMessage)
return messages
def registerMessageHandlerFactory(self, type, factory):
"""
Returns void
Parameters:
type: Stringfactory: MessageHandlerFactory
@Override
Java modifiers:
synchronized
"""
if self._manager.isConnected():
self.registerMessageHandlerFactoryInternal(type, factory)
else:
self._messageHandlerFactoriestobeAdded.put(type, factory)
def onConnected(self):
"""
Returns void
Java modifiers:
synchronized
"""
for type in self._messageHandlerFactoriestobeAdded.keys():
self.registerMessageHandlerFactoryInternal(
type, self._messageHandlerFactoriestobeAdded.get(type))
self._messageHandlerFactoriestobeAdded.clear()
def registerMessageHandlerFactoryInternal(self, type, factory):
"""
Returns void
Parameters:
type: Stringfactory: MessageHandlerFactory
"""
self._logger.info("registering msg factory for type " + type)
# int
threadpoolSize = HelixTaskExecutor.DEFAULT_PARALLEL_TASKS
# String
threadpoolSizeStr = None
# String
key = type + "." + HelixTaskExecutor.MAX_THREADS
# ConfigAccessor
configAccessor = self._manager.getConfigAccessor()
if configAccessor != None:
# ConfigScope
scope = None
if self._manager.getInstanceType(
) == InstanceType.PARTICIPANT or self._manager.getInstanceType(
) == InstanceType.CONTROLLER_PARTICIPANT:
scope = ConfigScopeBuilder().forCluster(
self._manager.getClusterName()).forParticipant(
self._manager.getInstanceName()).build()
threadpoolSizeStr = configAccessor.get(scope, key)
if threadpoolSizeStr == None:
scope = ConfigScopeBuilder().forCluster(
self._manager.getClusterName()).build()
threadpoolSizeStr = configAccessor.get(scope, key)
if threadpoolSizeStr != None:
try:
threadpoolSize = int(threadpoolSizeStr)
if threadpoolSize <= 0:
threadpoolSize = 1
except Exception, e:
self._logger.error("" + str(e))
self._taskExecutor.registerMessageHandlerFactory(
type, factory, threadpoolSize)
self.sendNopMessage()
