def update(self, currentData):
"""
Returns ZNRecord
Parameters:
currentData: ZNRecord
@Override
"""
if currentData == None:
currentData = ZNRecord(ConstraintType.MESSAGE_CONSTRAINT.toString())
# Map<String, String>
map = currentData.getMapField(constraintId)
if map == None:
map = TreeMap<String, String>()
currentData.setMapField(constraintId, map)
else:
logger.warn("Overwrite existing constraint " + constraintId + ": " + map)
for # String
key = None
in constraints.keySet()) # ConstraintAttribute
attr = ConstraintAttribute.valueOf(key.toUpperCase())
map.put(attr.toString(), constraints.get(key))
def addCluster(self, clusterName, overwritePrevRecord):
"""
Returns void
Parameters:
clusterName: StringoverwritePrevRecord: boolean
@Override
"""
# String
root = "/" + clusterName
# String
path = None
if _zkClient.exists(root):
logger.warn("Root directory exists.Cleaning the root directory:" + root + " overwritePrevRecord: " + overwritePrevRecord)
if overwritePrevRecord:
_zkClient.deleteRecursive(root)
else:
raise HelixException("Cluster " + clusterName + " already exists")
_zkClient.createPersistent(root)
_zkClient.createPersistent(HelixUtil.getIdealStatePath(clusterName))
path = PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.CLUSTER.toString(), clusterName)
_zkClient.createPersistent(path, True)
_zkClient.writeData(path, ZNRecord(clusterName))
path = PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.PARTICIPANT.toString())
_zkClient.createPersistent(path)
path = PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.RESOURCE.toString())
_zkClient.createPersistent(path)
path = PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, clusterName)
_zkClient.createPersistent(path)
_zkClient.createPersistent(HelixUtil.getLiveInstancesPath(clusterName))
_zkClient.createPersistent(HelixUtil.getMemberInstancesPath(clusterName))
_zkClient.createPersistent(HelixUtil.getExternalViewPath(clusterName))
_zkClient.createPersistent(HelixUtil.getStateModelDefinitionPath(clusterName))
_zkClient.createPersistent(HelixUtil.getControllerPath(clusterName))
path = PropertyPathConfig.getPath(PropertyType.HISTORY, clusterName)
# ZNRecord
emptyHistory = ZNRecord(PropertyType.HISTORY.toString())
# List<String>
emptyList = ArrayList<String>()
emptyHistory.setListField(clusterName, emptyList)
_zkClient.createPersistent(path, emptyHistory)
path = PropertyPathConfig.getPath(PropertyType.MESSAGES_CONTROLLER, clusterName)
_zkClient.createPersistent(path)
path = PropertyPathConfig.getPath(PropertyType.STATUSUPDATES_CONTROLLER, clusterName)
_zkClient.createPersistent(path)
path = PropertyPathConfig.getPath(PropertyType.ERRORS_CONTROLLER, clusterName)
_zkClient.createPersistent(path)
def bucketize(self, record):
"""
Returns Map<String, ZNRecord>
Parameters:
record: ZNRecord
"""
# Map<String, ZNRecord>
map = {}
if self._bucketSize == 0:
map.put(record.getId(), record)
return map
for partitionName in record.getListFields().keys(): # String
bucketName = self.getBucketName(partitionName)
if bucketName != None:
if not map.__contains__(bucketName):
map.__setitem__(bucketName, ZNRecord(bucketName))
# ZNRecord
bucketizedRecord = map.get(bucketName)
bucketizedRecord.setListField(
partitionName, record.getListField(partitionName))
else:
self.LOG.error("Can't bucketize " + str(partitionName) +
" in list field")
for partitionName in record.getMapFields().keys(): # String
bucketName = self.getBucketName(partitionName)
if bucketName != None:
if not map.__contains__(bucketName):
map.__setitem__(bucketName, ZNRecord(bucketName))
# ZNRecord
bucketizedRecord = map.get(bucketName)
bucketizedRecord.setMapField(partitionName,
record.getMapField(partitionName))
else:
self.LOG.error("Can't bucketize " + str(partitionName) +
" in map field")
for bucketizedRecord in map.values():
bucketizedRecord.setSimpleFields(record.getSimpleFields())
return map
def handleMessage(self):
"""
Returns HelixTaskResult
@Override
Throws:
InterruptedException
"""
# String
type = _message.getMsgType()
# HelixTaskResult
result = HelixTaskResult()
if not (type == MessageType.SCHEDULER_MSG.toString()):
raise HelixException("Unexpected msg type for message " + _message.getMsgId() + " type:" + _message.getMsgType())
# int
timeOut = -1
if _message.getRecord().getSimpleFields().containsKey("TIMEOUT"):
try:
timeOut = Integer.parseInt(_message.getRecord().getSimpleFields().get("TIMEOUT"))
except Exception, e:
# ZNRecord
record = ZNRecord("templateMessage")
record.getSimpleFields().putAll(_message.getRecord().getMapField("MessageTemplate"))
# Message
messageTemplate = Message(record)
# StringReader
sr = StringReader(_message.getRecord().getSimpleField("Criteria"))
# ObjectMapper
mapper = ObjectMapper()
# Criteria
recipientCriteria = None
try:
recipientCriteria = mapper.readValue(sr, Criteria.class)
except Exception, e:
_logger.error(""+ str(e))
result.setException(e)
result.setSuccess(False)
return result
def update(self, statsRec):
"""
Returns ZNRecord
Parameters:
statsRec: ZNRecord
@Override
"""
if statsRec == None:
statsRec = ZNRecord(PersistentStats.nodeName)
# Map<String, Map<String, String>>
currStatMap = statsRec.getMapFields()
# Map<String, Map<String, String>>
newStatMap = StatsHolder.parseStat(statName)
for # String
newStat = None
in newStatMap.keySet()) if not currStatMap.containsKey(newStat):
currStatMap.put(newStat, newStatMap.get(newStat))
def assemble(self, records):
"""
Returns ZNRecord
Parameters:
records: List<ZNRecord>
"""
# ZNRecord
assembledRecord = None
if records != None and records.size() > 0:
for record in records:
if record == None:
continue
if assembledRecord == None:
assembledRecord = ZNRecord(record.getId())
assembledRecord.merge(record)
return assembledRecord
def assemble(self, records):
"""
Returns ZNRecord
Parameters:
records: List<ZNRecord>
"""
# ZNRecord
assembledRecord = None
if records != None and records.size() > 0:
for record in records:
if record == None:
continue
if assembledRecord == None:
assembledRecord = ZNRecord(record.getId())
assembledRecord.merge(record)
return assembledRecord
def update(self, alertsRec):
"""
Returns ZNRecord
Parameters:
alertsRec: ZNRecord
@Override
"""
if alertsRec == None:
alertsRec = ZNRecord(Alerts.nodeName)
# Map<String, Map<String, String>>
currAlertMap = alertsRec.getMapFields()
# StringBuilder
newStatName = StringBuilder()
# Map<String, String>
newAlertMap = HashMap<String, String>()
AlertsHolder.parseAlert(alertName, newStatName, newAlertMap)
addStat(clusterName, newStatName.toString())
currAlertMap.put(alertName, newAlertMap)
alertsRec.setMapFields(currAlertMap)
return alertsRec
def set_list(self, scope, key_value_dict):
"""
Returns void
Parameters:
scope: ConfigScopekey: Stringvalue: String
"""
if scope is None or scope.getScope() is None:
self.LOG.error("Scope can't be null")
return
# String
clusterName = scope.getClusterName()
if not ZKUtil.isClusterSetup(clusterName, self.zkClient):
raise HelixException("cluster " + clusterName +
" is not setup yet")
# String
scopeStr = scope.getScopeStr()
# String[]
splits = re.split("[\\|]", scopeStr)
# String
#znodeId = splits[0].substring(splits[0].lastIndexOf('/') + 1)
znodeId = splits[0].split('/')[-1]
# ZNRecord
update = ZNRecord(znodeId)
for key, value in key_value_dict.iteritems():
if len(splits) == 1:
update.setSimpleField(key, value)
else:
if len(splits) == 2:
if update.getMapField(splits[1]) == None:
update.setMapField(splits[1], {})
update.getMapField(splits[1])[key] = value
ZKUtil.createOrUpdate(self.zkClient, splits[0], update, True, True)
return
def set_list(self, scope, key_value_dict):
"""
Returns void
Parameters:
scope: ConfigScopekey: Stringvalue: String
"""
if scope is None or scope.getScope() is None:
self.LOG.error("Scope can't be null")
return
# String
clusterName = scope.getClusterName()
if not ZKUtil.isClusterSetup(clusterName, self.zkClient):
raise HelixException("cluster " + clusterName + " is not setup yet")
# String
scopeStr = scope.getScopeStr()
# String[]
splits = re.split("[\\|]", scopeStr)
# String
#znodeId = splits[0].substring(splits[0].lastIndexOf('/') + 1)
znodeId = splits[0].split('/')[-1]
# ZNRecord
update = ZNRecord(znodeId)
for key, value in key_value_dict.iteritems():
if len(splits) == 1:
update.setSimpleField(key, value)
else:
if len(splits) == 2:
if update.getMapField(splits[1]) == None:
update.setMapField(splits[1], {})
update.getMapField(splits[1])[key] = value
ZKUtil.createOrUpdate(self.zkClient, splits[0], update, True, True)
return
def createMessageLogRecord(self, message):
"""
Returns ZNRecord
Parameters:
message: Message
"""
# ZNRecord
result = ZNRecord(self.getStatusUpdateRecordName(message))
# String
mapFieldKey = "MESSAGE " + message.getMsgId()
# result.setMapField(mapFieldKey, TreeMap<String, String>())
result.setMapField(mapFieldKey, {})
for simpleFieldKey in message.getRecord().getSimpleFields().keys():
result.getMapField(mapFieldKey).__setitem__(simpleFieldKey, message.getRecord().getSimpleField(simpleFieldKey))
if message.getResultMap() != None:
result.setMapField("MessageResult", message.getResultMap())
return result
def deserialize(self, bytes):
"""
Returns Object
Parameters:
bytes: byte[]
@Override
Throws:
ZkMarshallingError
"""
if bytes == None or len(bytes) == 0:
self.LOG.error("ZNode is empty.")
return None
# ByteArrayInputStream
inputJson = json.loads(str(bytes))
# ZNRecord
record = ZNRecord(str(inputJson["id"]))
record.setSimpleFields(inputJson["simpleFields"])
record.setListFields(inputJson["listFields"])
record.setMapFields(inputJson["mapFields"])
# record = None
# try:
# # JsonFactory
# f = JsonFactory()
# # JsonParser
# jp = f.createJsonParser(bais)
# jp.nextToken()
# while (jp.nextToken() != JsonToken.END_OBJECT:
# # String
# fieldname = jp.getCurrentName()
# jp.nextToken()
# if ("id" == fieldname):
# record = ZNRecord(jp.getText())
# else:
# if ("simpleFields" == fieldname):
# while (jp.nextToken() != JsonToken.END_OBJECT:
# # String
# key = jp.getCurrentName()
# jp.nextToken()
# record.setSimpleField(key, jp.getText())
#
# else:
# if ("mapFields" == fieldname):
# while (jp.nextToken() != JsonToken.END_OBJECT:
# # String
# key = jp.getCurrentName()
# record.setMapField(key, TreeMap<String, String>())
# jp.nextToken()
# while (jp.nextToken() != JsonToken.END_OBJECT:
# # String
# mapKey = jp.getCurrentName()
# jp.nextToken()
# record.getMapField(key).put(mapKey, jp.getText())
#
#
# else:
# if ("listFields" == fieldname):
# while (jp.nextToken() != JsonToken.END_OBJECT:
# # String
# key = jp.getCurrentName()
# record.setListField(key, ArrayList<String>())
# jp.nextToken()
# while (jp.nextToken() != JsonToken.END_ARRAY:
# record.getListField(key).add(jp.getText())
#
#
# else:
# raise IllegalStateException("Unrecognized field '" + fieldname + "'!")
#
#
#
# jp.close()
# except Exception, e:
# LOG.error("Exception during deserialization of bytes: " + str(String(bytes))+ str(e))
return record
def __init__record_id(self, record, id):
super(Message,self).__init__(ZNRecord(record,id))
# super(ZNRecord(record, id))
self.setMsgId(id)
def __init__(self, *args):
self._record = ZNRecord(*args)
class HelixProperty(object):
def __init__(self, *args):
self._record = ZNRecord(*args)
# """
#
# Parameters:
# String id
# """
# def __init__(self, id):
# self._record = ZNRecord(id)
#
#
# """
#
# Parameters:
# ZNRecord record
# """
# def __init__(self, record):
# self._record = ZNRecord(record)
def getId(self):
"""
Returns String
Java modifiers:
final
"""
return self._record.getId()
def getRecord(self):
"""
Returns ZNRecord
Java modifiers:
final
"""
return self._record
def setDeltaList(self, deltaList):
"""
Returns void
Parameters:
deltaList: List<ZNRecordDelta>
Java modifiers:
final
"""
self._record.setDeltaList(deltaList)
def toString(self):
"""
Returns String
@Override
"""
return self._record.toString()
def getBucketSize(self):
"""
Returns int
"""
# String
bucketSizeStr = self._record.getSimpleField('BUCKET_SIZE')
# int
bucketSize = 0
if bucketSizeStr != None:
try:
bucketSize = int(bucketSizeStr)
except ValueError, e:
pass
return bucketSize
def __init_resource__(self, resource):
super(ExternalView, self).__init__(ZNRecord(resource))
def deserialize(self, bytes):
"""
Returns Object
Parameters:
bytes: byte[]
@Override
Throws:
ZkMarshallingError
"""
if bytes == None or len(bytes) == 0:
self.LOG.error("ZNode is empty.")
return None
# ByteArrayInputStream
inputJson = json.loads(str(bytes))
# ZNRecord
record = ZNRecord(str(inputJson["id"]))
record.setSimpleFields(inputJson["simpleFields"])
record.setListFields(inputJson["listFields"])
record.setMapFields(inputJson["mapFields"])
# record = None
# try:
# # JsonFactory
# f = JsonFactory()
# # JsonParser
# jp = f.createJsonParser(bais)
# jp.nextToken()
# while (jp.nextToken() != JsonToken.END_OBJECT:
# # String
# fieldname = jp.getCurrentName()
# jp.nextToken()
# if ("id" == fieldname):
# record = ZNRecord(jp.getText())
# else:
# if ("simpleFields" == fieldname):
# while (jp.nextToken() != JsonToken.END_OBJECT:
# # String
# key = jp.getCurrentName()
# jp.nextToken()
# record.setSimpleField(key, jp.getText())
#
# else:
# if ("mapFields" == fieldname):
# while (jp.nextToken() != JsonToken.END_OBJECT:
# # String
# key = jp.getCurrentName()
# record.setMapField(key, TreeMap<String, String>())
# jp.nextToken()
# while (jp.nextToken() != JsonToken.END_OBJECT:
# # String
# mapKey = jp.getCurrentName()
# jp.nextToken()
# record.getMapField(key).put(mapKey, jp.getText())
#
#
# else:
# if ("listFields" == fieldname):
# while (jp.nextToken() != JsonToken.END_OBJECT:
# # String
# key = jp.getCurrentName()
# record.setListField(key, ArrayList<String>())
# jp.nextToken()
# while (jp.nextToken() != JsonToken.END_ARRAY:
# record.getListField(key).add(jp.getText())
#
#
# else:
# raise IllegalStateException("Unrecognized field '" + fieldname + "'!")
#
#
#
# jp.close()
# except Exception, e:
# LOG.error("Exception during deserialization of bytes: " + str(String(bytes))+ str(e))
return record