class ZNRecordStreamingSerializer(): """ Java modifiers: private static Type: Logger """ LOG = get_logger(__name__) @staticmethod def getListFieldBound(record): """ Returns int Parameters: record: ZNRecord Java modifiers: private static """ # int max = sys.maxint if record.getSimpleFields().__contains__(ZNRecord.LIST_FIELD_BOUND): # String maxStr = record.getSimpleField(ZNRecord.LIST_FIELD_BOUND) try: max = int(maxStr) except ValueError, e: ZNRecordStreamingSerializer.LOG.error( "IllegalNumberFormat for list field bound: " + str(maxStr)) return max
class ZNRecordSerializer(ZkSerializer): """ Java modifiers: private static Type: Logger """ logger = get_logger(__name__) def getListFieldBound(self, record): """ Returns int Parameters: record: ZNRecord Java modifiers: private static """ # int # max = Integer.MAX_VALUE max = sys.maxint if record.getSimpleFields().containsKey(ZNRecord.LIST_FIELD_BOUND): # String maxStr = record.getSimpleField(ZNRecord.LIST_FIELD_BOUND) try: max = int(maxStr) except ValueError, e: self.logger.error("IllegalNumberFormat for list field bound: " + str(maxStr)) return max
class ZNRecordJsonSerializer(PropertySerializer): """ Java modifiers: private static Type: Logger """ LOG = get_logger(__name__) def serialize(self, data): """ Returns byte[] Parameters: data: ZNRecord @Override Throws: PropertyStoreException """ return self._serializer.serialize(data) def deserialize(self, bytes): """ Returns ZNRecord Parameters: bytes: byte[] @Override Throws: PropertyStoreException """ # return (ZNRecord) self._serializer.deserialize(bytes) return self._serializer.deserialize(bytes)
class CriteriaEvaluator: """ Java modifiers: private static Type: Logger """ logger = get_logger(__name__) def evaluateCriteria(self, recipientCriteria, manager): """ Returns List<Map<String, String>> Parameters: recipientCriteria: Criteriamanager: HelixManager """ # List<Map<String, String>> selected = [] # # TODO: Skip the query for now # selected = ArrayList<Map<String, String>>() # # String # queryFields = ternary(not (recipientCriteria.getInstanceName() == ""), " " + ZNRecordRow.MAP_SUBKEY, " ''") + "," + (ternary(not (recipientCriteria.getResource() == ""), " " + ZNRecordRow.ZNRECORD_ID, " ''")) + "," + (ternary(not (recipientCriteria.getPartition() == ""), " " + ZNRecordRow.MAP_KEY, " ''")) + "," + (ternary(not (recipientCriteria.getPartitionState() == ""), " " + ZNRecordRow.MAP_VALUE, " '' ")) # # String # matchCondition = ZNRecordRow.MAP_SUBKEY + " LIKE '" + (ternary(not (recipientCriteria.getInstanceName() == ""), (recipientCriteria.getInstanceName() + "'"), "%' ")) + " AND " + ZNRecordRow.ZNRECORD_ID + " LIKE '" + (ternary(not (recipientCriteria.getResource() == ""), (recipientCriteria.getResource() + "'"), "%' ")) + " AND " + ZNRecordRow.MAP_KEY + " LIKE '" + (ternary(not (recipientCriteria.getPartition() == ""), (recipientCriteria.getPartition() + "'"), "%' ")) + " AND " + ZNRecordRow.MAP_VALUE + " LIKE '" + (ternary(not (recipientCriteria.getPartitionState() == ""), (recipientCriteria.getPartitionState() + "'"), "%' ")) + " AND " + ZNRecordRow.MAP_SUBKEY + " IN ((SELECT [*]id FROM :LIVEINSTANCES))" # # String # queryTarget = recipientCriteria.getDataSource().toString() + ClusterJosqlQueryProcessor.FLATTABLE # # String # josql = "SELECT DISTINCT " + queryFields + " FROM " + queryTarget + " WHERE " + matchCondition # # ClusterJosqlQueryProcessor # # TODO: Skip the query for now ## p = ClusterJosqlQueryProcessor(manager) # # List<Object> # result = [] ## result = ArrayList<Object>() # try: # self.logger.info("JOSQL query: " + josql) # # TODO: Skip the query for now ## result = p.runJoSqlQuery(josql, None, None) # except Exception, e: # self.logger.error(""+ str(e)) # return selected # # for o in result: # Map<String, String> ## resultRow = HashMap<String, String>() # resultRow = {} # # List<Object> # row = o ## row = (List<Object>) o # resultRow.__setitem__("instanceName", (row[0])) # resultRow.__setitem__("resourceName", (row[1])) # resultRow.__setitem__("partitionName",(row[2])) # resultRow.__setitem__("partitionState", (row[3])) # selected.append(resultRow) # # self.logger.info("JOSQL query return " + selected.size() + " rows") return selected
class ZNRecordBucketizer: """ Java modifiers: private static Type: Logger """ LOG = get_logger(__name__) """ Parameters: int bucketSize """ def __init__(self, bucketSize): if bucketSize <= 0: self.LOG.debug("bucketSize <= 0 (was " + str(bucketSize) + "). Set to 0 to use non-bucketized HelixProperty.") bucketSize = 0 self._bucketSize = bucketSize def getBucketName(self, key): """ Returns String Parameters: key: String """ if self._bucketSize == 0: return None # int idx = key.lastIndexOf('_') if idx < 0: raise IllegalArgumentException( "Could NOT find partition# in " + key + ". partitionName should be in format of resourceName_partition#" ) try: # int partitionNb = int(key.substring(idx + 1)) # int bucketNb = partitionNb / self._bucketSize # int startPartition = bucketNb * self._bucketSize # int endPartition = bucketNb * self._bucketSize + (self._bucketSize - 1) return key.substring( 0, idx) + "_p" + startPartition + "-p" + endPartition except ValueError, e: raise IllegalArgumentException("Could NOT parse partition# (" + key.substring(idx + 1) + ") in " + key)
class SystemUtil(object): OS_NAME = platform.system() logger = get_logger(__name__) @staticmethod def getProcessState(processId): """ Returns ProcessStateCode Parameters: processId: String Java modifiers: static Throws: Exception """ if (SystemUtil.OS_NAME == "Mac OS X") or (SystemUtil.OS_NAME == "Linux"): # ExternalCommand # cmd = ExternalCommand.start("ps", processId) # cmd.waitFor() output = os.popen("ps %s" % processId).read() # String # lines = cmd.getStringOutput().split("\n") lines = output.split("\n") if len(lines) != 2: SystemUtil.logger.info("process: " + processId + " not exist") return None attributes = re.split("\s+", lines[0].strip()) values = re.split("\s+", lines[1].strip()) # Character processStateCodeChar = None for i in range(len(attributes)): attribute = attributes[i] if ("STAT" == attribute) or ("S" == attribute): processStateCodeChar = values[i][0] break return ProcessStateCode.valueOf(processStateCodeChar) else: raise ("Not supported OS: " + SystemUtil.OS_NAME) @staticmethod def getPidFromFile(file): try: with open(file) as pidFile: line = pidFile.readline() return line except IOError, e: SystemUtil.logger.error("fail to read pid from pidFile: " + file, e) finally:
class HealthStat(HelixProperty): """ Java modifiers: private final static Type: Logger """ _logger = get_logger(__name__) """ Parameters: String id """ def __init__(self, id): super(id) """ Parameters: ZNRecord record """ def __init__(self, record): super(record) if self.getCreateTimeStamp() == 0: self._record.setSimpleField(Attributes.CREATE_TIMESTAMP.toString(), "" + time.time()) def getLastModifiedTimeStamp(self): """ Returns long """ return self._record.getModifiedTime() def getCreateTimeStamp(self): """ Returns long """ if self._record.getSimpleField( Attributes.CREATE_TIMESTAMP.toString()) == None: return 0 try: return long( self._record.getSimpleField( Attributes.toString(Attributes.CREATE_TIMESTAMP))) except ValueError, e: return 0
class GroupCommit: """ Java modifiers: private static Type: Logger """ # LOG = Logger.getLogger(GroupCommit.class) LOG = get_logger(__name__) # Java modifiers: # private static # class Queue: class Entry: """ Parameters: String key ZNRecord record """ def __init__(self, key, record): self._key = key self._record = record """ """ def __init__(self): self._queues = [] for i in range(100): self._queues.append(Queue()) def getQueue(self, key): """ Returns Queue Parameters: key: String Java modifiers: private """ return self._queues[hash(key) % self._queues.__len__()] def commit(self, accessor, options, key, record): accessor.set(key, record, options)
class ProcessMonitorThread(threading.Thread): """ Java modifiers: private final static Type: Logger """ logger = get_logger(__name__) """ Java modifiers: private final static Type: int """ MONITOR_PERIOD_BASE = 1000 """ Parameters: String pid """ def __init__(self, pid): self._pid = pid def run(self): """ Returns void @Override """ try: # ProcessStateCode processState = SystemUtil.getProcessState(self._pid) while processState != None: if processState == ProcessStateCode.Z: self.logger.error("process: " + self._pid + " is in zombie state") break time.sleep((random.randint( 0, ProcessMonitorThread.MONITOR_PERIOD_BASE) + ProcessMonitorThread.MONITOR_PERIOD_BASE) / 1000.0) processState = SystemUtil.getProcessState(self._pid) except Exception, e: ProcessMonitorThread.logger.error( "fail to monitor process: " + self._pid, e)
class ClusterConstraints(HelixProperty): """ Java modifiers: private static Type: Logger """ LOG = get_logger(__name__) class ConstraintItem: """ Parameters: Map<String, String> attributes """ def __init__(self, attributes): # self._attributes = TreeMap<ConstraintAttribute, String>() self._attributes = {} self._constraintValue = None if attributes != None: for key in attributes.keys: try: # ConstraintAttribute attr = getattr(ConstraintAttribute, key) if attr == ConstraintAttribute.CONSTRAINT_VALUE: # String value = attributes.get(key) try: ConstraintValue.valueOf(value) except Exception, e: try: int(value) except ValueError, ne: self.LOG.error("Invalid constraintValue " + str(key) + ":" + str(value)) continue self._constraintValue = attributes.get(key) else: self._attributes.__setitem__( attr, attributes.get(key)) except Exception, e: self.LOG.error("Invalid constraintAttribute " + str(key) + ":" + str(attributes.get(key))) continue
class HelixAgentMain(object): """ Java modifiers: private static Type: Logger """ logger = get_logger(__name__) """ Java modifiers: final static Type: String """ zkAddr = "zkSvr" """ Java modifiers: final static Type: String """ cluster = "cluster" """ Java modifiers: final static Type: String """ help = "help" """ Java modifiers: final static Type: String """ instanceName = "instanceName" """ Java modifiers: final static Type: String """ stateModel = "stateModel"
class ExternalCommand(object): logger = get_logger(__name__) @staticmethod def executeWithTimeout(workingDir, cmdValue, args): return ExternalCommand.executeWithTimeout(workingDir, cmdValue, 0, args) @staticmethod def executeWithTimeout(workingDir, cmdValue, timeoutValue, args): cmd = "%s %s" % (cmdValue, " ".join(args)) cmd_list = [cmdValue] if args: cmd_list.extend(cmd_list) cwd = os.getcwd() os.chdir(workingDir) # return os.system(cmds) #print "cmds = %s" % cmds subProc = subprocess.Popen(cmd_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True, shell=True) slept = 0 sleepInterval = 1 while (timeoutValue == 0 or slept < timeoutValue): if subProc.poll() is not None: break time.sleep(sleepInterval) slept += sleepInterval if slept >= timeoutValue and timeoutValue > 0: subProc.kill() time.sleep(1) raise TimeoutException("cmd = %s" % cmd) elif subProc.returncode != 0: print "OUTPUT=\n" + "".join(subProc.stdout.readlines()) return subProc.returncode
class HelixStateTransitionHandler(MessageHandler): """ Java modifiers: private static Type: Logger """ logger = get_logger(__name__) """ Parameters: StateModel stateModel Message message NotificationContext context CurrentState currentStateDelta HelixTaskExecutor executor """ def __init__(self, stateModel, message, context, currentStateDelta, executor): super(HelixStateTransitionHandler, self).__init__(message, context) self._stateModel = stateModel self._statusUpdateUtil = StatusUpdateUtil() self._transitionMethodFinder = StateModelParser() self._currentStateDelta = currentStateDelta self._executor = executor def prepareMessageExecution(self, manager, message): """ Returns void Parameters: manager: HelixManagermessage: Message Java modifiers: private Throws: HelixException HelixStateMismatchException """ if not message.isValid(): # String # errorMessage = "Invalid Message, ensure that message: " + message + " has all the required fields: " + str(Message.Attributes.values()) errorMessage = "Invalid Message, ensure that message: " + message + " has all the required fields: " + str( Attributes.values()) self._statusUpdateUtil.logError(message, HelixStateTransitionHandler, errorMessage, manager.getHelixDataAccessor()) self.logger.error(errorMessage) raise HelixException(errorMessage) # HelixDataAccessor accessor = manager.getHelixDataAccessor() # String partitionName = message.getPartitionName() # String fromState = message.getFromState() # String state = self._currentStateDelta.getState(partitionName) if fromState != None and ( fromState == "*") and not fromState.upper() == state.upper(): # String errorMessage = "Current state of stateModel does not match the fromState in Message" + ", Current State:" + str( state) + ", message expected:" + str( fromState) + ", partition: " + str( partitionName) + ", from: " + message.getMsgSrc( ) + ", to: " + message.getTgtName() self._statusUpdateUtil.logError(message, HelixStateTransitionHandler, errorMessage, accessor) self.logger.error(errorMessage) raise HelixStateMismatchException(errorMessage) def postExecutionMessage(self, manager, message, context, taskResult, exception): """ Returns void Parameters: manager: HelixManagermessage: Messagecontext: NotificationContexttaskResult: HelixTaskResultexception: Exception """ # String partitionKey = message.getPartitionName() # String resource = message.getResourceName() # String sessionId = message.getTgtSessionId() # String instanceName = manager.getInstanceName() # HelixDataAccessor accessor = manager.getHelixDataAccessor() # Builder keyBuilder = accessor.keyBuilder() # int bucketSize = message.getBucketSize() # ZNRecordBucketizer bucketizer = ZNRecordBucketizer(bucketSize) # TODO # synchronized (manager) if not (message.getTgtSessionId() == manager.getSessionId()): self.logger.warn( "Session id has changed. Skip postExecutionMessage. Old session " + message.getExecutionSessionId() + " , new session : " + manager.getSessionId()) return if taskResult.isSucess(): # String toState = message.getToState() self._currentStateDelta.setState(partitionKey, toState) if toState.upper() == "DROPPED": # ZNRecordDelta delta = ZNRecordDelta(self._currentStateDelta.getRecord(), MergeOperation.SUBTRACT) delta._record.getSimpleFields().clear() # List<ZNRecordDelta> deltaList = [] deltaList.append(delta) self._currentStateDelta.setDeltaList(deltaList) else: self._stateModel.updateState(toState) else: if type(exception) == HelixStateMismatchException: self.logger.warn( "Force CurrentState on Zk to be stateModel's CurrentState. partitionKey: " + partitionKey + ", currentState: " + self._stateModel.getCurrentState() + ", message: " + message) self._currentStateDelta.setState( partitionKey, self._stateModel.getCurrentState()) else: # StateTransitionError error = StateTransitionError(ErrorType.INTERNAL, ErrorCode.ERROR, exception) # TODO: # if type(exception) == InterruptedException: # if _isTimeout: # error = StateTransitionError(ErrorType.INTERNAL, ErrorCode.TIMEOUT, exception) # else: # self.logger.error("State transition interrupted but not timeout. Not updating state. Partition : " + str(message.getPartitionName())+ " MsgId : " + str(message.getMsgId())) # return self._stateModel.rollbackOnError(message, context, error) self._currentStateDelta.setState(partitionKey, "ERROR") self._stateModel.updateState("ERROR") # synchronize manager try: # PropertyKey key = keyBuilder.currentState( instanceName, sessionId, resource, bucketizer.getBucketName(partitionKey)) if not self._message.getGroupMessageMode(): accessor.updateProperty(key, self._currentStateDelta) else: self._executor._groupMsgHandler.addCurStateUpdate( self._message, key, self._currentStateDelta) except Exception, e: self.logger.error("Error when updating the state " + str(e)) # StateTransitionError error = StateTransitionError(ErrorType.FRAMEWORK, ErrorCode.ERROR, e) self._stateModel.rollbackOnError(message, context, error) self._statusUpdateUtil.logError(message, HelixStateTransitionHandler, e, "Error when update the state ", accessor)
class ZNRecord: """ Java modifiers: static Type: Logger """ # _logger = Logger.getLogger(__name__) _logger = get_logger(__name__) # console = logging.StreamHandler() # formatter = logging.Formatter('%(asctime)-15s [%(process)d] [%(levelname)s] %(message)s') # console.setFormatter(formatter) # console.setLevel(logging.INFO) # _logger.addHandler(console) _logger.setLevel(logging.INFO) """ # Annotation: @JsonIgnore(True) Java modifiers: final static Type: String """ LIST_FIELD_BOUND = "listField.bound" """ # Annotation: @JsonIgnore(True) Java modifiers: final static Type: int """ SIZE_LIMIT = 1000 * 1024 def __init__(self, *args): if len(args) == 1 and (isinstance(args[0], str) or isinstance(args[0], unicode)): self.__init_id__(args[0]) elif len(args) == 1 and isinstance(args[0], ZNRecord): self.__init_record__(args[0]) elif len(args) == 2 and isinstance(args[0],ZNRecord) and (isinstance(args[1], str) or isinstance(args[1], unicode)): self.__init_record_id__(self, *args) elif len(args) == 2 and isinstance(args[0],ZNRecord) and isinstance(args[1], int): self.__init_record_version__(self, *args) else: raise IllegalArgumentException("Input arguments not supported. args = %s" % list(args)) """ @JsonCreator Parameters: String id """ def __init_id__(self, id): self.id = id # simpleFields = TreeMap<String, String>() # mapFields = TreeMap<String, Map<String, String>>() # listFields = TreeMap<String, List<String>>() self.simpleFields = {} self.mapFields = {} self.listFields = {} self._version = 0 # TODO: What should be the default? self._creationTime = None self._modifiedTime = None self._deltaList = [] """ Parameters: ZNRecord record """ def __init_record__(self, record): self.__init_record_id__(record, record.getId()) """ Parameters: ZNRecord record String id """ def __init_record_id__(self, record, id): self.__init_id__(id) self.simpleFields= copy.deepcopy(record.getSimpleFields()) self.mapFields= copy.deepcopy(record.getMapFields()) self.listFields = copy.deepcopy(record.getListFields()) self._version = record.getVersion() self._creationTime = record.getCreationTime() self._modifiedTime = record.getModifiedTime() """ Parameters: ZNRecord record int version """ def __init_record_version__(self, record, version): self.__init__(record) self._version = version def setDeltaList(self, deltaList): """ Returns void Parameters: deltaList: List<ZNRecordDelta> # Annotation: @JsonIgnore(True) """ self._deltaList = deltaList def getDeltaList(self): """ Returns List<ZNRecordDelta> # Annotation: @JsonIgnore(True) """ return self._deltaList def getSimpleFields(self): """ Returns Map<String, String> @JsonProperty """ return self.simpleFields def setSimpleFields(self, simpleFields): """ Returns void Parameters: simpleFields: Map<String, String> @JsonProperty """ self.simpleFields = simpleFields def getMapFields(self): """ Returns Map<String, Map<String, String>> @JsonProperty """ return self.mapFields def setMapFields(self, mapFields): """ Returns void Parameters: mapFields: Map<String, Map<String, String>> @JsonProperty """ self.mapFields = mapFields def getListFields(self): """ Returns Map<String, List<String>> @JsonProperty """ return self.listFields def setListFields(self, listFields): """ Returns void Parameters: listFields: Map<String, List<String>> @JsonProperty """ self.listFields = listFields def setSimpleField(self, k, v): """ Returns void Parameters: k: Stringv: String @JsonProperty """ self.simpleFields.__setitem__(k, v) def getId(self): """ Returns String @JsonProperty """ return self.id def setMapField(self, k, v): """ Returns void Parameters: k: Stringv: Map<String, String> """ self.mapFields.__setitem__(k, v) def setListField(self, k, v): """ Returns void Parameters: k: Stringv: List<String> """ self.listFields.__setitem__(k, v) def getSimpleField(self, k): """ Returns String Parameters: k: String """ # return str(self.simpleFields.get(k)) # get rid of the unicode, this will cause the None to return as 'None' return self.simpleFields.get(k) # get rid of the unicode def getMapField(self, k): """ Returns Map<String, String> Parameters: k: String """ return self.mapFields.get(k) def getListField(self, k): """ Returns List<String> Parameters: k: String """ return self.listFields.get(k) def __str__(self): """ Returns String @Override """ # StringBuffer return self.id + ", " + str(self.simpleFields) + str(self.mapFields) + str(self.listFields) # sb = [] # sb.append(id + ", ") # if self.simpleFields != None: # sb.append(simpleFields) # # if mapFields != None: # sb.append(mapFields) # # if listFields != None: # sb.append(listFields) # # return sb.toString() def merge_record(self, record): """ Returns void Parameters: record: ZNRecord """ if record == None: return if len(record.getDeltaList()) > 0: self._logger.info("Merging with delta list, recordId = " + self.id + " other:" + record.getId()) self.merge(record.getDeltaList()) return #simpleFields.putAll(record.simpleFields) self.simpleFields.update(record.simpleFields) for key in record.mapFields.keys(): map = record.mapFields.get(key) if map != None: map.update(record.mapFields[key]) # map.putAll(record.mapFields.get(key)) else: self.mapFields.__setitem__(key, record.mapFields.get(key)) for key in record.listFields.keys(): # List<String> list = self.listFields.get(key) if list != None: list.extend(record.listFields.get(key)) else: self.listFields.__setitem__(key, record.listFields.get(key)) def merge_delta(self, delta): """ Returns void Parameters: delta: ZNRecordDelta """ if delta.getMergeOperation() == MergeOperation.ADD: self.merge(delta.getRecord()) else: if delta.getMergeOperation() == MergeOperation.SUBTRACT: self.subtract(delta.getRecord()) def merge_delta_list(self, deltaList): """ Returns void Parameters: deltaList: List<ZNRecordDelta> """ for delta in deltaList: self.merge(delta) def equals(self, obj): """ Returns boolean Parameters: obj: Object @Override """ if not (type(obj) == ZNRecord): return False # ZNRecord that = obj # that = (ZNRecord) obj if self.getSimpleFields().size() != that.getSimpleFields().size(): return False if self.getMapFields().size() != that.getMapFields().size(): return False if self.getListFields().size() != that.getListFields().size(): return False if not (self.getSimpleFields() == that.getSimpleFields()): return False if not (self.getMapFields() == that.getMapFields()): return False if not (self.getListFields() == that.getListFields()): return False return True def subtract(self, value): """ Returns void Parameters: value: ZNRecord """ for key in value.getSimpleFields().keys(): if key in self.simpleFields: self.simpleFields.pop(key) for key in value.getListFields().keys(): if key in self.listFields.containsKey: self.listFields.pop(key) for key in value.getMapFields().keys(): if key in self.mapFields: self.mapFields.pop(key) def getVersion(self): """ Returns int # Annotation: @JsonIgnore(True) """ return self._version def setVersion(self, version): """ Returns void Parameters: version: int # Annotation: @JsonIgnore(True) """ self._version = version def getCreationTime(self): """ Returns long # Annotation: @JsonIgnore(True) """ return self._creationTime def setCreationTime(self, creationTime): """ Returns void Parameters: creationTime: long # Annotation: @JsonIgnore(True) """ _creationTime = creationTime def getModifiedTime(self): """ Returns long # Annotation: @JsonIgnore(True) """ return self._modifiedTime def setModifiedTime(self, modifiedTime): """ Returns void Parameters: modifiedTime: long # Annotation: @JsonIgnore(True) """ _modifiedTime = modifiedTime def merge(self, *args): if len(args) == 1 and isinstance(args[0], ZNRecord): self.merge_record(*args) elif len(args) == 1 and isinstance(args[0], ZNRecordDelta): self.merge_delta(*args) elif len(args) == 1 and isinstance(args[0], list): self.merge_delta_list(*args) else: raise IllegalArgumentException("Input arguments not supported. args = %s" % args)
class ConfigAccessor: """ Java modifiers: private static Type: Logger """ LOG = get_logger(__name__) """ Java modifiers: private final static Type: StringTemplate """ template = StringTemplate() template.addEntry(ConfigScopeProperty.CLUSTER, 1, "/{clusterName}/CONFIGS/CLUSTER") template.addEntry( ConfigScopeProperty.CLUSTER, 2, "/{clusterName}/CONFIGS/CLUSTER/{clusterName}|SIMPLEKEYS") template.addEntry(ConfigScopeProperty.PARTICIPANT, 1, "/{clusterName}/CONFIGS/PARTICIPANT") template.addEntry( ConfigScopeProperty.PARTICIPANT, 2, "/{clusterName}/CONFIGS/PARTICIPANT/{participantName}|SIMPLEKEYS") template.addEntry(ConfigScopeProperty.RESOURCE, 1, "/{clusterName}/CONFIGS/RESOURCE") template.addEntry( ConfigScopeProperty.RESOURCE, 2, "/{clusterName}/CONFIGS/RESOURCE/{resourceName}|SIMPLEKEYS") template.addEntry( ConfigScopeProperty.PARTITION, 2, "/{clusterName}/CONFIGS/RESOURCE/{resourceName}|MAPKEYS") template.addEntry( ConfigScopeProperty.PARTITION, 3, "/{clusterName}/CONFIGS/RESOURCE/{resourceName}|MAPMAPKEYS|{partitionName}" ) """ Parameters: ZkClient zkClient """ def __init__(self, zkClient): self.zkClient = zkClient def get(self, *args): if len(args) == 2 and (isinstance(args[1], str) or isinstance(args[1], unicode)): return self.get_single(*args) elif len(args) == 2 and isinstance(args[1], list): return self.get_list(*args) else: raise IllegalArgumentException( "Input arguments not supported. args = %s" % list(args)) def get_single(self, scope, key): """ Returns String Parameters: scope: ConfigScopekey: String """ ret = self.get_list(scope, [key]) # not empty dict if ret: return ret[key] def get_list(self, scope, keyList): if scope is None or scope.getScope() is None: self.LOG.error("Scope can't be null") return None # String value = None # String clusterName = scope.getClusterName() if not ZKUtil.isClusterSetup(clusterName, self.zkClient): raise HelixException("cluster " + clusterName + " is not setup yet") # String scopeStr = scope.getScopeStr() retDict = {} # String[] # splits = scopeStr.split("\|") splits = re.split("[\\|]", scopeStr) # ZNRecord record = self.zkClient.readData(splits[0], True) if record is not None: for key in keyList: if len(splits) == 1: value = record.getSimpleField(key) else: if splits.length == 2: if record.getMapField(splits[1]) is not None: value = record.getMapField(splits[1]).get(key) retDict[key] = value return retDict # return value def set(self, *args): if len(args) == 2 and (isinstance(args[1], str) or isinstance(args[1], unicode)): self.set_single(*args) elif len(args) == 2 and isinstance(args[1], dict): self.set_list(*args) else: raise IllegalArgumentException( "Input arguments not supported. args = %s" % list(args)) def set_single(self, scope, key, value): ''' Returns void Parameters: scope: ConfigScopekey: Stringvalue: String ''' self.set_list(scope, {key: value}) def set_list(self, scope, key_value_dict): """ Returns void Parameters: scope: ConfigScopekey: Stringvalue: String """ if scope is None or scope.getScope() is None: self.LOG.error("Scope can't be null") return # String clusterName = scope.getClusterName() if not ZKUtil.isClusterSetup(clusterName, self.zkClient): raise HelixException("cluster " + clusterName + " is not setup yet") # String scopeStr = scope.getScopeStr() # String[] splits = re.split("[\\|]", scopeStr) # String #znodeId = splits[0].substring(splits[0].lastIndexOf('/') + 1) znodeId = splits[0].split('/')[-1] # ZNRecord update = ZNRecord(znodeId) for key, value in key_value_dict.iteritems(): if len(splits) == 1: update.setSimpleField(key, value) else: if len(splits) == 2: if update.getMapField(splits[1]) == None: update.setMapField(splits[1], {}) update.getMapField(splits[1])[key] = value ZKUtil.createOrUpdate(self.zkClient, splits[0], update, True, True) return
#from org.apache.helix.manager.zk import * #from java.util.concurrent.atomic import AtomicBoolean #from org.apache.log4j import Logger #from org.apache.zookeeper.AsyncCallback import DataCallback #from org.apache.zookeeper.AsyncCallback import StatCallback #from org.apache.zookeeper.AsyncCallback import StringCallback #from org.apache.zookeeper.AsyncCallback import VoidCallback #from org.apache.zookeeper.KeeperException import Code #from org.apache.zookeeper.data import Stat import threading from kazoo.exceptions import NoNodeError, NodeExistsError, BadVersionError from org.apache.helix.util.logger import get_logger from kazoo.interfaces import IAsyncResult LOG = get_logger(__name__) #class ZkAsyncCallbacks: # # """ # Java modifiers: # private static # Type: # Logger # """ class DefaultCallback(object): def __init__(self): self._asyncResult = None
class ConfigScopeBuilder: """ Java modifiers: private static Type: Logger """ LOG = get_logger(__name__) """ Java modifiers: private static Type: StringTemplate """ template = StringTemplate() template.addEntry(ConfigScopeProperty.CLUSTER, 1, "CLUSTER={clusterName}") template.addEntry(ConfigScopeProperty.RESOURCE, 2, "CLUSTER={clusterName},RESOURCE={resourceName}") template.addEntry(ConfigScopeProperty.PARTITION, 3, "CLUSTER={clusterName},RESOURCE={resourceName},PARTITION={partitionName}") template.addEntry(ConfigScopeProperty.PARTICIPANT, 2, "CLUSTER={clusterName},PARTICIPANT={participantName}") def getScopeMap(self): """ Returns Map<ConfigScopeProperty, String> """ return self._scopeMap """ """ def __init__(self): self._scopeMap = {} # self._scopeMap = HashMap<ConfigScopeProperty, String>() def forCluster(self, clusterName): """ Returns ConfigScopeBuilder Parameters: clusterName: String """ self._scopeMap.__setitem__(ConfigScopeProperty.CLUSTER, clusterName) return self def forParticipant(self, participantName): """ Returns ConfigScopeBuilder Parameters: participantName: String """ self._scopeMap.__setitem__(ConfigScopeProperty.PARTICIPANT, participantName) return self def forResource(self, resourceName): """ Returns ConfigScopeBuilder Parameters: resourceName: String """ self._scopeMap.__setitem__(ConfigScopeProperty.RESOURCE, resourceName) return self def forPartition(self, partitionName): """ Returns ConfigScopeBuilder Parameters: partitionName: String """ self._scopeMap.__setitem__(ConfigScopeProperty.PARTITION, partitionName) return self def build(self, *args): """ Returns ConfigScope """ if len(args)==0: return ConfigScope(self) elif len(args)==3: return self.build_3(*args) elif len(args)==1: return self.build_1(*args) else: raise IllegalArgumentException("Wrong args: %s" % args) def build_3(self, scope, clusterName, scopeKeys): """ Returns ConfigScope Parameters: scope: ConfigScopePropertyclusterName: StringscopeKeys: String """ if scopeKeys == None: scopeKeys = {} # scopeKeys = new String[] {} # String[] # args = new String[1 + scopeKeys.length] # args[0] = clusterName # System.arraycopy(scopeKeys, 0, args, 1, scopeKeys.length) args=[clusterName] args.extend(scopeKeys[1:]) # String scopePairs = ConfigScopeBuilder.template.instantiate(scope, args) return self.build(scopePairs) def build_1(self, scopePairs): """ Returns ConfigScope Parameters: scopePairs: String """ # String[] # scopes = scopePairs.split("[\\s,]+") # TODO: re split scopes = scopePairs.split("[\s,]+") for scope in scopes: try: # int idx = scope.indexOf('=') if idx == -1: self.LOG.error("Invalid scope string: " + str(scope)) continue # String scopeStr = scope.substring(0, idx) # String value = scope.substring(idx + 1) # ConfigScopeProperty scopeProperty = ConfigScopeProperty.valueOf(scopeStr) self._scopeMap.put(scopeProperty, value) except Exception, e: self.LOG.error("Invalid scope string: " + str(scope)) continue return self.build()
class StateModel(object): """ Java modifiers: final static Type: String """ DEFAULT_INITIAL_STATE = "OFFLINE" logger = get_logger(__name__) def __init__(self): self._currentState = StateModel.DEFAULT_INITIAL_STATE; def getCurrentState(self): """ Returns String """ return self._currentState def defaultTransitionHandler(self): """ Returns void """ self.logger.error("Default default handler. The idea is to invoke this if no transition method is found. Yet to be implemented") def updateState(self, newState): """ Returns boolean Parameters: newState: String """ _currentState = newState return True def rollbackOnError(self, message, context, error): """ Returns void Parameters: message: Messagecontext: NotificationContexterror: StateTransitionError """ self.logger.error("Default rollback method invoked on error. Error Code:" + str(error.getCode())) def reset(self): """ Returns void """ self.logger.warn("Default reset method invoked. Either because the process longer own this resource or session timedout")
class PersistentStats(HelixProperty): """ Java modifiers: private final static Type: Logger """ _logger = get_logger(__name__) """ Java modifiers: final static Type: String """ nodeName = "PersistentStats" """ Parameters: String id """ def __init__(self, id): super(id) """ Parameters: ZNRecord record """ def __init__(self, record): super(record) def setSessionId(self, sessionId): """ Returns void Parameters: sessionId: String """ self._record.setSimpleField( PersistentStatsProperty.SESSION_ID.toString(), sessionId) def getSessionId(self): """ Returns String """ return self._record.getSimpleField( PersistentStatsProperty.SESSION_ID.toString()) def getInstanceName(self): """ Returns String """ return self._record.getId() def getMapFields(self): """ Returns Map<String, Map<String, String>> """ return self._record.getMapFields() def getStatFields(self, statName): """ Returns Map<String, String> Parameters: statName: String """ return self._record.getMapField(statName) def isValid(self): """ Returns boolean @Override """ return True
class LiveInstance(HelixProperty): """ Java modifiers: private final static Type: Logger """ _logger = get_logger(__name__) """ Parameters: String id """ def __init__(self, *args): super(LiveInstance, self).__init__(*args) # def __init__(self, id): # super(id) # # """ # # Parameters: # ZNRecord record # """ # def __init__(self, record): # super(record) def setSessionId(self, sessionId): """ Returns void Parameters: sessionId: String """ self._record.setSimpleField( LiveInstanceProperty.toString(LiveInstanceProperty.SESSION_ID), sessionId) def getSessionId(self): """ Returns String """ return self._record.getSimpleField( LiveInstanceProperty.toString(LiveInstanceProperty.SESSION_ID)) def getInstanceName(self): """ Returns String """ return self._record.getId() def getHelixVersion(self): """ Returns String """ return self._record.getSimpleField( LiveInstanceProperty.toString(LiveInstanceProperty.HELIX_VERSION)) def setHelixVersion(self, helixVersion): """ Returns void Parameters: helixVersion: String """ self._record.setSimpleField( LiveInstanceProperty.toString(LiveInstanceProperty.HELIX_VERSION), helixVersion) def getLiveInstance(self): """ Returns String """ return self._record.getSimpleField( LiveInstanceProperty.toString(LiveInstanceProperty.LIVE_INSTANCE)) def setLiveInstance(self, leader): """ Returns void Parameters: leader: String """ self._record.setSimpleField( LiveInstanceProperty.toString(LiveInstanceProperty.LIVE_INSTANCE), leader) def getModifiedTime(self): """ Returns long """ return self._record.getModifiedTime() def getWebserviceUrl(self): """ Returns String """ return self._record.getSimpleField( LiveInstanceProperty.toString( LiveInstanceProperty.ZKPROPERTYTRANSFERURL)) def setWebserviceUrl(self, url): """ Returns void Parameters: url: String """ self._record.setSimpleField( LiveInstanceProperty.toString( LiveInstanceProperty.ZKPROPERTYTRANSFERURL), url) def isValid(self): """ Returns boolean @Override """ if self.getSessionId() == None: self._logger.error("liveInstance does not have session id. id:" + str(self._record.getId())) return False if self.getHelixVersion() == None: self._logger.error("liveInstance does not have CLM verion. id:" + str(self._record.getId())) return False return True
class ConfigScope: """ Java modifiers: private static Type: Logger """ # LOG = Logger.getLogger(ConfigScope.class) LOG = get_logger(__name__) """ Java modifiers: private final static Type: List<ConfigScopeProperty> """ # scopePriority = ArrayList<ConfigScopeProperty>() scopePriority = [] """ Java modifiers: private final static Type: Map<ConfigScopeProperty, Map<ConfigScopeProperty, ConfigScopeProperty>> """ scopeTransition = {} # scopeTransition = HashMap<ConfigScopeProperty, Map<ConfigScopeProperty, ConfigScopeProperty>>() """ Java modifiers: private final static Type: StringTemplate """ template = StringTemplate() scopePriority.append(ConfigScopeProperty.CLUSTER) scopePriority.append(ConfigScopeProperty.PARTICIPANT) scopePriority.append(ConfigScopeProperty.RESOURCE) scopePriority.append(ConfigScopeProperty.PARTITION) # scopeTransition.__setitem__(ConfigScopeProperty.CLUSTER, HashMap<ConfigScopeProperty, ConfigScopeProperty>()) scopeTransition.__setitem__(ConfigScopeProperty.CLUSTER, {}) scopeTransition.get(ConfigScopeProperty.CLUSTER).__setitem__( ConfigScopeProperty.PARTICIPANT, ConfigScopeProperty.PARTICIPANT) scopeTransition.get(ConfigScopeProperty.CLUSTER).__setitem__( ConfigScopeProperty.RESOURCE, ConfigScopeProperty.RESOURCE) scopeTransition.__setitem__(ConfigScopeProperty.RESOURCE, {}) # scopeTransition.put(ConfigScopeProperty.RESOURCE, HashMap<ConfigScopeProperty, ConfigScopeProperty>()) scopeTransition.get(ConfigScopeProperty.RESOURCE).__setitem__( ConfigScopeProperty.PARTITION, ConfigScopeProperty.PARTITION) template.addEntry(ConfigScopeProperty.CLUSTER, 2, "/{clusterName}/CONFIGS/CLUSTER/{clusterName}") template.addEntry(ConfigScopeProperty.PARTICIPANT, 2, "/{clusterName}/CONFIGS/PARTICIPANT/{participantName}") template.addEntry(ConfigScopeProperty.RESOURCE, 2, "/{clusterName}/CONFIGS/RESOURCE/{resourceName}") template.addEntry( ConfigScopeProperty.PARTITION, 3, "/{clusterName}/CONFIGS/RESOURCE/{resourceName}|{partitionName}") """ Parameters: ConfigScopeBuilder configScopeBuilder """ def __init__(self, configScopeBuilder): # Map<ConfigScopeProperty, String> scopeMap = configScopeBuilder.getScopeMap() # List<String> keys = [] # keys = ArrayList<String>() # ConfigScopeProperty curScope = None for scope in self.scopePriority: if scopeMap.__contains__(scope): if curScope == None and scope == ConfigScopeProperty.CLUSTER: keys.append(scopeMap.get(scope)) curScope = ConfigScopeProperty.CLUSTER else: if curScope == None: raise IllegalArgumentException( "Missing CLUSTER scope. Can't build scope using " + configScopeBuilder) else: if not self.scopeTransition.__contains__( curScope) or not self.scopeTransition.get( curScope).__contains__(scope): raise IllegalArgumentException( "Can't build scope using " + configScopeBuilder) keys.append(scopeMap.get(scope)) curScope = self.scopeTransition.get(curScope).get( scope) if curScope == ConfigScopeProperty.CLUSTER: keys.append(scopeMap.get(ConfigScopeProperty.CLUSTER)) # String scopeStr = self.template.instantiate(curScope, keys) # scopeStr = self.template.instantiate(curScope, keys.toArray(new String[0])) self._clusterName = keys[0] self._scopeStr = scopeStr self._scope = curScope def getScope(self): """ Returns ConfigScopeProperty """ return self._scope def getClusterName(self): """ Returns String """ return self._clusterName def getScopeStr(self): """ Returns String """ return self._scopeStr # def toString(self): # """ # Returns String # @Override # # # """ # return super(null, self).toString() + ": " + _scopeStr def __repr__(self): """ Returns String @Override """ return ": " + self._scopeStr
#from org.apache.helix.manager.zk import * #from java.util.concurrent.atomic import AtomicBoolean #from org.apache.log4j import Logger #from org.apache.zookeeper.AsyncCallback import DataCallback #from org.apache.zookeeper.AsyncCallback import StatCallback #from org.apache.zookeeper.AsyncCallback import StringCallback #from org.apache.zookeeper.AsyncCallback import VoidCallback #from org.apache.zookeeper.KeeperException import Code #from org.apache.zookeeper.data import Stat import threading from kazoo.exceptions import NoNodeError, NodeExistsError, BadVersionError from org.apache.helix.util.logger import get_logger from kazoo.interfaces import IAsyncResult LOG = get_logger(__name__) #class ZkAsyncCallbacks: # # """ # Java modifiers: # private static # Type: # Logger # """ class DefaultCallback(object): def __init__(self): self._asyncResult = None self._cv = threading.Condition()
class AsyncCallbackService(MessageHandlerFactory): """ Java modifiers: private static Type: Logger """ _logger = get_logger(__name__) """ """ def __init__(self): pass def registerAsyncCallback(self, correlationId, callback): """ Returns void Parameters: correlationId: Stringcallback: AsyncCallback """ if self._callbackMap.containsKey(correlationId): self._logger.warn("correlation id " + correlationId + " already registered") self._logger.info("registering correlation id " + correlationId) self._callbackMap.put(correlationId, callback) def verifyMessage(self, message): """ Returns void Parameters: message: Message """ if not message.getMsgType().toString().upper() == MessageType.TASK_REPLY.toString().upper(): # String errorMsg = "Unexpected msg type for message " + message.getMsgId() + " type:" + message.getMsgType() + " Expected : " + MessageType.TASK_REPLY self._logger.error(errorMsg) raise HelixException(errorMsg) # String correlationId = message.getCorrelationId() if correlationId == None: # String errorMsg = "Message " + message.getMsgId() + " does not have correlation id" self._logger.error(errorMsg) raise HelixException(errorMsg) if not self._callbackMap.containsKey(correlationId): # String errorMsg = "Message " + message.getMsgId() + " does not have correponding callback. Probably timed out already. Correlation id: " + correlationId self._logger.error(errorMsg) raise HelixException(errorMsg) self._logger.info("Verified reply message " + message.getMsgId() + " correlation:" + correlationId) def createHandler(self, message, context): """ Returns MessageHandler Parameters: message: Messagecontext: NotificationContext @Override """ self.verifyMessage(message) return AsyncCallbackMessageHandler(message.getCorrelationId(), message, context) def getMessageType(self): """ Returns String @Override """ return MessageType.toString(MessageType.TASK_REPLY) def reset(self): """
class HelixTask(): #class HelixTask(Callable<HelixTaskResult>): """ Java modifiers: private static Type: Logger """ logger = get_logger(__name__) # class TimeoutCancelTask(TimerTask): # # """ # # Parameters: # HelixTaskExecutor executor # Message message # NotificationContext context # """ # def __init__(self, executor, message, context): # self._executor = executor # self._message = message # self._context = context # # # def run(self): # """ # Returns void # @Override # # # """ # self._isTimeout = True # self.logger.warn("Message time out, canceling. id:" + self._message.getMsgId() + " timeout : " + self._message.getExecutionTimeout()) # self._handler.onTimeout() # self._executor.cancelTask(self._message, self._context) # """ Parameters: Message message NotificationContext notificationContext MessageHandler handler HelixTaskExecutor executor Throws: Exception """ def __init__(self, message, notificationContext, handler, executor): self._notificationContext = notificationContext self._message = message self._handler = handler self._manager = notificationContext.getManager() self._statusUpdateUtil = StatusUpdateUtil() self._executor = executor self._isTimeout = False def call(self): """ Returns HelixTaskResult @Override """ # Timer # TODO: enable timer timer = None # if self._message.getExecutionTimeout() > 0: # timer = Timer(True) # timer.schedule(TimeoutCancelTask(self._executor, self._message, self._notificationContext), self._message.getExecutionTimeout()) # self.logger.info("Message starts with timeout " + self._message.getExecutionTimeout() + " MsgId:" + self._message.getMsgId()) # else: # self.logger.info("Message does not have timeout. MsgId:" + self._message.getMsgId() + "/" + self._message.getPartitionName()) # HelixTaskResult taskResult = HelixTaskResult() # Exception exception = None # ErrorType type = ErrorType.INTERNAL # ErrorCode code = ErrorCode.ERROR # long start = time.time() self.logger.info("msg:" + str(self._message.getMsgId()) + " handling task begin, at: " + str(start)) # HelixDataAccessor accessor = self._manager.getHelixDataAccessor() self._statusUpdateUtil.logInfo(self._message, HelixTask, "Message handling task begin execute", accessor) self._message.setExecuteStartTimeStamp(time.time()) try: taskResult = self._handler.handleMessage() exception = taskResult.getException() # except Exception, e: except KeyboardInterrupt, e: self._statusUpdateUtil.logError(self._message, HelixTask, e, "State transition interrupted, timeout:" + str(self._isTimeout), accessor) self.logger.info("Message " + self._message.getMsgId() + " is interrupted") taskResult.setInterrupted(True) taskResult.setException(e) exception = e # except Exception, e: # # String # errorMessage = "Exception while executing a message. " + e + " msgId: " + self._message.getMsgId() + " type: " + self._message.getMsgType() # self.logger.error(errorMessage+ str(e)) # self._statusUpdateUtil.logError(self._message, HelixTask, e, errorMessage, accessor) # taskResult.setSuccess(False) # taskResult.setException(e) # taskResult.setMessage(e.getMessage()) # exception = e if timer != None: timer.cancel() if taskResult.isSucess(): self._statusUpdateUtil.logInfo(self._message, self._handler.__class__, "Message handling task completed successfully", accessor) self.logger.info("Message " + self._message.getMsgId() + " completed.") else: if taskResult.isInterrupted(): self.logger.info("Message " + self._message.getMsgId() + " is interrupted") code = ternary(self._isTimeout, ErrorCode.TIMEOUT, ErrorCode.CANCEL) if self._isTimeout: # int retryCount = self._message.getRetryCount() self.logger.info("Message timeout, retry count: " + retryCount + " MSGID:" + self._message.getMsgId()) self._statusUpdateUtil.logInfo(self._message, self._handler.__class__, "Message handling task timeout, retryCount:" + retryCount, accessor) if retryCount > 0: self._message.setRetryCount(retryCount - 1) self._executor.scheduleTask(self._message, self._handler, self._notificationContext) return taskResult else: # String errorMsg = "Message execution failed. msgId: " + self._message.getMsgId() + taskResult.getMessage() if exception is not None: errorMsg += exception self.logger.error(errorMsg+ str(exception)) self._statusUpdateUtil.logError(self._message, self._handler.getClass(), errorMsg, accessor) try: if not self._message.getGroupMessageMode(): self.removeMessageFromZk(accessor, self._message) self.reportMessageStat(self._manager, self._message, taskResult) self.sendReply(accessor, self._message, taskResult) else: # GroupMessageInfo info = self._executor._groupMsgHandler.onCompleteSubMessage(self._message) if info is not None: # Map<PropertyKey, CurrentState> curStateMap = info.merge() for key in curStateMap.keys(): accessor.updateProperty(key, curStateMap.get(key)) self.removeMessageFromZk(accessor, self._message) self.reportMessageStat(self._manager, self._message, taskResult) self.sendReply(accessor, self._message, taskResult) self._executor.reportCompletion(self._message) # except KeyboardInterrupt, e: #TODO: should we print the error except Exception as e: # String errorMessage = "Exception after executing a message, msgId: " + self._message.getMsgId() + e self.logger.error(errorMessage+ str(e)) self._statusUpdateUtil.logError(self._message, HelixTask, errorMessage, accessor) exception = e type = ErrorType.FRAMEWORK code = ErrorCode.ERROR finally: # long end = time.time() # end = System.currentTimeMillis() self.logger.info("msg:" + self._message.getMsgId() + " handling task completed, results:" + str(taskResult.isSucess()) + ", at: " + str(end) + ", took:" + str(end - start)) if exception is not None: self._handler.onError(exception, code, type) return taskResult
class HelixCustomCodeRunner: """ Java modifiers: private final static Type: String """ LEADER_STANDBY = "LeaderStandby" """ Java modifiers: private static Type: Logger """ LOG = get_logger(__name__) """ Java modifiers: private static Type: String """ PARTICIPANT_LEADER = "PARTICIPANT_LEADER" """ Parameters: HelixManager manager String zkAddr """ def __init__(self, manager, zkAddr): self._manager = manager self._zkAddr = zkAddr def invoke(self, callback): """ Returns HelixCustomCodeRunner Parameters: callback: CustomCodeCallbackHandler """ self._callback = callback return self def on(self, notificationTypes): """ Returns HelixCustomCodeRunner Parameters: notificationTypes: ChangeType """ self._notificationTypes = notificationTypes return self def usingLeaderStandbyModel(self, id): """ Returns HelixCustomCodeRunner Parameters: id: String """ self._resourceName = HelixCustomCodeRunner.PARTICIPANT_LEADER + "_" + id return self def start(self): """ Returns void Throws: Exception """ if self._callback == None or self._notificationTypes == None or self._notificationTypes.size( ) == 0 or self._resourceName == None: raise IllegalArgumentException( "Require callback | notificationTypes | resourceName") self.LOG.info("Register participantLeader on " + self._notificationTypes + " using " + self._resourceName) _stateModelFty = GenericLeaderStandbyStateModelFactory( self._callback, self._notificationTypes) # StateMachineEngine stateMach = self._manager.getStateMachineEngine() stateMach.registerStateModelFactory( HelixCustomCodeRunner.LEADER_STANDBY, _stateModelFty, self._resourceName) # ZkClient zkClient = None try: zkClient = ZkClient(self._zkAddr, ZkClient.DEFAULT_CONNECTION_TIMEOUT) zkClient.setZkSerializer(ZNRecordSerializer()) # HelixDataAccessor accessor = ZKHelixDataAccessor(self._manager.getClusterName(), ZkBaseDataAccessor(zkClient)) # Builder keyBuilder = accessor.keyBuilder() # IdealState idealState = IdealState(self._resourceName) idealState.setIdealStateMode( IdealStateModeProperty.AUTO.toString()) idealState.setReplicas(StateModelToken.ANY_LIVEINSTANCE.toString()) idealState.setNumPartitions(1) idealState.setStateModelDefRef( HelixCustomCodeRunner.LEADER_STANDBY) idealState.setStateModelFactoryName(self._resourceName) # List<String> prefList = [ StateModelToken.toString(StateModelToken.ANY_LIVEINSTANCE) ] idealState.getRecord().setListField(self._resourceName + "_0", prefList) # List<String> idealStates = accessor.getChildNames(keyBuilder.idealStates()) while (idealStates == None or not idealStates.__contains__(self._resourceName)): accessor.setProperty( keyBuilder.idealStates(self._resourceName), idealState) idealStates = accessor.getChildNames(keyBuilder.idealStates()) self.LOG.info("Set idealState for participantLeader:" + self._resourceName + ", idealState:" + idealState) finally: if zkClient != None and zkClient.getConnection() != None: zkClient.close() def stop(self): """ Returns void """ self.LOG.info("Removing stateModelFactory for " + self._resourceName) self._manager.getStateMachineEngine().removeStateModelFactory( HelixCustomCodeRunner.LEADER_STANDBY, self._stateModelFty, self._resourceName)
class PropertyKey: """ Java modifiers: private static Type: Logger """ # LOG = Logger.getLogger(PropertyKey.class) LOG = get_logger(__name__) LOG.setLevel(logging.INFO) """ Parameters: PropertyType type Class<? extends HelixProperty> typeClazz String... params """ def __init__(self, type, typeClazz, *params): self._type = type # if params == None or params.length == 0 or Arrays.asList(params).contains(None): if not params: raise IllegalArgumentException("params cannot be null") self._params = params self._typeClazz = typeClazz # def hashCode(self): # """ # Returns int # @Override # # # """ # return super(null, self).hashCode() def getPath(self): """ Returns String """ # String clusterName = self._params[0] # String[] # subKeys = Arrays.copyOfRange(self._params, 1, self._params.length) subKeys = self._params[1:] # String path = PropertyPathConfig.getPath(self._type, clusterName, *subKeys) if path == None: # self.LOG.error("Invalid property key with type:" + str(self._type)+ "subKeys:" + str(Arrays.toString(self._params))) self.LOG.error("Invalid property key with type:" + str(self._type) + "subKeys:" + str(str(self._params))) return path # Java modifiers: # static def getType(self): """ Returns PropertyType """ return self._type def getParams(self): """ Returns String[] """ return self._params def getTypeClass(self): """ Returns Class<? extends HelixProperty> """ return self._typeClazz def main(args): """ Returns void Parameters: args: String[] Java modifiers: static """ pass # # ZkClient # zkClient = ZkClient("localhost:2181") # zkClient.waitUntilConnected(10, TimeUnit.SECONDS) # # BaseDataAccessor # baseDataAccessor = ZkBaseDataAccessor(zkClient) # # HelixDataAccessor # accessor = ZKHelixDataAccessor("test-cluster", baseDataAccessor) # # Builder # builder = PropertyKey.Builder("test-cluster") # # HelixProperty # value = IdealState("test-resource") # accessor.createProperty(builder.idealStates("test-resource"), value) class Builder: """ Parameters: String clusterName """ def __init__(self, clusterName): self._clusterName = clusterName def idealStates(self): """ Returns PropertyKey """ return PropertyKey(PropertyType.IDEALSTATES, IdealState, self._clusterName) def idealStates(self, resourceName): """ Returns PropertyKey Parameters: resourceName: String """ return PropertyKey(PropertyType.IDEALSTATES, IdealState, self._clusterName, resourceName) def stateModelDefs(self): """ Returns PropertyKey """ return PropertyKey(PropertyType.STATEMODELDEFS, StateModelDefinition, self._clusterName) def stateModelDef(self, stateModelName): """ Returns PropertyKey Parameters: stateModelName: String """ return PropertyKey(PropertyType.STATEMODELDEFS, StateModelDefinition, self._clusterName, stateModelName) def clusterConfig(self): """ Returns PropertyKey """ return PropertyKey( PropertyType.CONFIGS, None, self._clusterName, ConfigScopeProperty.toString(ConfigScopeProperty.CLUSTER)) def instanceConfigs(self): """ Returns PropertyKey """ return PropertyKey( PropertyType.CONFIGS, InstanceConfig, self._clusterName, ConfigScopeProperty.toString(ConfigScopeProperty.PARTICIPANT)) def instanceConfig(self, instanceName): """ Returns PropertyKey Parameters: instanceName: String """ return PropertyKey( PropertyType.CONFIGS, InstanceConfig, self._clusterName, ConfigScopeProperty.toString(ConfigScopeProperty.PARTICIPANT), instanceName) def resourceConfig(self, resourceName): """ Returns PropertyKey Parameters: resourceName: String """ return PropertyKey( PropertyType.CONFIGS, None, self._clusterName, ConfigScopeProperty.toString(ConfigScopeProperty.RESOURCE), resourceName) def resourceConfig(self, instanceName, resourceName): """ Returns PropertyKey Parameters: instanceName: StringresourceName: String """ return PropertyKey( PropertyType.CONFIGS, None, self._clusterName, ConfigScopeProperty.toString(ConfigScopeProperty.RESOURCE), resourceName) def partitionConfig(self, resourceName, partitionName): """ Returns PropertyKey Parameters: resourceName: StringpartitionName: String """ return PropertyKey( PropertyType.CONFIGS, None, self._clusterName, ConfigScopeProperty.toString(ConfigScopeProperty.RESOURCE), resourceName) def partitionConfig(self, instanceName, resourceName, partitionName): """ Returns PropertyKey Parameters: instanceName: StringresourceName: StringpartitionName: String """ return PropertyKey( PropertyType.CONFIGS, None, self._clusterName, ConfigScopeProperty.toString(ConfigScopeProperty.RESOURCE), resourceName) def constraints(self): """ Returns PropertyKey """ return PropertyKey( PropertyType.CONFIGS, ClusterConstraints, self._clusterName, ConfigScopeProperty.toString(ConfigScopeProperty.CONSTRAINT)) def constraint(self, constraintType): """ Returns PropertyKey Parameters: constraintType: String """ return PropertyKey( PropertyType.CONFIGS, ClusterConstraints, self._clusterName, ConfigScopeProperty.toString(ConfigScopeProperty.CONSTRAINT), constraintType) def liveInstances(self): """ Returns PropertyKey """ return PropertyKey(PropertyType.LIVEINSTANCES, LiveInstance, self._clusterName) def liveInstance(self, instanceName): """ Returns PropertyKey Parameters: instanceName: String """ return PropertyKey(PropertyType.LIVEINSTANCES, LiveInstance, self._clusterName, instanceName) def instances(self): """ Returns PropertyKey """ return PropertyKey(PropertyType.CONFIGS, None, self._clusterName) def messages(self, instanceName): """ Returns PropertyKey Parameters: instanceName: String """ return PropertyKey(PropertyType.MESSAGES, Message, self._clusterName, instanceName) def message(self, instanceName, messageId): """ Returns PropertyKey Parameters: instanceName: StringmessageId: String """ return PropertyKey(PropertyType.MESSAGES, Message, self._clusterName, instanceName, messageId) def sessions(self, instanceName): """ Returns PropertyKey Parameters: instanceName: String """ return PropertyKey(PropertyType.CURRENTSTATES, CurrentState, self._clusterName, instanceName) def currentStates(self, instanceName, sessionId): """ Returns PropertyKey Parameters: instanceName: StringsessionId: String """ return PropertyKey(PropertyType.CURRENTSTATES, CurrentState, self._clusterName, instanceName, sessionId) # def currentState(self, instanceName, sessionId, resourceName): # """ # Returns PropertyKey # Parameters: # instanceName: StringsessionId: StringresourceName: String # # # """ # return PropertyKey(PropertyType.CURRENTSTATES, CurrentState, self._clusterName, instanceName, sessionId, resourceName) def currentState(self, instanceName, sessionId, resourceName, bucketName=None): """ Returns PropertyKey Parameters: instanceName: StringsessionId: StringresourceName: StringbucketName: String """ if bucketName == None: return PropertyKey(PropertyType.CURRENTSTATES, CurrentState, self._clusterName, instanceName, sessionId, resourceName) else: return PropertyKey(PropertyType.CURRENTSTATES, CurrentState, self._clusterName, instanceName, sessionId, resourceName, bucketName) def stateTransitionStatus(self, *args): if len(args) == 4: return self.stateTransitionStatus_4(*args) elif len(args) == 3: return self.stateTransitionStatus_3(*args) elif len(args) == 2: return self.stateTransitionStatus_2(*args) else: raise IllegalArgumentException( "Input arguments not supported. args = %s" % args) def stateTransitionStatus_4(self, instanceName, sessionId, resourceName, partitionName): """ Returns PropertyKey Parameters: instanceName: StringsessionId: StringresourceName: StringpartitionName: String """ return PropertyKey(PropertyType.STATUSUPDATES, StatusUpdate, self._clusterName, instanceName, sessionId, resourceName, partitionName) def stateTransitionStatus_3(self, instanceName, sessionId, resourceName): """ Returns PropertyKey Parameters: instanceName: StringsessionId: StringresourceName: String """ return PropertyKey(PropertyType.STATUSUPDATES, StatusUpdate, self._clusterName, instanceName, sessionId, resourceName) def stateTransitionStatus_2(self, instanceName, sessionId): """ Returns PropertyKey Parameters: instanceName: StringsessionId: String """ return PropertyKey(PropertyType.STATUSUPDATES, StatusUpdate, self._clusterName, instanceName, sessionId) def taskStatus(self, instanceName, sessionId, msgType, msgId): """ Returns PropertyKey Parameters: instanceName: StringsessionId: StringmsgType: StringmsgId: String """ return PropertyKey(PropertyType.STATUSUPDATES, StatusUpdate, self._clusterName, instanceName, sessionId, msgType, msgId) def stateTransitionError(self, instanceName, sessionId, resourceName, partitionName): """ Returns PropertyKey Parameters: instanceName: StringsessionId: StringresourceName: StringpartitionName: String """ return PropertyKey(PropertyType.ERRORS, Error, self._clusterName, instanceName, sessionId, resourceName, partitionName) def stateTransitionErrors(self, instanceName, sessionId, resourceName): """ Returns PropertyKey Parameters: instanceName: StringsessionId: StringresourceName: String """ return PropertyKey(PropertyType.ERRORS, Error, self._clusterName, instanceName, sessionId, resourceName) def taskError(self, instanceName, sessionId, msgType, msgId): """ Returns PropertyKey Parameters: instanceName: StringsessionId: StringmsgType: StringmsgId: String """ return PropertyKey(PropertyType.ERRORS, None, self._clusterName, instanceName, sessionId, msgType, msgId) def externalViews(self): """ Returns PropertyKey """ return PropertyKey(PropertyType.EXTERNALVIEW, ExternalView, self._clusterName) def externalView(self, resourceName): """ Returns PropertyKey Parameters: resourceName: String """ return PropertyKey(PropertyType.EXTERNALVIEW, ExternalView, self._clusterName, resourceName) def controller(self): """ Returns PropertyKey """ return PropertyKey(PropertyType.CONTROLLER, None, self._clusterName) def controllerTaskErrors(self): """ Returns PropertyKey """ return PropertyKey(PropertyType.ERRORS_CONTROLLER, StatusUpdate, self._clusterName) def controllerTaskError(self, messageType, messageId): """ Returns PropertyKey Parameters: messageType: StringmessageId: String """ return PropertyKey(PropertyType.ERRORS_CONTROLLER, Error, self._clusterName, messageType, messageId) def controllerTaskStatuses(self, messageType): """ Returns PropertyKey Parameters: messageType: String """ return PropertyKey(PropertyType.STATUSUPDATES_CONTROLLER, StatusUpdate, self._clusterName, messageType) def controllerTaskStatus(self, messageType, messageId): """ Returns PropertyKey Parameters: messageType: StringmessageId: String """ return PropertyKey(PropertyType.STATUSUPDATES_CONTROLLER, StatusUpdate, self._clusterName, messageType, messageId) def controllerMessages(self): """ Returns PropertyKey """ return PropertyKey(PropertyType.MESSAGES_CONTROLLER, Message, self._clusterName) def controllerMessage(self, msgId): """ Returns PropertyKey Parameters: msgId: String """ return PropertyKey(PropertyType.MESSAGES_CONTROLLER, Message, self._clusterName, msgId) def controllerLeaderHistory(self): """ Returns PropertyKey """ return PropertyKey(PropertyType.HISTORY, LeaderHistory, self._clusterName) def controllerLeader(self): """ Returns PropertyKey """ return PropertyKey(PropertyType.LEADER, LiveInstance, self._clusterName) def pause(self): """ Returns PropertyKey """ return PropertyKey(PropertyType.PAUSE, PauseSignal, self._clusterName) def persistantStat(self): """ Returns PropertyKey """ return PropertyKey(PropertyType.PERSISTENTSTATS, PersistentStats, self._clusterName) def alerts(self): """ Returns PropertyKey """ return PropertyKey(PropertyType.ALERTS, Alerts, self._clusterName) def alertStatus(self): """ Returns PropertyKey """ return PropertyKey(PropertyType.ALERT_STATUS, AlertStatus, self._clusterName) def alertHistory(self): """ Returns PropertyKey """ return PropertyKey(PropertyType.ALERT_HISTORY, AlertHistory, self._clusterName) def healthReport(self, instanceName, id): """ Returns PropertyKey Parameters: instanceName: Stringid: String """ raise BaseException("Not implemented") # return PropertyKey(PropertyType.HEALTHREPORT, HealthStat, self._clusterName, instanceName, id) def healthReports(self, instanceName): """ Returns PropertyKey Parameters: instanceName: String """ raise BaseException("Not implemented")
class ZkBaseDataAccessor(BaseDataAccessor): """ Java modifiers: private static Type: Logger """ LOG = get_logger(__name__) """ Parameters: ZkClient zkClient """ def __init__(self, zkClient): self._zkClient = zkClient def create(self, *args): if len(args) == 3 and isinstance(args[1], ZNRecord): return self.create_path(args[0], args[1], None, args[2]) == RetCode.OK elif len(args) == 4 and (isinstance(args[1], ZNRecord) or args[1] == None): return self.create_path(*args) == RetCode.OK elif len(args) == 5 and isinstance(args[3], list): return self.create_paths(*args) else: import pdb pdb.set_trace() raise IllegalArgumentException("Wrong args: %s" % str(args)) # def create(self, path, record, options): # """ # Returns boolean # Parameters: # path: Stringrecord: Toptions: int # @Override # # # """ # return self.create(path, record, None, options) == RetCode.OK def create_path(self, path, record, pathCreated, options): """ Returns RetCode Parameters: path: Stringrecord: TpathCreated: List<String>options: int """ # CreateMode mode = AccessOption.getMode(options) if mode == None: self.LOG.error("Invalid create mode. options: " + str(options)) return RetCode.ERROR # boolean retry = True while retry: retry = False try: self._zkClient.create(path, record, mode) if pathCreated != None: pathCreated.add(path) return RetCode.OK except NoNodeException, e: # String parentPath = os.path.abspath(os.path.join( path, os.path.pardir)) # parentPath = File(path).getParent() try: # RetCode rc = self.create(parentPath, None, pathCreated, AccessOption.PERSISTENT) if rc == RetCode.OK or rc == RetCode.NODE_EXISTS: retry = True # TODO: # except Exception, e1: except KeyboardInterrupt, e1: self.LOG.error("Exception while creating path: " + str(parentPath) + str(e1)) return RetCode.ERROR except NodeExistsException, e: self.LOG.warn("Node already exists. path: " + path) return RetCode.NODE_EXISTS
class CurrentState(HelixProperty): """ Java modifiers: private static Type: Logger """ LOG = get_logger(__name__) def __init__(self, *args): super(CurrentState,self).__init__(*args) # if len(args)==1 and (isinstance(args[0],str) or isinstance((args[0])) # else: # raise IllegalArgumentException("Input arguments not supported. args = %s" % args) # # """ # # Parameters: # String resourceName # """ # def __init__(self, resourceName): # super(resourceName) # # # """ # # Parameters: # ZNRecord record # """ # def __init__(self, record): # super(record) def getResourceName(self): """ Returns String """ return self._record.getId() def getPartitionStateMap(self): """ Returns Map<String, String> """ # Map<String, String> map = {} # Map<String, Map<String, String>> mapFields = self._record.getMapFields() for partitionName in mapFields.keys(): # Map<String, String> tempMap = mapFields.get(partitionName) if tempMap != None: map.__setitem__(partitionName, tempMap.get(CurrentStateProperty.toString(CurrentStateProperty.CURRENT_STATE))) return map def getSessionId(self): """ Returns String """ return self._record.getSimpleField(CurrentStateProperty.toString(CurrentStateProperty.SESSION_ID)) def setSessionId(self, sessionId): """ Returns void Parameters: sessionId: String """ self._record.setSimpleField(CurrentStateProperty.toString(CurrentStateProperty.SESSION_ID), sessionId) def getState(self, partitionName): """ Returns String Parameters: partitionName: String """ # Map<String, Map<String, String>> mapFields = self._record.getMapFields() # Map<String, String> mapField = mapFields.get(partitionName) if mapField != None: return mapField.get(CurrentStateProperty.toString(CurrentStateProperty.CURRENT_STATE)) return None def setStateModelDefRef(self, stateModelName): """ Returns void Parameters: stateModelName: String """ self._record.setSimpleField(CurrentStateProperty.toString(CurrentStateProperty.STATE_MODEL_DEF), stateModelName) def getStateModelDefRef(self): """ Returns String """ return self._record.getSimpleField(CurrentStateProperty.toString(CurrentStateProperty.STATE_MODEL_DEF)) def setState(self, partitionName, state): """ Returns void Parameters: partitionName: Stringstate: String """ # Map<String, Map<String, String>> mapFields = self._record.getMapFields() if mapFields.get(partitionName) == None: mapFields.__setitem__(partitionName, {}) # mapFields.put(partitionName, TreeMap<String, String>()) mapFields.get(partitionName).__setitem__(CurrentStateProperty.toString(CurrentStateProperty.CURRENT_STATE), state) def setStateModelFactoryName(self, factoryName): """ Returns void Parameters: factoryName: String """ self._record.setSimpleField(CurrentStateProperty.toString(CurrentStateProperty.STATE_MODEL_FACTORY_NAME), factoryName) def getStateModelFactoryName(self): """ Returns String """ return self._record.getSimpleField(CurrentStateProperty.toString(CurrentStateProperty.STATE_MODEL_FACTORY_NAME)) def isValid(self): """ Returns boolean @Override """ if self.getStateModelDefRef() == None: self.LOG.error("Current state does not contain state model ref. id:" + str(self.getResourceName())) return False if self.getSessionId() == None: self.LOG.error("CurrentState does not contain session id, id : " + str(self.getResourceName())) return False return True
class IdealState(HelixProperty): """ Java modifiers: final static Type: String """ QUERY_LIST = "PREFERENCE_LIST_QUERYS" """ Java modifiers: private final static Type: Logger """ logger = get_logger(__name__) def __init__(self, *args): super(IdealState, self).__init__(*args) # """ # # Parameters: # String resourceName # """ # def __init__(self, resourceName): # super(resourceName) # # # """ # # Parameters: # ZNRecord record # """ # def __init__(self, record): # super(record) def getResourceName(self): """ Returns String """ return self._record.getId() def setIdealStateMode(self, mode): """ Returns void Parameters: mode: String """ self._record.setSimpleField( IdealStateProperty.IDEAL_STATE_MODE.toString(), mode) def getIdealStateMode(self): """ Returns IdealStateModeProperty """ # String mode = self._record.getSimpleField( IdealStateProperty.IDEAL_STATE_MODE.toString()) try: return IdealStateModeProperty.valueOf(mode) except Exception, e: return IdealStateModeProperty.AUTO
class ZKUtil: """ Java modifiers: private static Type: Logger """ logger = get_logger(__name__) """ Java modifiers: private static Type: int """ RETRYLIMIT = 3 """ Java modifiers: private """ def __init__(self): pass @staticmethod def isClusterSetup(clusterName, zkClient): """ Returns boolean Parameters: clusterName: StringzkClient: ZkClient Java modifiers: static """ if clusterName == None or zkClient == None: return False # boolean isValid = zkClient.exists(PropertyPathConfig.getPath(PropertyType.IDEALSTATES, clusterName)) \ and zkClient.exists(PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.toString(ConfigScopeProperty.CLUSTER), clusterName)) \ and zkClient.exists(PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.toString(ConfigScopeProperty.PARTICIPANT))) \ and zkClient.exists(PropertyPathConfig.getPath(PropertyType.CONFIGS, clusterName, ConfigScopeProperty.toString(ConfigScopeProperty.RESOURCE))) \ and zkClient.exists(PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, clusterName)) \ and zkClient.exists(PropertyPathConfig.getPath(PropertyType.LIVEINSTANCES, clusterName)) \ and zkClient.exists(PropertyPathConfig.getPath(PropertyType.INSTANCES, clusterName)) \ and zkClient.exists(PropertyPathConfig.getPath(PropertyType.EXTERNALVIEW, clusterName)) \ and zkClient.exists(PropertyPathConfig.getPath(PropertyType.CONTROLLER, clusterName)) \ and zkClient.exists(PropertyPathConfig.getPath(PropertyType.STATEMODELDEFS, clusterName)) \ and zkClient.exists(PropertyPathConfig.getPath(PropertyType.MESSAGES_CONTROLLER, clusterName)) \ and zkClient.exists(PropertyPathConfig.getPath(PropertyType.ERRORS_CONTROLLER, clusterName)) \ and zkClient.exists(PropertyPathConfig.getPath(PropertyType.STATUSUPDATES_CONTROLLER, clusterName)) \ and zkClient.exists(PropertyPathConfig.getPath(PropertyType.HISTORY, clusterName)) return isValid @staticmethod def createOrUpdate(client, path, record, persistent, mergeOnUpdate): """ Returns void Parameters: client: ZkClientpath: Stringrecord: ZNRecordpersistent: booleanmergeOnUpdate: boolean Java modifiers: static """ # int retryCount = 0 while retryCount < ZKUtil.RETRYLIMIT: try: if not client.exists(path): client.createPersistent(path, True) stat = HelixZNodeStat() currentRecord = client.readDataAndStat(path, stat, True) if currentRecord is not None and mergeOnUpdate: currentRecord.merge(record) client.writeData(path, currentRecord, stat.getVersion()) break except Exception as e: retryCount = retryCount + 1 ZKUtil.logger.warn("Exception trying to update " + path + " Exception:" + str(e) + ". Will retry.")
class DefaultMessagingService(ClusterMessagingService): """ Java modifiers: private static Type: Logger """ _logger = get_logger(__name__) """ Parameters: HelixManager manager """ def __init__(self, manager): self._manager = manager self._evaluator = CriteriaEvaluator() self._taskExecutor = HelixTaskExecutor() self._asyncCallbackService = AsyncCallbackService() self._taskExecutor.registerMessageHandlerFactory( MessageType.toString(MessageType.TASK_REPLY), self._asyncCallbackService) self._messageHandlerFactoriestobeAdded = { } #ConcurrentHashMap<String, MessageHandlerFactory> # def send(self, recipientCriteria, messageTemplate): # """ # Returns int # Parameters: # recipientCriteria: CriteriamessageTemplate: Message # @Override # # # """ # return self.send(recipientCriteria, messageTemplate, None, -1) # # # def send(self, recipientCriteria, message, callbackOnReply, timeOut): # """ # Returns int # Parameters: # recipientCriteria: Criteriamessage: MessagecallbackOnReply: AsyncCallbacktimeOut: int # @Override # # # """ # return self.send(recipientCriteria, message, callbackOnReply, timeOut, 0) # # def send(self, recipientCriteria, message, callbackOnReply=None, timeOut=-1, retryCount=0): """ Returns int Parameters: recipientCriteria: Criteriamessage: MessagecallbackOnReply: AsyncCallbacktimeOut: intretryCount: int @Override """ # Map<InstanceType, List<Message>> generateMessage = self.generateMessage(recipientCriteria, message) # int totalMessageCount = 0 for messages in generateMessage.values(): totalMessageCount += messages.size() self._logger.info("Send " + totalMessageCount + " messages with criteria " + recipientCriteria) if totalMessageCount == 0: return 0 # String correlationId = None if callbackOnReply != None: # int totalTimeout = timeOut * (retryCount + 1) if totalTimeout < 0: totalTimeout = -1 callbackOnReply.setTimeout(totalTimeout) # correlationId = UUID.randomUUID().toString() correlationId = str(uuid.uuid4()) for messages in generateMessage.values(): callbackOnReply.setMessagesSent(messages) self._asyncCallbackService.registerAsyncCallback( correlationId, callbackOnReply) for receiverType in generateMessage.keys(): # List<Message> list = generateMessage.get(receiverType) for tempMessage in list: tempMessage.setRetryCount(retryCount) tempMessage.setExecutionTimeout(timeOut) tempMessage.setSrcInstanceType(self._manager.getInstanceType()) if correlationId != None: tempMessage.setCorrelationId(correlationId) # HelixDataAccessor accessor = self._manager.getHelixDataAccessor() # Builder keyBuilder = accessor.keyBuilder() if receiverType == InstanceType.CONTROLLER: accessor.setProperty( keyBuilder.controllerMessage(tempMessage.getId()), tempMessage) if receiverType == InstanceType.PARTICIPANT: accessor.setProperty( keyBuilder.message(tempMessage.getTgtName(), tempMessage.getId()), tempMessage) if callbackOnReply != None: callbackOnReply.startTimer() return totalMessageCount def generateMessage(self, recipientCriteria, message): """ Returns Map<InstanceType, List<Message>> Parameters: recipientCriteria: Criteriamessage: Message Java modifiers: private """ # Map<InstanceType, List<Message>> messagesToSendMap = {} # messagesToSendMap = HashMap<InstanceType, List<Message>>() # InstanceType instanceType = recipientCriteria.getRecipientInstanceType() if instanceType == InstanceType.CONTROLLER: # List<Message> messages = self.generateMessagesForController(message) messagesToSendMap.put(InstanceType.CONTROLLER, messages) else: if instanceType == InstanceType.PARTICIPANT: # List<Message> messages = [] # messages = ArrayList<Message>() # List<Map<String, String>> matchedList = self._evaluator.evaluateCriteria( recipientCriteria, self._manager) if not matchedList.isEmpty(): # Map<String, String> sessionIdMap = {} # sessionIdMap = HashMap<String, String>() if recipientCriteria.isSessionSpecific(): # HelixDataAccessor accessor = self._manager.getHelixDataAccessor() # Builder keyBuilder = accessor.keyBuilder() # List<LiveInstance> liveInstances = accessor.getChildValues( keyBuilder.liveInstances()) for liveInstance in liveInstances: sessionIdMap.__setitem__( liveInstance.getInstanceName(), liveInstance.getSessionId()) for map in matchedList: id = str(uuid.uuid4()) # Message newMessage = Message(message.getRecord(), id) # String srcInstanceName = self._manager.getInstanceName() # String tgtInstanceName = map.get("instanceName") if recipientCriteria.isSelfExcluded( ) and srcInstanceName.upper() == tgtInstanceName.upper( ): continue newMessage.setSrcName(srcInstanceName) newMessage.setTgtName(tgtInstanceName) newMessage.setResourceName(map.get("resourceName")) newMessage.setPartitionName(map.get("partitionName")) if recipientCriteria.isSessionSpecific(): newMessage.setTgtSessionId( sessionIdMap.get(tgtInstanceName)) messages.append(newMessage) messagesToSendMap.__setitem__(InstanceType.PARTICIPANT, messages) return messagesToSendMap def generateMessagesForController(self, message): """ Returns List<Message> Parameters: message: Message Java modifiers: private """ # List<Message> # messages = ArrayList<Message>() messages = [] # String id = str(uuid.uuid4()) # Message newMessage = Message(message.getRecord(), id) newMessage.setMsgId(id) newMessage.setSrcName(self._manager.getInstanceName()) newMessage.setTgtName("Controller") messages.add(newMessage) return messages def registerMessageHandlerFactory(self, type, factory): """ Returns void Parameters: type: Stringfactory: MessageHandlerFactory @Override Java modifiers: synchronized """ if self._manager.isConnected(): self.registerMessageHandlerFactoryInternal(type, factory) else: self._messageHandlerFactoriestobeAdded.put(type, factory) def onConnected(self): """ Returns void Java modifiers: synchronized """ for type in self._messageHandlerFactoriestobeAdded.keys(): self.registerMessageHandlerFactoryInternal( type, self._messageHandlerFactoriestobeAdded.get(type)) self._messageHandlerFactoriestobeAdded.clear() def registerMessageHandlerFactoryInternal(self, type, factory): """ Returns void Parameters: type: Stringfactory: MessageHandlerFactory """ self._logger.info("registering msg factory for type " + type) # int threadpoolSize = HelixTaskExecutor.DEFAULT_PARALLEL_TASKS # String threadpoolSizeStr = None # String key = type + "." + HelixTaskExecutor.MAX_THREADS # ConfigAccessor configAccessor = self._manager.getConfigAccessor() if configAccessor != None: # ConfigScope scope = None if self._manager.getInstanceType( ) == InstanceType.PARTICIPANT or self._manager.getInstanceType( ) == InstanceType.CONTROLLER_PARTICIPANT: scope = ConfigScopeBuilder().forCluster( self._manager.getClusterName()).forParticipant( self._manager.getInstanceName()).build() threadpoolSizeStr = configAccessor.get(scope, key) if threadpoolSizeStr == None: scope = ConfigScopeBuilder().forCluster( self._manager.getClusterName()).build() threadpoolSizeStr = configAccessor.get(scope, key) if threadpoolSizeStr != None: try: threadpoolSize = int(threadpoolSizeStr) if threadpoolSize <= 0: threadpoolSize = 1 except Exception, e: self._logger.error("" + str(e)) self._taskExecutor.registerMessageHandlerFactory( type, factory, threadpoolSize) self.sendNopMessage()