def __init__(self, selfNodeAddr, otherNodeAddrs,
testType = TEST_TYPE.DEFAULT,
compactionMinEntries = 0,
dumpFile = None,
journalFile = None,
password = None,
dynamicMembershipChange = False,
useFork = True,
testBindAddr = False):
cfg = SyncObjConf(autoTick=False, appendEntriesUseBatch=False)
cfg.appendEntriesPeriod = 0.1
cfg.raftMinTimeout = 0.5
cfg.raftMaxTimeout = 1.0
cfg.dynamicMembershipChange = dynamicMembershipChange
if testBindAddr:
cfg.bindAddress = selfNodeAddr
if dumpFile is not None:
cfg.fullDumpFile = dumpFile
if password is not None:
cfg.password = password
cfg.useFork = useFork
if testType == TEST_TYPE.COMPACTION_1:
cfg.logCompactionMinEntries = compactionMinEntries
cfg.logCompactionMinTime = 0.1
cfg.appendEntriesUseBatch = True
if testType == TEST_TYPE.COMPACTION_2:
cfg.logCompactionMinEntries = 99999
cfg.logCompactionMinTime = 99999
cfg.fullDumpFile = dumpFile
if testType == TEST_TYPE.RAND_1:
cfg.autoTickPeriod = 0.05
cfg.appendEntriesPeriod = 0.02
cfg.raftMinTimeout = 0.1
cfg.raftMaxTimeout = 0.2
cfg.logCompactionMinTime = 9999999
cfg.logCompactionMinEntries = 9999999
cfg.journalFile = journalFile
if testType == TEST_TYPE.JOURNAL_1:
cfg.logCompactionMinTime = 999999
cfg.logCompactionMinEntries = 999999
cfg.fullDumpFile = dumpFile
cfg.journalFile = journalFile
if testType == TEST_TYPE.AUTO_TICK_1:
cfg.autoTick = True
cfg.pollerType = 'select'
if testType == TEST_TYPE.WAIT_BIND:
cfg.maxBindRetries = 1
cfg.autoTick = True
super(TestObj, self).__init__(selfNodeAddr, otherNodeAddrs, cfg)
self.__counter = 0
self.__data = {}
def __init__(self, selfNodeAddr, otherNodeAddrs,
compactionTest = 0,
dumpFile = None,
compactionTest2 = False,
password = None):
cfg = SyncObjConf(autoTick=False, commandsQueueSize=10000, appendEntriesUseBatch=False)
if compactionTest:
cfg.logCompactionMinEntries = compactionTest
cfg.logCompactionMinTime = 0.1
cfg.appendEntriesUseBatch = True
cfg.fullDumpFile = dumpFile
if compactionTest2:
cfg.logCompactionMinEntries = 99999
cfg.logCompactionMinTime = 99999
cfg.fullDumpFile = dumpFile
cfg.sendBufferSize = 2 ** 21
cfg.recvBufferSize = 2 ** 21
cfg.appendEntriesBatchSize = 10
cfg.maxCommandsPerTick = 5
if password is not None:
cfg.password = password
super(TestObj, self).__init__(selfNodeAddr, otherNodeAddrs, cfg)
self.__counter = 0
self.__data = {}
def __init__(self, selfAddr, otherAddrs, consumers): cfg = SyncObjConf(autoTick=False) super(ZeroDeployTestObj, self).__init__(selfAddr, otherAddrs, cfg, consumers=consumers)
def __init__(self, selfNodeAddr, otherNodeAddrs,
testType = TEST_TYPE.DEFAULT,
compactionMinEntries = 0,
dumpFile = None,
password = None,
dynamicMembershipChange = False):
cfg = SyncObjConf(autoTick=False, appendEntriesUseBatch=False)
cfg.appendEntriesPeriod = 0.1
cfg.raftMinTimeout = 0.5
cfg.raftMaxTimeout = 1.0
cfg.dynamicMembershipChange = dynamicMembershipChange
if dumpFile is not None:
cfg.fullDumpFile = dumpFile
if password is not None:
cfg.password = password
if testType == TEST_TYPE.COMPACTION_1:
cfg.logCompactionMinEntries = compactionMinEntries
cfg.logCompactionMinTime = 0.1
cfg.appendEntriesUseBatch = True
if testType == TEST_TYPE.COMPACTION_2:
cfg.logCompactionMinEntries = 99999
cfg.logCompactionMinTime = 99999
cfg.fullDumpFile = dumpFile
if testType == TEST_TYPE.RAND_1:
cfg.autoTickPeriod = 0.05
cfg.appendEntriesPeriod = 0.02
cfg.raftMinTimeout = 0.1
cfg.raftMaxTimeout = 0.2
cfg.logCompactionMinTime = 9999999
cfg.logCompactionMinEntries = 9999999
super(TestObj, self).__init__(selfNodeAddr, otherNodeAddrs, cfg)
self.__counter = 0
self.__data = {}
def __init__(self, selfNodeAddr, otherNodeAddrs,
compactionTest = 0,
dumpFile = None,
compactionTest2 = False,
password = None,
randTest = False):
cfg = SyncObjConf(autoTick=False, appendEntriesUseBatch=False)
cfg.appendEntriesPeriod = 0.1
cfg.raftMinTimeout = 0.5
cfg.raftMaxTimeout = 1.0
if compactionTest:
cfg.logCompactionMinEntries = compactionTest
cfg.logCompactionMinTime = 0.1
cfg.appendEntriesUseBatch = True
cfg.fullDumpFile = dumpFile
if compactionTest2:
cfg.logCompactionMinEntries = 99999
cfg.logCompactionMinTime = 99999
cfg.fullDumpFile = dumpFile
if password is not None:
cfg.password = password
if randTest:
cfg.autoTickPeriod = 0.05
cfg.appendEntriesPeriod = 0.02
cfg.raftMinTimeout = 0.1
cfg.raftMaxTimeout = 0.2
cfg.logCompactionMinTime = 9999999
cfg.logCompactionMinEntries = 9999999
super(TestObj, self).__init__(selfNodeAddr, otherNodeAddrs, cfg)
self.__counter = 0
self.__data = {}
def setUp(self):
self.conf = SyncObjConf(appendEntriesUseBatch=False,
dynamicMembershipChange=True,
autoTick=False)
self.so = DynMemberSyncObj('127.0.0.1:1234', ['127.0.0.1:1235'],
self.conf)
def __init__(self, my_addr, peers):
cfg = SyncObjConf(dynamicMembershipChange=True)
super().__init__(my_addr, peers, cfg)
self.__data = []
def __init__(self, port, members, secret=""):
JSBASE.__init__(self)
self._members = members
self.port = port
self.dict1 = ReplDict()
remotes = ["%s:%s" % item for item in self.members]
cfg = SyncObjConf(autoTick=True)
cfg.onReady = self.onReady
if secret is not "" and secret is not None:
print("SECRET")
cfg.password = secret
cfg.appendEntriesPeriod = 0.01
cfg.appendEntriesUseBatch = True
cfg.raftMinTimeout = 0.4
cfg.raftMaxTimeout = 1.4
cfg.dynamicMembershipChange = True
cfg.onStateChanged = None
cfg.commandsWaitLeader = False
cfg.connectionRetryTime = 5.0 # connect to other down nodes every so many secs
cfg.connectionTimeout = 3.5
cfg.leaderFallbackTimeout = 10.0
cfg.journalFile = "/tmp/raft/raft_%s" % self.port
cfg.leaderFallbackTimeout = True
cfg.logCompactionMinEntries = 1000
cfg.logCompactionMinTime = 60
self._log_debug("port:%s" % self.port)
self._log_debug("members:%s" % remotes)
# self._log_debug("secret:%s"%secret)
self.syncobj = SyncObj("localhost:%s" % port,
remotes,
consumers=[self.dict1],
conf=cfg)
# for i in range(100000000):
# time.sleep(0.001)
# # self.syncobj.doTick()
# from IPython import embed;embed(colors='Linux')
# s
while self.syncobj.isReady() == False:
time.sleep(1)
print("wait sync")
time.sleep(1)
self.start()
_g_success += 1
callTime, recvTime = res
delay = time.time() - callTime
_g_delays.append(delay)
else:
_g_error += 1
_g_errors[err] += 1
def getRandStr(l):
f = '%0' + str(l) + 'x'
return f % random.randrange(16 ** l)
if __name__ == '__main__':
# Parse parameters
numCommands, cmdSize, quorumSize1, quorumSize2, drop_ratio, selfAddr, partners = parseParams(sys.argv)
maxCommandsQueueSize = int(0.9 * SyncObjConf().commandsQueueSize / len(partners))
# Init a TestObj
obj = TestObj(selfAddr, partners, quorumSize1, quorumSize2, drop_ratio)
while obj._getLeader() is None:
time.sleep(0.5)
time.sleep(4.0)
startTime = time.time()
while time.time() - startTime < 25.0:
st = time.time()
for i in range(0, numCommands):
obj.testMethod(getRandStr(cmdSize), time.time(), callback=clbck)
def __init__(self, selfNodeAddr, otherNodeAddrs, quorumSize1=0, quorumSize2=0, drop_ratio=0.0):
cfg = SyncObjConf(
appendEntriesUseBatch=False,
)
super(TestObj, self).__init__(selfNodeAddr, otherNodeAddrs, quorumSize1, quorumSize2, drop_ratio, cfg)
self.__appliedCommands = 0
def start_manager(host='localhost',
port=7070,
peer_addr=None,
snapshot_file='/tmp/cockatrice/management.zip',
log_compaction_min_entries=5000,
log_compaction_min_time=300,
data_dir='/tmp/cockatrice/management',
grpc_port=5050,
grpc_max_workers=10,
http_port=8080,
log_level='DEBUG',
log_file=None,
log_file_max_bytes=512000000,
log_file_backup_count=5,
http_log_file=None,
http_log_file_max_bytes=512000000,
http_log_file_backup_count=5):
# create logger and handler
logger = getLogger(NAME)
log_handler = StreamHandler()
# determine log destination
if log_file is not None:
log_handler = RotatingFileHandler(log_file,
'a+',
maxBytes=log_file_max_bytes,
backupCount=log_file_backup_count)
# determine log level
if log_level in ['CRITICAL', 'FATAL']:
logger.setLevel(CRITICAL)
log_handler.setLevel(CRITICAL)
elif log_level == 'ERROR':
logger.setLevel(ERROR)
log_handler.setLevel(ERROR)
elif log_level in ['WARNING', 'WARN']:
logger.setLevel(WARNING)
log_handler.setLevel(WARNING)
elif log_level == 'INFO':
logger.setLevel(INFO)
log_handler.setLevel(INFO)
elif log_level == 'DEBUG':
logger.setLevel(DEBUG)
log_handler.setLevel(DEBUG)
elif log_level == 'NOTSET':
logger.setLevel(NOTSET)
log_handler.setLevel(NOTSET)
else:
logger.setLevel(INFO)
log_handler.setLevel(INFO)
# set log format
handler_format = Formatter(
'%(asctime)s - %(levelname)s - %(pathname)s:%(lineno)d - %(message)s')
log_handler.setFormatter(handler_format)
# add log handler
logger.addHandler(log_handler)
# create http logger and handler
http_logger = getLogger(NAME + '_http')
http_log_handler = StreamHandler()
# determine http log destination
if http_log_file is not None:
http_log_handler = RotatingFileHandler(
http_log_file,
'a+',
maxBytes=http_log_file_max_bytes,
backupCount=http_log_file_backup_count)
# determine http log level
http_logger.setLevel(INFO)
http_log_handler.setLevel(INFO)
# set http log format
http_handler_format = Formatter('%(message)s')
http_log_handler.setFormatter(http_handler_format)
# add http log handler
http_logger.addHandler(http_log_handler)
# metrics registry
metrics_registry = CollectorRegistry()
# sync config
os.makedirs(os.path.dirname(snapshot_file), exist_ok=True)
conf = SyncObjConf()
conf.fullDumpFile = snapshot_file
conf.logCompactionMinEntries = log_compaction_min_entries
conf.logCompactionMinTime = log_compaction_min_time
conf.dynamicMembershipChange = True
conf.validate()
supervisor = None
try:
supervisor = Manager(host=host,
port=port,
seed_addr=peer_addr,
conf=conf,
data_dir=data_dir,
grpc_port=grpc_port,
grpc_max_workers=grpc_max_workers,
http_port=http_port,
logger=logger,
http_logger=http_logger,
metrics_registry=metrics_registry)
while True:
signal.pause()
except Exception as ex:
print(ex)
finally:
if supervisor is not None:
supervisor.stop()
def __init__(self, selfAddress, partnerAddrs):
cfg = SyncObjConf(dynamicMembershipChange = True)
super(KVStorage, self).__init__(selfAddress, partnerAddrs, cfg)
self.__data = {}
def __init__(self, selfNodeAddr, otherNodeAddrs,
compactionTest = 0,
dumpFile = None,
compactionTest2 = False):
cfg = SyncObjConf(autoTick=False, commandsQueueSize=10000, appendEntriesUseBatch=False)
if compactionTest:
cfg.logCompactionMinEntries = compactionTest
cfg.logCompactionMinTime = 0.1
cfg.appendEntriesUseBatch = True
cfg.fullDumpFile = dumpFile
if compactionTest2:
cfg.logCompactionMinEntries = 99999
cfg.logCompactionMinTime = 99999
cfg.fullDumpFile = dumpFile
cfg.sendBufferSize = 2 ** 21
cfg.recvBufferSize = 2 ** 21
cfg.appendEntriesBatchSize = 10
cfg.maxCommandsPerTick = 5
super(TestObj, self).__init__(selfNodeAddr, otherNodeAddrs, cfg)
self.__counter = 0
self.__data = {}
def __init__(self, selfAddress, partnerAddress):
cfg = SyncObjConf(dynamicMembershipChange=True)
super(FTStack, self).__init__(selfAddress, partnerAddress, cfg)
self.data = {}
self.sid = {}
self.counter = -1
def __init__(self,
selfNodeAddr,
otherNodeAddrs,
testType=TEST_TYPE.DEFAULT,
compactionMinEntries=0,
dumpFile=None,
journalFile=None,
password=None,
dynamicMembershipChange=False,
useFork=True):
cfg = SyncObjConf(autoTick=False, appendEntriesUseBatch=False)
cfg.appendEntriesPeriod = 0.1
cfg.raftMinTimeout = 0.5
cfg.raftMaxTimeout = 1.0
cfg.dynamicMembershipChange = dynamicMembershipChange
if dumpFile is not None:
cfg.fullDumpFile = dumpFile
if password is not None:
cfg.password = password
cfg.useFork = useFork
if testType == TEST_TYPE.COMPACTION_1:
cfg.logCompactionMinEntries = compactionMinEntries
cfg.logCompactionMinTime = 0.1
cfg.appendEntriesUseBatch = True
if testType == TEST_TYPE.COMPACTION_2:
cfg.logCompactionMinEntries = 99999
cfg.logCompactionMinTime = 99999
cfg.fullDumpFile = dumpFile
if testType == TEST_TYPE.RAND_1:
cfg.autoTickPeriod = 0.05
cfg.appendEntriesPeriod = 0.02
cfg.raftMinTimeout = 0.1
cfg.raftMaxTimeout = 0.2
cfg.logCompactionMinTime = 9999999
cfg.logCompactionMinEntries = 9999999
cfg.journalFile = journalFile
if testType == TEST_TYPE.JOURNAL_1:
cfg.logCompactionMinTime = 999999
cfg.logCompactionMinEntries = 999999
cfg.fullDumpFile = dumpFile
cfg.journalFile = journalFile
if testType == TEST_TYPE.AUTO_TICK_1:
cfg.autoTick = True
cfg.pollerType = 'select'
super(TestObj, self).__init__(selfNodeAddr, otherNodeAddrs, cfg)
self.__counter = 0
self.__data = {}
