ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(logformat)
fh.setFormatter(formatter)
ch.setFormatter(formatter)
logger.addHandler(fh)
logger.addHandler(ch)
SINGLE_THREAD_THROUGHPUT = 400
if filename != "" or isbatch == True:
SINGLE_THREAD_THROUGHPUT = 2000
numthreads = int(math.ceil(float(opspersecond) / SINGLE_THREAD_THROUGHPUT))
lastthreadthroughput = opspersecond % SINGLE_THREAD_THROUGHPUT
timeAccessGenerator = DistributionFactory.createSegmentDistribution(
accessdistribution)
periodAccessGenerator = DistributionFactory.createSegmentDistribution(
perioddistribution)
newquery = PyDruid(config.getBrokerNodeUrl(), config.getBrokerEndpoint())
tb = newquery.time_boundary(datasource=config.getDataSource())
startdict = tb[0]
start = startdict['result']['minTime']
start = datetime.strptime(start, '%Y-%m-%dT%H:%M:%S.%fZ')
start = utc.localize(start)
end = startdict['result']['maxTime']
end = datetime.strptime(end, '%Y-%m-%dT%H:%M:%S.%fZ')
end = utc.localize(end)
minqueryperiod = 0
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(logformat)
fh.setFormatter(formatter)
ch.setFormatter(formatter)
logger.addHandler(fh)
logger.addHandler(ch)
SINGLE_THREAD_THROUGHPUT = 400
if filename != "" or isbatch == True:
SINGLE_THREAD_THROUGHPUT = 2000
numthreads = int(math.ceil(float(opspersecond) / SINGLE_THREAD_THROUGHPUT))
lastthreadthroughput = opspersecond % SINGLE_THREAD_THROUGHPUT
timeAccessGenerator = DistributionFactory.createSegmentDistribution(accessdistribution)
periodAccessGenerator = DistributionFactory.createSegmentDistribution(perioddistribution)
newquery = PyDruid(config.getBrokerNodeUrl(), config.getBrokerEndpoint())
tb = newquery.time_boundary(datasource=config.getDataSource())
startdict = tb[0]
start = startdict['result']['minTime']
start = datetime.strptime(start, '%Y-%m-%dT%H:%M:%S.%fZ')
start = utc.localize(start)
end = startdict['result']['maxTime']
end = datetime.strptime(end, '%Y-%m-%dT%H:%M:%S.%fZ')
end = utc.localize(end)
minqueryperiod = 0
maxqueryperiod = int((end - start).total_seconds())
print("Segments per Interval: %s" % segmentsperinterval)
#### LOAD Phase ####
print "Pre loading segments and adding to deep storage"
segmentlist = RealTimeNode.generateSegments(preloadsegment)
deepstorage.extend(segmentlist)
segmentrunningcount = len(deepstorage)
RealTimeNode.printlist(segmentlist)
#### RUN Phase ####
for time in xrange(totaltime):
#Generating Queries
print "Generating Queries"
maxquerysize = min(segmentrunningcount, querymaxsize)
minquerysize = min(queryminsize, maxquerysize)
newquerylist = QueryGenerator.generateQueries(queryperinterval, segmentrunningcount, DistributionFactory.createSegmentDistribution(querysegmentdistribution), minquerysize, maxquerysize, DistributionFactory.createSizeDistribution(querysizedistribution));
Utils.printQueryList(newquerylist)
allquerylist.extend(newquerylist)
#Routing Queries
for strategy in dynamicstrategies:
strategy.routeQueries(newquerylist, segmentrunningcount, time)
if time % segmentinterval == 0:
#Generating Segments indexed starting from 1
print "Generating Segments and adding to deep storage"
newsegments = RealTimeNode.generateSegments(segmentsperinterval)
RealTimeNode.printlist(newsegments)
segmentlist.extend(newsegments)
deepstorage.extend(newsegments)
segmentrunningcount = len(deepstorage)
querysizedistribution = config.getQuerySizeDistribution() queryminsize = config.getQueryMinSize() querymaxsize = config.getQueryMaxSize() historicalnodecount = config.getHistoricalNodeCount() placementstrategy = config.getPlacementStrategy() replicationfactor = config.getReplicationFactor() percentreplicate = config.getPercentReplicate() #Generating Segments indexed starting from 1 print "Generating Segments" segmentlist = RealTimeNode.generateSegments(segmentcount) RealTimeNode.printlist(segmentlist) #Generating Queries print "Generating Queries" querylist = QueryGenerator.generateQueries(querycount, segmentcount, DistributionFactory.createSegmentDistribution(querysegmentdistribution), queryminsize, querymaxsize, DistributionFactory.createSizeDistribution(querysizedistribution)); printQueryList(querylist) ### DRUID COST BASED placementstrategy = "druidcostbased" for replicationfactor in xrange(2, 3): ### Random Routing Stretagy routingstrategy = "random" runExperiment(historicalnodecount, segmentlist, percentreplicate, replicationfactor, querylist, placementstrategy, routingstrategy) ### Connection Count Based Strategy routingstrategy = "chooseleastloaded" runExperiment(historicalnodecount, segmentlist, percentreplicate, replicationfactor, querylist, placementstrategy, routingstrategy) ### RANDOM BALL BASED
#### LOAD Phase ####
print "Pre loading segments and adding to deep storage"
segmentlist = RealTimeNode.generateSegments(preloadsegment)
deepstorage.extend(segmentlist)
segmentrunningcount = len(deepstorage)
RealTimeNode.printlist(segmentlist)
#### RUN Phase ####
for time in xrange(totaltime):
#Generating Queries
print "Generating Queries"
maxquerysize = min(segmentrunningcount, querymaxsize)
minquerysize = min(queryminsize, maxquerysize)
newquerylist = QueryGenerator.generateQueries(
queryperinterval, segmentrunningcount,
DistributionFactory.createSegmentDistribution(
querysegmentdistribution), minquerysize, maxquerysize,
DistributionFactory.createSizeDistribution(querysizedistribution))
Utils.printQueryList(newquerylist)
allquerylist.extend(newquerylist)
#Routing Queries
for strategy in dynamicstrategies:
strategy.routeQueries(newquerylist, segmentrunningcount, time)
if time % segmentinterval == 0:
#Generating Segments indexed starting from 1
print "Generating Segments and adding to deep storage"
newsegments = RealTimeNode.generateSegments(segmentsperinterval)
RealTimeNode.printlist(newsegments)
segmentlist.extend(newsegments)
deepstorage.extend(newsegments)
if time >= warmuptime:
#Generating Queries
print "Generating Queries"
if changequerydistribution == True and time == changedistributionat:
print "Distribution Change"
querysegmentdistribution=newquerysegmentdistribution
maxqueryperiod = min(time, querymaxsize)
minqueryperiod = min(queryminsize, maxqueryperiod)
numqueries = queryperinterval;
if burstyquery == True and time % burstyqueryinterval == 0:
print "Query Burst"
numqueries *= burstyquerymultiplier
newquerylist = QueryGenerator.generateQueries(time, numqueries, deepstorage, DistributionFactory.createSegmentDistribution(querysegmentdistribution), minqueryperiod, maxqueryperiod, DistributionFactory.createSizeDistribution(querysizedistribution));
Utils.printQueryList(newquerylist)
allquerylist.extend(newquerylist)
#Routing Queries
for strategy in dynamicstrategies:
strategy.routeQueries(newquerylist, segmentrunningcount, time)
#Placing Segments
if time % coordinatorinterval == 0:
for strategy in dynamicstrategies:
strategy.placeSegments(segmentlist, time, config)
segmentlist = []
#Print Statistics
for strategy in dynamicstrategies:
