def printresults():
logger.info('{} {} {} {}'.format(dataStartTime.strftime("%Y-%m-%d %H:%M:%S"), dataEndTime.strftime("%Y-%m-%d %H:%M:%S"), runTime, queryPerSec))
line = list()
querypermin = queryPerSec * 60
endtime = datetime.now(timezone('UTC')) + timedelta(minutes=runtime)
popularitylist = list()
while True:
tm.sleep(5)
time = datetime.now(timezone('UTC'))
logger.info("Time: {}".format(time.strftime("%Y-%m-%d %H:%M:%S")))
if time >= endtime:
break
#Query generated every minute. This is to optimize the overhead of query generation and also because segment granularity is minute
newquerylist = list()
if filename != "":
newquerylist = QueryGenerator.generateQueriesFromFile(dataStartTime, dataEndTime, querypermin, timeAccessGenerator, periodAccessGenerator, filename)
elif isbatch == True:
newquerylist = QueryGenerator.generateQueries(dataStartTime, dataEndTime, querypermin, timeAccessGenerator, periodAccessGenerator, popularitylist)
else:
#newquerylist = QueryGenerator.generateQueries(dataStartTime, time, querypermin, timeAccessGenerator, periodAccessGenerator, popularitylist)
newquerylist = QueryGenerator.generateQueries(dataStartTime, time, querypermin, timeAccessGenerator, periodAccessGenerator, popularitylist)
#print str(newquerylist)
for query in newquerylist:
print query.interval
try:
line.append(applyOperation(query, config, logger))
except Exception as inst:
logger.info(type(inst)) # the exception instance
logger.info(inst.args) # arguments stored in .args
logger.info(inst) # __str__ allows args to be printed directly
x, y = inst.args
logger.info('x =', x)
logger.info('y =', y)
nextminute = time + timedelta(minutes=1)
timediff = (nextminute - datetime.now(timezone('UTC'))).total_seconds()
if timediff > 0:
tm.sleep(timediff)
# yield gen.sleep(timediff)
wait_iterator = gen.WaitIterator(*line)
while not wait_iterator.done():
try:
result = yield wait_iterator.next()
except Exception as e:
logger.info("Error {} from {}".format(e, wait_iterator.current_future))
else:
logger.info("Result {} received from {} at {}".format(
result, wait_iterator.current_future,
wait_iterator.current_index))
def threadoperation(start, time, numqueries, timeAccessGenerator, minqueryperiod, maxqueryperiod, periodAccessGenerator, config, logger, x, values):
successfulquerytime = 0
successfulquerycount = 0
failedquerytime = 0
failedquerycount = 0
totalquerytime = 0
totalquerycount = 0
endtime = datetime.now() + timedelta(minutes=runtime)
while True:
if datetime.now() >= endtime:
break
time = datetime.now(timezone('UTC'))
newquerylist = QueryGenerator.generateQueries(start, time, numqueries, timeAccessGenerator, minqueryperiod, maxqueryperiod, periodAccessGenerator);
line = applyOperation(newquerylist[0], config,logger)
#print line[0:10]
if (line[0][0:10] == "Successful"):
print line[1].count
successfulquerytime += float(line[0][12:])
successfulquerycount += 1
totalquerytime += float(line[0][12:])
totalquerycount += 1
elif (line[0][0:6] == "Failed"):
failedquerytime += float(line[0][8:])
failedquerycount += 1
totalquerytime += float(line[0][8:])
totalquerycount += 1
datastructure = [successfulquerytime, successfulquerycount, failedquerytime, failedquerycount, totalquerytime, totalquerycount]
values.put(datastructure)
def printresults():
logger.info('{} {} {} {}'.format(start.strftime("%Y-%m-%d %H:%M:%S"),
end.strftime("%Y-%m-%d %H:%M:%S"),
runtime, queryPerSec))
line = list()
querypermin = queryPerSec * 60
endtime = datetime.now(timezone('UTC')) + timedelta(minutes=runtime)
popularitylist = list()
newquerylist = list()
if filename != "":
newquerylist = QueryGenerator.generateQueriesFromFile(
start, end, querypermin * runtime, timeAccessGenerator,
periodAccessGenerator, filename)
elif isbatch == True:
newquerylist = QueryGenerator.generateQueries(
start, end, querypermin * runtime, timeAccessGenerator,
periodAccessGenerator, popularitylist)
def printresults():
logger.log(
STATS, '{} {} {} {} {}'.format(start.strftime("%Y-%m-%d %H:%M:%S"),
end.strftime("%Y-%m-%d %H:%M:%S"),
runtime, queryPerSec, queryratio))
querypermin = queryPerSec * 60
endtime = datetime.now(timezone('UTC')) + timedelta(minutes=runtime)
line = list()
popularitylist = list()
newquerylist = list()
if filename != "":
newquerylist = QueryGenerator.generateQueriesFromFile(
start, end, querypermin * runtime, timeAccessGenerator,
periodAccessGenerator, querytype, queryratio, filename)
elif isbatch == True:
newquerylist = QueryGenerator.generateQueries(
start, end, querypermin * runtime, timeAccessGenerator,
periodAccessGenerator, popularitylist, querytype, queryratio,
logger)
else:
#logger.info("Run.py start queryendtime "+str(start)+", "+str(endtime))
queryStartInterval = start
queryEndInterval = start + timedelta(minutes=1)
for i in range(0, runtime):
logger.info("Start generating queries for interval " +
str(queryStartInterval) + " - " +
str(queryEndInterval))
newquerylist.extend(
QueryGenerator.generateQueries(
queryStartInterval, queryEndInterval, querypermin,
timeAccessGenerator, periodAccessGenerator,
popularitylist, querytype, queryratio, logger))
queryEndInterval = queryEndInterval + timedelta(minutes=1)
logger.info("Finished generating queries. num queries generated " +
str(len(newquerylist)))
if filename != "" or isbatch == True:
count = 0
time = datetime.now(timezone('UTC'))
logger.info("Time: {}".format(time.strftime("%Y-%m-%d %H:%M:%S")))
nextminute = time + timedelta(minutes=1)
for query in newquerylist:
try:
line.append(
applyOperation(query, config, brokernameurl, logger))
except Exception as inst:
logger.error(type(inst)) # the exception instance
logger.error(inst.args) # arguments stored in .args
logger.error(
inst) # __str__ allows args to be printed directly
x, y = inst.args
logger.error('x =', x)
logger.error('y =', y)
count = count + 1
if count >= querypermin:
timediff = (nextminute -
datetime.now(timezone('UTC'))).total_seconds()
if timediff > 0:
yield gen.sleep(timediff)
count = 0
time = datetime.now(timezone('UTC'))
logger.info("Time: {}".format(
time.strftime("%Y-%m-%d %H:%M:%S")))
nextminute = time + timedelta(minutes=1)
else:
# frequency of queries per millisecond
queryPerMilliSecond = float(queryPerSec) / 1000
# number of samples spaced by 1 millisecond
numSamples = runtime * 60 * 1000
numQueries, querySchedule = genPoissonQuerySchedule(
queryPerMilliSecond, numSamples)
logger.info("Poisson numQueries = " + str(numQueries))
queryScheduleIdx = 0
count = 0
while count < len(newquerylist):
sample = querySchedule[queryScheduleIdx]
#logger.info("Poisson sample is "+str(sample[0])+", "+str(sample[1]))
if (sample[0] == 0):
#logger.info("Sleeping for "+str(sample[1]))
yield gen.sleep(
float(sample[1]) /
1000) # divide by 1000 to convert it into seconds
else:
for i in range(0, sample[0]):
try:
line.append(
applyOperation(newquerylist[count], config,
brokernameurl, logger))
#applyOperation(newquerylist[count], config, brokernameurl, logger)
newquerylist[count].setTxTime(datetime.now())
#logger.info("Running query "+str(sample[0]))
except Exception as inst:
logger.error(type(inst)) # the exception instance
logger.error(
inst.args) # arguments stored in .args
logger.error(
inst
) # __str__ allows args to be printed directly
count = count + 1
if count >= len(newquerylist):
break
queryScheduleIdx = queryScheduleIdx + 1
wait_iterator = gen.WaitIterator(*line)
while not wait_iterator.done():
try:
result = yield wait_iterator.next()
except Exception as e:
logger.error("Error {} from {}".format(
e, wait_iterator.current_future))
def threadoperation(start, time, numqueries, timeAccessGenerator,
minqueryperiod, maxqueryperiod, periodAccessGenerator,
config, logger, x, values):
successfulquerytime = 0
successfulquerycount = 0
failedquerytime = 0
failedquerycount = 0
totalquerytime = 0
totalquerycount = 0
endtime = datetime.now() + dt.timedelta(minutes=runtime)
currentSegRank = []
#oldest_timestamp = start.total_seconds()
break_flag = 0
#print "16"
while True:
#while break_flag <1:
break_flag = break_flag + 1
#print "1"
if datetime.now() >= endtime:
break
time = datetime.now(timezone('UTC'))
#newquerylist = QueryGenerator.generateQueries(start, time, numqueries, timeAccessGenerator, minqueryperiod, maxqueryperiod, periodAccessGenerator);
if (len(currentSegRank) == 0):
#print "4"
y = time - start
z = y.total_seconds()
x = dt.timedelta(seconds=z)
distance = x.total_seconds()
for i in range(0, int(round(distance))):
timepoint = start + dt.timedelta(0, i)
currentSegRank.append(timepoint)
else:
#y = time.total_seconds()
#z = start.total_seconds()
#print "2"
new_interval = (time - start).total_seconds()
#print "time"
print time
#print "start"
print start
print new_interval
for i in range(0, int(round(new_interval))):
samples = randZipf(1 + len(currentSegRank), 1.2, 1)
timepoint = start + dt.timedelta(0, i)
print "add to rank: index"
print samples[0]
print "timepoint"
print timepoint
print "list seg size %d " % len(currentSegRank)
#len(currentSegRank)
currentSegRank.insert(samples[0], timepoint)
print currentSegRank
print filename
if (filename != ""):
#break_flag = 1
newquerylist = QueryGenerator.generateQueriesFromFile(
start, time, timeAccessGenerator, minqueryperiod,
maxqueryperiod, periodAccessGenerator, filename)
#elif(accessdistribution == "dynamiczip"):
#newquerylist = QueryGenerator.generateQueries(start, time, numqueries, timeAccessGenerator, minqueryperiod, maxqueryperiod, periodAccessGenerator, currentSegRank)
else:
#print "3"
newquerylist = QueryGenerator.generateQueries(
start, time, numqueries, timeAccessGenerator, minqueryperiod,
maxqueryperiod, periodAccessGenerator, currentSegRank)
line = applyOperation(newquerylist[0], config, logger)
print line[0:10]
if (line[0][0:10] == "Successful"):
print line[1].count
successfulquerytime += float(line[0][12:])
successfulquerycount += 1
totalquerytime += float(line[0][12:])
totalquerycount += 1
elif (line[0][0:6] == "Failed"):
failedquerytime += float(line[0][8:])
failedquerycount += 1
totalquerytime += float(line[0][8:])
totalquerycount += 1
datastructure = [
successfulquerytime, successfulquerycount, failedquerytime,
failedquerycount, totalquerytime, totalquerycount
]
values.put(datastructure)
print("Segments per Interval: %s" % segmentsperinterval)
#### LOAD Phase ####
print "Pre loading segments and adding to deep storage"
segmentlist = RealTimeNode.generateSegments(preloadsegment)
deepstorage.extend(segmentlist)
segmentrunningcount = len(deepstorage)
RealTimeNode.printlist(segmentlist)
#### RUN Phase ####
for time in xrange(totaltime):
#Generating Queries
print "Generating Queries"
maxquerysize = min(segmentrunningcount, querymaxsize)
minquerysize = min(queryminsize, maxquerysize)
newquerylist = QueryGenerator.generateQueries(queryperinterval, segmentrunningcount, DistributionFactory.createSegmentDistribution(querysegmentdistribution), minquerysize, maxquerysize, DistributionFactory.createSizeDistribution(querysizedistribution));
Utils.printQueryList(newquerylist)
allquerylist.extend(newquerylist)
#Routing Queries
for strategy in dynamicstrategies:
strategy.routeQueries(newquerylist, segmentrunningcount, time)
if time % segmentinterval == 0:
#Generating Segments indexed starting from 1
print "Generating Segments and adding to deep storage"
newsegments = RealTimeNode.generateSegments(segmentsperinterval)
RealTimeNode.printlist(newsegments)
segmentlist.extend(newsegments)
deepstorage.extend(newsegments)
segmentrunningcount = len(deepstorage)
querysizedistribution = config.getQuerySizeDistribution() queryminsize = config.getQueryMinSize() querymaxsize = config.getQueryMaxSize() historicalnodecount = config.getHistoricalNodeCount() placementstrategy = config.getPlacementStrategy() replicationfactor = config.getReplicationFactor() percentreplicate = config.getPercentReplicate() #Generating Segments indexed starting from 1 print "Generating Segments" segmentlist = RealTimeNode.generateSegments(segmentcount) RealTimeNode.printlist(segmentlist) #Generating Queries print "Generating Queries" querylist = QueryGenerator.generateQueries(querycount, segmentcount, DistributionFactory.createSegmentDistribution(querysegmentdistribution), queryminsize, querymaxsize, DistributionFactory.createSizeDistribution(querysizedistribution)); printQueryList(querylist) ### DRUID COST BASED placementstrategy = "druidcostbased" for replicationfactor in xrange(2, 3): ### Random Routing Stretagy routingstrategy = "random" runExperiment(historicalnodecount, segmentlist, percentreplicate, replicationfactor, querylist, placementstrategy, routingstrategy) ### Connection Count Based Strategy routingstrategy = "chooseleastloaded" runExperiment(historicalnodecount, segmentlist, percentreplicate, replicationfactor, querylist, placementstrategy, routingstrategy) ### RANDOM BALL BASED
#### LOAD Phase ####
print "Pre loading segments and adding to deep storage"
segmentlist = RealTimeNode.generateSegments(preloadsegment)
deepstorage.extend(segmentlist)
segmentrunningcount = len(deepstorage)
RealTimeNode.printlist(segmentlist)
#### RUN Phase ####
for time in xrange(totaltime):
#Generating Queries
print "Generating Queries"
maxquerysize = min(segmentrunningcount, querymaxsize)
minquerysize = min(queryminsize, maxquerysize)
newquerylist = QueryGenerator.generateQueries(
queryperinterval, segmentrunningcount,
DistributionFactory.createSegmentDistribution(
querysegmentdistribution), minquerysize, maxquerysize,
DistributionFactory.createSizeDistribution(querysizedistribution))
Utils.printQueryList(newquerylist)
allquerylist.extend(newquerylist)
#Routing Queries
for strategy in dynamicstrategies:
strategy.routeQueries(newquerylist, segmentrunningcount, time)
if time % segmentinterval == 0:
#Generating Segments indexed starting from 1
print "Generating Segments and adding to deep storage"
newsegments = RealTimeNode.generateSegments(segmentsperinterval)
RealTimeNode.printlist(newsegments)
segmentlist.extend(newsegments)
if time >= warmuptime:
#Generating Queries
print "Generating Queries"
if changequerydistribution == True and time == changedistributionat:
print "Distribution Change"
querysegmentdistribution=newquerysegmentdistribution
maxqueryperiod = min(time, querymaxsize)
minqueryperiod = min(queryminsize, maxqueryperiod)
numqueries = queryperinterval;
if burstyquery == True and time % burstyqueryinterval == 0:
print "Query Burst"
numqueries *= burstyquerymultiplier
newquerylist = QueryGenerator.generateQueries(time, numqueries, deepstorage, DistributionFactory.createSegmentDistribution(querysegmentdistribution), minqueryperiod, maxqueryperiod, DistributionFactory.createSizeDistribution(querysizedistribution));
Utils.printQueryList(newquerylist)
allquerylist.extend(newquerylist)
#Routing Queries
for strategy in dynamicstrategies:
strategy.routeQueries(newquerylist, segmentrunningcount, time)
#Placing Segments
if time % coordinatorinterval == 0:
for strategy in dynamicstrategies:
strategy.placeSegments(segmentlist, time, config)
segmentlist = []
#Print Statistics
for strategy in dynamicstrategies:
