def stormDump(dmonEndpoint, qgte, qlte, qsize, qinterval):
'''
:param dmonEndpoint: -> DMON endpoint
:param qgte: -> greater than timestamp
:param qlte: -> less than timestamp
:param qsize: -> query size
:param qinterval: -> query interval
:return:
'''
dmonConnector = Connector(dmonEndpoint)
stormTopology = dmonConnector.getStormTopology()
bolts = stormTopology['bolts']
spouts = stormTopology['spouts']
print "Detected %s bolts" % str(bolts)
print "Detected %s spouts" % str(spouts)
qConstructor = QueryConstructor()
dformat = DataFormatter(dataDir)
storm, storm_file = qConstructor.stormString()
print "Query string -> %s" % storm
qstorm = qConstructor.stormQuery(storm, qgte, qlte, qsize, qinterval, bolts=bolts, spouts=spouts)
print "Query -> %s" % qstorm
gstorm = dmonConnector.aggQuery(qstorm)
print "Response:"
print gstorm
dformat.dict2csv(gstorm, qstorm, storm_file)
def cassandraDump(dmonEndpoint, qgte, qlte, qsize, qinterval):
'''
:param dmonEndpoint: -> DMON endpoint
:param qgte: -> greater than timestamp
:param qlte: -> less than timestamp
:param qsize: -> query size
:param qinterval: -> query interval
:return:
'''
dmonConnector = Connector(dmonEndpoint)
qConstructor = QueryConstructor()
dformat = DataFormatter(dataDir)
roles = dmonConnector.roles()
cassandra_hosts = []
for el in roles['Nodes']:
for k, v in el.iteritems():
if 'cassandra' in v:
print 'Found cassandra role at %s' % k
cassandra_hosts.append(k)
for host in cassandra_hosts:
print "-" * 50
cassandra, cassandra_file = qConstructor.cassandraCounterString(host=host)
print "Query string -> %s" % cassandra
qcassandra = qConstructor.cassandraQuery(cassandra, qgte, qlte, qsize, qinterval)
print "Query -> %s" % qcassandra
gcassandra = dmonConnector.aggQuery(qcassandra)
print "Response:"
print gcassandra
dformat.dict2csv(gcassandra, qcassandra, cassandra_file)
cassandragauge, cassandragauge_file = qConstructor.cassandraGaugeString(host=host)
qcassandragauge = qConstructor.cassandraQuery(cassandragauge, qgte, qlte, qsize, qinterval)
print "Query -> %s" % qcassandragauge
gcassandragauge = dmonConnector.aggQuery(qcassandragauge)
print "Response:"
print gcassandragauge
dformat.dict2csv(gcassandragauge, qcassandragauge, cassandragauge_file)
print "-" * 50
# Queries
qload = qConstructor.systemLoadQuery(load, qgte, qlte, qsize, qinterval)
qmemory = qConstructor.systemMemoryQuery(memory, qgte, qlte, qsize, qinterval)
qinterface = qConstructor.systemInterfaceQuery(interface, qgte, qlte, qsize, qinterval)
qpacket = qConstructor.systemInterfaceQuery(packet, qgte, qlte, qsize, qinterval)
qnodeManager = qConstructor.yarnNodeManager(nodeManager, qgte, qlte, qsize, qinterval)
qjvmNodeManager = qConstructor.jvmNNquery(jvmNodeManager, qgte, qlte, qsize, qinterval)
#qjvmMapTask = qConstructor.jvmNNquery(jvmMapTask, qgte, qlte, qsize, qinterval)
qdatanode = qConstructor.datanodeMetricsQuery(datanode, qgte, qlte, qsize, qinterval)
qshuffle = qConstructor.shuffleQuery(shuffle, qgte, qlte, qsize, qinterval)
# Execute query and convert response to csv
qloadResponse = dmonConnector.aggQuery(qload)
dformat.dict2csv(qloadResponse, qload, load_file)
gmemoryResponse = dmonConnector.aggQuery(qmemory)
#print gmemoryResponse
dformat.dict2csv(gmemoryResponse, qmemory, memory_file)
ginterfaceResponse = dmonConnector.aggQuery(qinterface)
dformat.dict2csv(ginterfaceResponse, qinterface, interface_file)
gpacketResponse = dmonConnector.aggQuery(qpacket)
dformat.dict2csv(gpacketResponse, qpacket, packet_file)
gnodeManagerResponse = dmonConnector.aggQuery(qnodeManager)
if gnodeManagerResponse['aggregations'].values()[0].values()[0]:
dformat.dict2csv(gnodeManagerResponse, qnodeManager, nodeManager_file)
else:
qsize, qinterval)
qpacket = qConstructor.systemInterfaceQuery(packet, qgte, qlte, qsize,
qinterval)
qnodeManager = qConstructor.yarnNodeManager(nodeManager, qgte, qlte,
qsize, qinterval)
qjvmNodeManager = qConstructor.jvmNNquery(jvmNodeManager, qgte, qlte,
qsize, qinterval)
#qjvmMapTask = qConstructor.jvmNNquery(jvmMapTask, qgte, qlte, qsize, qinterval)
qdatanode = qConstructor.datanodeMetricsQuery(datanode, qgte, qlte,
qsize, qinterval)
qshuffle = qConstructor.shuffleQuery(shuffle, qgte, qlte, qsize,
qinterval)
# Execute query and convert response to csv
qloadResponse = dmonConnector.aggQuery(qload)
dformat.dict2csv(qloadResponse, qload, load_file)
gmemoryResponse = dmonConnector.aggQuery(qmemory)
#print gmemoryResponse
dformat.dict2csv(gmemoryResponse, qmemory, memory_file)
ginterfaceResponse = dmonConnector.aggQuery(qinterface)
dformat.dict2csv(ginterfaceResponse, qinterface, interface_file)
gpacketResponse = dmonConnector.aggQuery(qpacket)
dformat.dict2csv(gpacketResponse, qpacket, packet_file)
gnodeManagerResponse = dmonConnector.aggQuery(qnodeManager)
if list(gnodeManagerResponse['aggregations'].values())[0].values()[0]:
dformat.dict2csv(gnodeManagerResponse, qnodeManager,
nodeManager_file)
