def _run(self, resource_manager: ResourceManager, on_finish,
sleep_during_loop):
cmd = " ".join(self.command_line())
if self.print_command_line:
print("Start {} with cmd: {}".format(self.id, cmd))
subprocess.Popen(cmd, shell=True)
while not resource_manager.is_application_finished(self.id):
if not self.is_running and resource_manager.is_application_running(
self.id):
self.is_running = True
time.sleep(sleep_during_loop)
print("Application {} has finished".format(self))
if callable(on_finish):
on_finish(self)
def _run(self, resource_manager: ResourceManager, on_finish, sleep_during_loop):
cmd = " ".join(self.command_line())
#if self.print_command_line:
print("Start {} with cmd: {}".format(self.id, cmd))
subprocess.Popen(cmd, shell=True)
self.start_at = datetime.datetime.utcnow()
time.sleep(sleep_during_loop + 30)
while not resource_manager.is_application_finished(self.id):
if not self.is_running and resource_manager.is_application_running(self.id):
self.is_running = True
time.sleep(sleep_during_loop)
print("Application {} has finished".format(self))
self.end_at = datetime.datetime.utcnow()
host_list = "|".join([address for address in self.nodes])
export_file_name = self.id + "_" + self.name
cmd_query_cpu = "\nmkdir /data/vinh.tran/new/expData/{}/{}" \
"&& influx -precision rfc3339 -username root -password root" \
" -database 'telegraf' -host 'localhost' -execute 'SELECT usage_user,usage_iowait " \
"FROM \"telegraf\".\"autogen\".\"cpu\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/ " \
"AND cpu = '\\''cpu-total'\\'' GROUP BY host' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/cpu_{}.csv" \
.format(self.experiment_name,
export_file_name,
self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
host_list,
self.experiment_name,
export_file_name,
self.name)
print(cmd_query_cpu)
# subprocess.Popen(cmd_query_cpu, shell=True)
cmd_query_cpu_mean = "\ninflux -precision rfc3339 -username root -password root" \
" -database 'telegraf' -host 'localhost' -execute 'SELECT mean(usage_user) as \"mean_cpu_percent\",mean(usage_iowait) as \"mean_io_wait\" " \
"FROM \"telegraf\".\"autogen\".\"cpu\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/ " \
"AND cpu = '\\''cpu-total'\\'' GROUP BY time(10s)' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/cpu_{}_mean.csv" \
.format(self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
host_list,
self.experiment_name,
export_file_name,
self.name)
print(cmd_query_cpu_mean)
cmd_query_mem = "\ninflux -precision rfc3339 -username root -password root " \
"-database 'telegraf' -host 'localhost' -execute 'SELECT used_percent " \
"FROM \"telegraf\".\"autogen\".\"mem\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/ " \
"GROUP BY host' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/mem_{}.csv" \
.format(self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
host_list,
self.experiment_name,
export_file_name,
self.name)
print(cmd_query_mem)
cmd_query_mem_mean = "\ninflux -precision rfc3339 -username root -password root " \
"-database 'telegraf' -host 'localhost' -execute 'SELECT mean(used_percent) " \
"FROM \"telegraf\".\"autogen\".\"mem\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/ " \
"GROUP BY time(10s)' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/mem_{}_mean.csv" \
.format(self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
host_list,
self.experiment_name,
export_file_name,
self.name)
print(cmd_query_mem_mean)
cmd_query_disk = "\ninflux -precision rfc3339 -username root -password root " \
"-database 'telegraf' -host 'localhost' -execute 'SELECT sum(read_bytes),sum(write_bytes) " \
"FROM (SELECT derivative(last(\"read_bytes\"),1s) as \"read_bytes\",derivative(last(\"write_bytes\"),1s) as \"write_bytes\",derivative(last(\"io_time\"),1s) as \"io_time\" " \
"FROM \"telegraf\".\"autogen\".\"diskio\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/ " \
"GROUP BY \"host\",\"name\",time(10s)) WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' GROUP BY host,time(10s)' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/disk_{}.csv" \
.format(self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
host_list,
self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
self.experiment_name,
export_file_name,
self.name)
print(cmd_query_disk)
cmd_query_disk_mean = "\ninflux -precision rfc3339 -username root -password root " \
"-database 'telegraf' -host 'localhost' -execute 'SELECT sum(read_bytes),sum(write_bytes) " \
"FROM (SELECT derivative(last(\"read_bytes\"),1s) as \"read_bytes\",derivative(last(\"write_bytes\"),1s) as \"write_bytes\",derivative(last(\"io_time\"),1s) as \"io_time\" " \
"FROM \"telegraf\".\"autogen\".\"diskio\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/ " \
"GROUP BY \"host\",\"name\",time(10s)) WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' GROUP BY time(10s)' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/disk_{}_mean.csv" \
.format(self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
host_list,
self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
self.experiment_name,
export_file_name,
self.name)
print(cmd_query_disk_mean)
cmd_query_net = "\ninflux -precision rfc3339 -username root -password root " \
"-database 'telegraf' -host 'localhost' -execute 'SELECT sum(download_bytes),sum(upload_bytes) FROM (SELECT derivative(first(\"bytes_recv\"),1s) " \
"as \"download_bytes\",derivative(first(\"bytes_sent\"),1s) as \"upload_bytes\"" \
"FROM \"telegraf\".\"autogen\".\"net\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/ " \
"GROUP BY \"host\",time(10s)) WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' GROUP BY host,time(10s)' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/net_{}.csv" \
.format(self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
host_list,
self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
self.experiment_name,
export_file_name,
self.name)
print(cmd_query_net)
cmd_query_net_mean = "\ninflux -precision rfc3339 -username root -password root " \
"-database 'telegraf' -host 'localhost' -execute 'SELECT sum(download_bytes),sum(upload_bytes) FROM (SELECT derivative(first(\"bytes_recv\"),1s) " \
"as \"download_bytes\",derivative(first(\"bytes_sent\"),1s) as \"upload_bytes\"" \
"FROM \"telegraf\".\"autogen\".\"net\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/ " \
"GROUP BY \"host\",time(10s)) WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' GROUP BY time(10s)' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/net_{}_mean.csv" \
.format(self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
host_list,
self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
self.experiment_name,
export_file_name,
self.name)
print(cmd_query_net_mean)
subprocess.Popen(cmd_query_cpu + " && " + cmd_query_mem + " && " + cmd_query_disk + " && " + cmd_query_net + " && "
+ cmd_query_cpu_mean + " && " + cmd_query_mem_mean + " && " + cmd_query_disk_mean + " && " + cmd_query_net_mean, shell=True)
time.sleep(1)
with open("/data/vinh.tran/new/expData/{}/{}/cmd_{}.txt".format(self.experiment_name, export_file_name, self.name), 'a') as file:
file.write("{}\n\n{}\n\n{}\n\n{}\n\n\n\n{}\n\n{}\n\n{}\n\n{}\n".
format(cmd_query_cpu, cmd_query_mem, cmd_query_disk, cmd_query_net,
cmd_query_cpu_mean, cmd_query_mem_mean, cmd_query_disk_mean, cmd_query_net_mean))
if callable(on_finish):
on_finish(self)
