def _run(self, resource_manager: ResourceManager, on_finish, sleep_during_loop): cmd = " ".join(self.command_line()) if self.print_command_line: print("Start {} with cmd: {}".format(self.id, cmd)) subprocess.Popen(cmd, shell=True) while not resource_manager.is_application_finished(self.id): if not self.is_running and resource_manager.is_application_running( self.id): self.is_running = True time.sleep(sleep_during_loop) print("Application {} has finished".format(self)) if callable(on_finish): on_finish(self)
def _run(self, resource_manager: ResourceManager, on_finish, sleep_during_loop): cmd = " ".join(self.command_line()) #if self.print_command_line: print("Start {} with cmd: {}".format(self.id, cmd)) subprocess.Popen(cmd, shell=True) self.start_at = datetime.datetime.utcnow() time.sleep(sleep_during_loop + 30) while not resource_manager.is_application_finished(self.id): if not self.is_running and resource_manager.is_application_running(self.id): self.is_running = True time.sleep(sleep_during_loop) print("Application {} has finished".format(self)) self.end_at = datetime.datetime.utcnow() host_list = "|".join([address for address in self.nodes]) export_file_name = self.id + "_" + self.name cmd_query_cpu = "\nmkdir /data/vinh.tran/new/expData/{}/{}" \ "&& influx -precision rfc3339 -username root -password root" \ " -database 'telegraf' -host 'localhost' -execute 'SELECT usage_user,usage_iowait " \ "FROM \"telegraf\".\"autogen\".\"cpu\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/ " \ "AND cpu = '\\''cpu-total'\\'' GROUP BY host' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/cpu_{}.csv" \ .format(self.experiment_name, export_file_name, self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'), self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'), host_list, self.experiment_name, export_file_name, self.name) print(cmd_query_cpu) # subprocess.Popen(cmd_query_cpu, shell=True) cmd_query_cpu_mean = "\ninflux -precision rfc3339 -username root -password root" \ " -database 'telegraf' -host 'localhost' -execute 'SELECT mean(usage_user) as \"mean_cpu_percent\",mean(usage_iowait) as \"mean_io_wait\" " \ "FROM \"telegraf\".\"autogen\".\"cpu\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/ " \ "AND cpu = '\\''cpu-total'\\'' GROUP BY time(10s)' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/cpu_{}_mean.csv" \ .format(self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'), self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'), host_list, self.experiment_name, export_file_name, self.name) print(cmd_query_cpu_mean) cmd_query_mem = "\ninflux -precision rfc3339 -username root -password root " \ "-database 'telegraf' -host 'localhost' -execute 'SELECT used_percent " \ "FROM \"telegraf\".\"autogen\".\"mem\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/ " \ "GROUP BY host' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/mem_{}.csv" \ .format(self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'), self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'), host_list, self.experiment_name, export_file_name, self.name) print(cmd_query_mem) cmd_query_mem_mean = "\ninflux -precision rfc3339 -username root -password root " \ "-database 'telegraf' -host 'localhost' -execute 'SELECT mean(used_percent) " \ "FROM \"telegraf\".\"autogen\".\"mem\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/ " \ "GROUP BY time(10s)' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/mem_{}_mean.csv" \ .format(self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'), self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'), host_list, self.experiment_name, export_file_name, self.name) print(cmd_query_mem_mean) cmd_query_disk = "\ninflux -precision rfc3339 -username root -password root " \ "-database 'telegraf' -host 'localhost' -execute 'SELECT sum(read_bytes),sum(write_bytes) " \ "FROM (SELECT derivative(last(\"read_bytes\"),1s) as \"read_bytes\",derivative(last(\"write_bytes\"),1s) as \"write_bytes\",derivative(last(\"io_time\"),1s) as \"io_time\" " \ "FROM \"telegraf\".\"autogen\".\"diskio\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/ " \ "GROUP BY \"host\",\"name\",time(10s)) WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' GROUP BY host,time(10s)' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/disk_{}.csv" \ .format(self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'), self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'), host_list, self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'), self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'), self.experiment_name, export_file_name, self.name) print(cmd_query_disk) cmd_query_disk_mean = "\ninflux -precision rfc3339 -username root -password root " \ "-database 'telegraf' -host 'localhost' -execute 'SELECT sum(read_bytes),sum(write_bytes) " \ "FROM (SELECT derivative(last(\"read_bytes\"),1s) as \"read_bytes\",derivative(last(\"write_bytes\"),1s) as \"write_bytes\",derivative(last(\"io_time\"),1s) as \"io_time\" " \ "FROM \"telegraf\".\"autogen\".\"diskio\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/ " \ "GROUP BY \"host\",\"name\",time(10s)) WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' GROUP BY time(10s)' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/disk_{}_mean.csv" \ .format(self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'), self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'), host_list, self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'), self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'), self.experiment_name, export_file_name, self.name) print(cmd_query_disk_mean) cmd_query_net = "\ninflux -precision rfc3339 -username root -password root " \ "-database 'telegraf' -host 'localhost' -execute 'SELECT sum(download_bytes),sum(upload_bytes) FROM (SELECT derivative(first(\"bytes_recv\"),1s) " \ "as \"download_bytes\",derivative(first(\"bytes_sent\"),1s) as \"upload_bytes\"" \ "FROM \"telegraf\".\"autogen\".\"net\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/ " \ "GROUP BY \"host\",time(10s)) WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' GROUP BY host,time(10s)' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/net_{}.csv" \ .format(self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'), self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'), host_list, self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'), self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'), self.experiment_name, export_file_name, self.name) print(cmd_query_net) cmd_query_net_mean = "\ninflux -precision rfc3339 -username root -password root " \ "-database 'telegraf' -host 'localhost' -execute 'SELECT sum(download_bytes),sum(upload_bytes) FROM (SELECT derivative(first(\"bytes_recv\"),1s) " \ "as \"download_bytes\",derivative(first(\"bytes_sent\"),1s) as \"upload_bytes\"" \ "FROM \"telegraf\".\"autogen\".\"net\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/ " \ "GROUP BY \"host\",time(10s)) WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' GROUP BY time(10s)' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/net_{}_mean.csv" \ .format(self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'), self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'), host_list, self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'), self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'), self.experiment_name, export_file_name, self.name) print(cmd_query_net_mean) subprocess.Popen(cmd_query_cpu + " && " + cmd_query_mem + " && " + cmd_query_disk + " && " + cmd_query_net + " && " + cmd_query_cpu_mean + " && " + cmd_query_mem_mean + " && " + cmd_query_disk_mean + " && " + cmd_query_net_mean, shell=True) time.sleep(1) with open("/data/vinh.tran/new/expData/{}/{}/cmd_{}.txt".format(self.experiment_name, export_file_name, self.name), 'a') as file: file.write("{}\n\n{}\n\n{}\n\n{}\n\n\n\n{}\n\n{}\n\n{}\n\n{}\n". format(cmd_query_cpu, cmd_query_mem, cmd_query_disk, cmd_query_net, cmd_query_cpu_mean, cmd_query_mem_mean, cmd_query_disk_mean, cmd_query_net_mean)) if callable(on_finish): on_finish(self)