Ejemplo n.º 1
0
    def _run(self, resource_manager: ResourceManager, on_finish,
             sleep_during_loop):
        cmd = " ".join(self.command_line())
        if self.print_command_line:
            print("Start {} with cmd: {}".format(self.id, cmd))
        subprocess.Popen(cmd, shell=True)

        while not resource_manager.is_application_finished(self.id):
            if not self.is_running and resource_manager.is_application_running(
                    self.id):
                self.is_running = True

            time.sleep(sleep_during_loop)

        print("Application {} has finished".format(self))

        if callable(on_finish):
            on_finish(self)
    def _run(self, resource_manager: ResourceManager, on_finish, sleep_during_loop):
        cmd = " ".join(self.command_line())
        #if self.print_command_line:
        print("Start {} with cmd: {}".format(self.id, cmd))
        subprocess.Popen(cmd, shell=True)

        self.start_at = datetime.datetime.utcnow()

        time.sleep(sleep_during_loop + 30)
        while not resource_manager.is_application_finished(self.id):
            if not self.is_running and resource_manager.is_application_running(self.id):
                self.is_running = True

            time.sleep(sleep_during_loop)

        print("Application {} has finished".format(self))

        self.end_at = datetime.datetime.utcnow()

        host_list = "|".join([address for address in self.nodes])
        export_file_name = self.id + "_" + self.name

        cmd_query_cpu = "\nmkdir /data/vinh.tran/new/expData/{}/{}" \
                        "&& influx -precision rfc3339 -username root -password root" \
                        " -database 'telegraf' -host 'localhost' -execute 'SELECT usage_user,usage_iowait " \
                        "FROM \"telegraf\".\"autogen\".\"cpu\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/  " \
                        "AND cpu = '\\''cpu-total'\\'' GROUP BY host' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/cpu_{}.csv" \
            .format(self.experiment_name,
                    export_file_name,
                    self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    host_list,
                    self.experiment_name,
                    export_file_name,
                    self.name)
        print(cmd_query_cpu)
        # subprocess.Popen(cmd_query_cpu, shell=True)

        cmd_query_cpu_mean = "\ninflux -precision rfc3339 -username root -password root" \
                        " -database 'telegraf' -host 'localhost' -execute 'SELECT mean(usage_user) as \"mean_cpu_percent\",mean(usage_iowait) as \"mean_io_wait\" " \
                        "FROM \"telegraf\".\"autogen\".\"cpu\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/  " \
                        "AND cpu = '\\''cpu-total'\\'' GROUP BY time(10s)' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/cpu_{}_mean.csv" \
            .format(self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    host_list,
                    self.experiment_name,
                    export_file_name,
                    self.name)
        print(cmd_query_cpu_mean)

        cmd_query_mem = "\ninflux -precision rfc3339 -username root -password root " \
                        "-database 'telegraf' -host 'localhost' -execute 'SELECT used_percent " \
                        "FROM \"telegraf\".\"autogen\".\"mem\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/  " \
                        "GROUP BY host' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/mem_{}.csv" \
            .format(self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    host_list,
                    self.experiment_name,
                    export_file_name,
                    self.name)
        print(cmd_query_mem)

        cmd_query_mem_mean = "\ninflux -precision rfc3339 -username root -password root " \
                        "-database 'telegraf' -host 'localhost' -execute 'SELECT mean(used_percent) " \
                        "FROM \"telegraf\".\"autogen\".\"mem\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/  " \
                        "GROUP BY time(10s)' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/mem_{}_mean.csv" \
            .format(self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    host_list,
                    self.experiment_name,
                    export_file_name,
                    self.name)
        print(cmd_query_mem_mean)

        cmd_query_disk = "\ninflux -precision rfc3339 -username root -password root " \
                        "-database 'telegraf' -host 'localhost' -execute 'SELECT sum(read_bytes),sum(write_bytes) " \
                        "FROM (SELECT derivative(last(\"read_bytes\"),1s) as \"read_bytes\",derivative(last(\"write_bytes\"),1s) as \"write_bytes\",derivative(last(\"io_time\"),1s) as \"io_time\" " \
                        "FROM \"telegraf\".\"autogen\".\"diskio\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/  " \
                        "GROUP BY \"host\",\"name\",time(10s)) WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' GROUP BY host,time(10s)' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/disk_{}.csv" \
            .format(self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    host_list,
                    self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    self.experiment_name,
                    export_file_name,
                    self.name)
        print(cmd_query_disk)

        cmd_query_disk_mean = "\ninflux -precision rfc3339 -username root -password root " \
                         "-database 'telegraf' -host 'localhost' -execute 'SELECT sum(read_bytes),sum(write_bytes) " \
                         "FROM (SELECT derivative(last(\"read_bytes\"),1s) as \"read_bytes\",derivative(last(\"write_bytes\"),1s) as \"write_bytes\",derivative(last(\"io_time\"),1s) as \"io_time\" " \
                         "FROM \"telegraf\".\"autogen\".\"diskio\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/  " \
                         "GROUP BY \"host\",\"name\",time(10s)) WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' GROUP BY time(10s)' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/disk_{}_mean.csv" \
            .format(self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    host_list,
                    self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    self.experiment_name,
                    export_file_name,
                    self.name)
        print(cmd_query_disk_mean)

        cmd_query_net = "\ninflux -precision rfc3339 -username root -password root " \
                         "-database 'telegraf' -host 'localhost' -execute 'SELECT sum(download_bytes),sum(upload_bytes) FROM (SELECT  derivative(first(\"bytes_recv\"),1s) " \
                         "as \"download_bytes\",derivative(first(\"bytes_sent\"),1s) as \"upload_bytes\"" \
                         "FROM \"telegraf\".\"autogen\".\"net\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/  " \
                         "GROUP BY \"host\",time(10s)) WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' GROUP BY host,time(10s)' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/net_{}.csv" \
            .format(self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    host_list,
                    self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    self.experiment_name,
                    export_file_name,
                    self.name)
        print(cmd_query_net)

        cmd_query_net_mean = "\ninflux -precision rfc3339 -username root -password root " \
                        "-database 'telegraf' -host 'localhost' -execute 'SELECT sum(download_bytes),sum(upload_bytes) FROM (SELECT  derivative(first(\"bytes_recv\"),1s) " \
                        "as \"download_bytes\",derivative(first(\"bytes_sent\"),1s) as \"upload_bytes\"" \
                        "FROM \"telegraf\".\"autogen\".\"net\" WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' AND host =~ /{}/  " \
                        "GROUP BY \"host\",time(10s)) WHERE time > '\\''{}'\\'' and time < '\\''{}'\\'' GROUP BY time(10s)' -format 'csv' > /data/vinh.tran/new/expData/{}/{}/net_{}_mean.csv" \
            .format(self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    host_list,
                    self.start_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    self.end_at.strftime('%Y-%m-%dT%H:%M:%SZ'),
                    self.experiment_name,
                    export_file_name,
                    self.name)
        print(cmd_query_net_mean)

        subprocess.Popen(cmd_query_cpu + " && " + cmd_query_mem + " && " + cmd_query_disk + " && " + cmd_query_net + " && "
                         + cmd_query_cpu_mean + " && " + cmd_query_mem_mean + " && " + cmd_query_disk_mean + " && " + cmd_query_net_mean, shell=True)

        time.sleep(1)

        with open("/data/vinh.tran/new/expData/{}/{}/cmd_{}.txt".format(self.experiment_name, export_file_name, self.name), 'a') as file:
            file.write("{}\n\n{}\n\n{}\n\n{}\n\n\n\n{}\n\n{}\n\n{}\n\n{}\n".
                       format(cmd_query_cpu, cmd_query_mem, cmd_query_disk, cmd_query_net,
                              cmd_query_cpu_mean, cmd_query_mem_mean, cmd_query_disk_mean, cmd_query_net_mean))

        if callable(on_finish):
            on_finish(self)