Exemplo n.º 1
0
    def run(self):

        batch_index = 0
        for batch in self.batch_files:

            last_line = os.popen("tail -n 1 %s" % batch).read()
            # if sleep time is specified
            if ("SLEEP" in last_line) or ("sleep" in last_line):
                # last item in the line
                sleep_time = int(last_line.split()[-1:][0])
            else:
                sleep_time = 0

            # Copy the batch file to the log directory
            copy_string = "cp %s %s/" % (batch, self.config.log_dir)
            copy_cmd = Command(copy_string)
            code = copy_cmd.execute()
            if code == 0:
                #LOG.info("Workload %s file has been copied successfully to the log directory" % (batch))
                pass

            # Scp this file to the master
            scp_string = "scp %s %s@%s:~/%s" % (batch, self.config.workload.user, self.master.dns, self.config.workload.submit_remote)
            scp_cmd = Command(scp_string)
            code = scp_cmd.execute()
            if code == 0:
                #LOG.info("Batch file %s has been copied to the master node" % (batch))
                pass
            else:
                LOG.error("Error occurred during copying batch file %s to the master node" % (batch))

            # Send this batch to the work queue
            exec_cmd = RemoteCommand(
                config = self.config,
                hostname = self.master.dns,
                ssh_private_key = self.config.globals.priv_path,
                user = self.config.workload.user,
                command = 'condor_submit %s' % (self.config.workload.submit_remote))
            code = exec_cmd.execute()
            if code == 0:
                #LOG.info("Batch file %s has been submitted to the work queue" % (batch))
                pass
            else:
                LOG.error("Error occurred during submission of batch file %s" % (batch))

            LOG.info("%s" % (batch))

            batch_index += 1

            # Sleep for a while if this is specified in the batch file
            time.sleep(sleep_time)

            # Periodic log saving, every 100 jobs
            if batch_index == 100:
                self.scp_log_back()
                batch_index = 0

        # To give it enough time so the jobs are scheduled; unless specified otherwise
        if sleep_time == 0:
            time.sleep(60)

        # After this for loop, go into monitor mode (run while there are jobs in the queue)
        LOG.info("Workload turns into monitor mode: this thread will stop when there are no more jobs in the queue. Sleep interval: %d" % (self.interval))
        jobs = Jobs(self.config, self.master.dns)
        count = jobs.get_current_number()
        print "Initial job count: %d" % (count)

        counter = 0
        while  count > 0:
            time.sleep(self.interval)
            count = jobs.get_current_number()
            print "Job count: %d" % (count)
            counter += 1

            # Periodic log saving, every 20 iterations
            if counter == 20:
                self.scp_log_back()
                counter = 0


        # check that the queue is empty one more time
        time.sleep(120)
        LOG.info("Checking that the queue is empty one more time after 120 seconds")
        count = jobs.get_current_number()
        while  count > 0:
            time.sleep(self.interval)
            count = jobs.get_current_number()
            print "Job count: %d" % (count)

        LOG.info("Workload completed")
Exemplo n.º 2
0
    def run(self):

        LOG.info("Activating OO. Sleep period: %d sec" % (self.interval))
        jobs = Jobs(self.config, self.master.dns)

        while(not self.stop_event.is_set()):

            self.stop_event.wait(self.interval)

            # Figure out what type of iteration this is:
            # - either when the counter equals the limit -- then try marking nodes offline
            # - any other iteration -- then only terminate idle instances if any in the marked_offline_list
            self.check_within_interval_counter += 1
            if self.check_within_interval_counter == self.self.check_within_interval_limit:
                allow_marking_offline = True
                # Rest and go back to the beginning of the cycle
                self.check_within_interval_counter = 0
                LOG.info("OI's iteration with marking nodes offline and termination of idle instances")
            else:
                allow_marking_offline = False
                LOG.info("OI's iteration with termination of previously marked instances")

            curr_dict = self.get_current_dict()
            jobs.update_current_list()

            pool_dict_str = "%s," % (time.time())
            for cloud_name, instance_count in curr_dict.iteritems():
                pool_dict_str += "%s:%d," % (cloud_name,instance_count)
            pool_dict_str = pool_dict_str[:-1]
            filelog(self.config.worker_pool_log, pool_dict_str)

            diff_dict = {}

            for cloud_name in curr_dict:
                up_diff =  self.desired_dict[cloud_name] - curr_dict[cloud_name]
                diff_dict[cloud_name] = up_diff


            for cloud_name in curr_dict:
                if curr_dict[cloud_name] > self.desired_dict[cloud_name]:
                    LOG.info("Downscaling in %s" % (cloud_name))
                    down_diff = - diff_dict[cloud_name]

                    if not allow_marking_offline:
                        # give me all idle_instances that are in self.marked_offline_list
                        # only these instances are allowed to be terminated
                        if self.marked_offline_list:
                            candidates = self.get_candidates(cloud_name, jobs, down_diff, return_only_all_idle=True)
                            idle_candidates = []
                            for cand in candidates:
                                ins_id = cand[0]
                                if tuple_id in self.marked_offline_list:
                                    idle_candidates.append(ins_id)
                                    LOG.info("Selecting idle offline instance for termination: %s" % (ins_id))
                                    self.marked_offline_list.remove(ins_id)
                        else:
                            idle_candidates = []
                    else:
                        idle_candidates, nonidle_candidates = self.get_candidates(cloud_name, jobs, down_diff, return_only_all_idle=False)

                    for instance_tuple in idle_candidates:
                        instance_id = instance_tuple[0]
                        instance_info = instance_tuple[1]
                        dns = instance_info['public_dns']
                        LOG.info("OO terminated idle instance %s in %s" % (cloud_name, instance_id))
                        filelog(self.config.discarded_work_log, "DISCARDED,%s,%s,%d" % (cloud_name, dns, 0))
                        filelog(self.config.node_log, "TERMINATED WORKER cloud: %s, instance: %s, dns: %s"
                                                      % (cloud_name, instance_id, dns))

                        LOG.info("Desired capacity (before termination) is %d" % (self.phantom_client.asg.desired_capacity))
                        Worker(self.config, instance_id, instance_info).terminate_condor(self.master.dns)
                        self.phantom_client.terminate_instance(instance_id)
                        LOG.info("Desired capacity (after termination) is %d" % (self.phantom_client.asg.desired_capacity))

                        # upscale
                        sorted_diff_dict = sorted(diff_dict.iteritems(), key=operator.itemgetter(1), reverse=True)
                        if sorted_diff_dict[0][1] > 0:
                            cloud_to_upscale = sorted_diff_dict[0][0]


                            if cloud_to_upscale != cloud_name:
                                # create new tag :
                                current_cloud_tag = self.phantom_client.cloud_list.split(",")
                                new_cloud_tag = ""
                                new_cloud_count = 0
                                LOG.info("Current cloud tag is %s" % (self.phantom_client.cloud_list))
                                LOG.info("Current dict is %s" % (str(curr_dict)))
                                LOG.info("Diff dict is %s" % (str(diff_dict)))
                                for each_cloud in current_cloud_tag:
                                    tmp_cloud_name = each_cloud.split(":")[0]
                                    tmp_cloud_count = int(each_cloud.split(":")[1])
                                    if tmp_cloud_name == cloud_to_upscale:
                                        new_cloud_tag += "%s:%d," % (tmp_cloud_name, tmp_cloud_count +1)
                                        curr_dict[tmp_cloud_name] += 1
                                        diff_dict[tmp_cloud_name] -= 1
                                    elif tmp_cloud_name == cloud_name:
                                        new_cloud_tag += "%s:%d," % (tmp_cloud_name, tmp_cloud_count - 1)
                                        curr_dict[tmp_cloud_name] -= 1
                                        diff_dict[tmp_cloud_name] += 1
                                    else:
                                        new_cloud_tag += "%s:%d," % (tmp_cloud_name, tmp_cloud_count)
                                    new_cloud_count += curr_dict[tmp_cloud_name]

                                new_cloud_tag_no_comma = new_cloud_tag[:-1]
                                LOG.info("New cloud tag is %s" % (new_cloud_tag_no_comma))
                                LOG.info("New Current dict is %s" % (str(curr_dict)))
                                LOG.info("New Diff dict is %s" % (str(diff_dict)))
                                LOG.info("New Desired capacity (after recounting) is %d" % (new_cloud_count))

                                self.phantom_client.update_tags(new_cloud_tag_no_comma, new_cloud_count)
                                self.phantom_client.cloud_list = new_cloud_tag_no_comma
                                self.phantom_client.asg.set_capacity(new_cloud_count)
                            else:
                                LOG.info("Trying to upscale and downscale in the same cloud .. STOPPED")

                    if allow_marking_offline:
                        for instance_tuple in nonidle_candidates:
                            instance_id = instance_tuple[0]
                            instance_info = instance_tuple[1]
                            dns = instance_info['public_dns']
                            LOG.info("OO marked instance offline %s in %s" % (cloud_name, instance_id))
                            filelog(self.config.node_log, "OFFLINED WORKER cloud: %s, instance: %s, dns: %s"
                                                          % (cloud_name, instance_id, dns))
                            filelog(self.config.discarded_work_log, "OFFLINE,%s,%s,%d" % (cloud_name, dns, 0))

                            worker = Worker(self.config, instance_id, instance_info)
                            worker.offline(self.master.dns) # marks node offline (it later becomes idle and get terminated)
                            self.marked_offline_list.append(instance_id)
Exemplo n.º 3
0
    def run(self):

        LOG.info("Activating AD. Sleep period: %d sec" % (self.interval))
        jobs = Jobs(self.config, self.master.dns)
        while(not self.stop_event.is_set()):
            self.stop_event.wait(self.interval)

            curr_dict = self.get_current_dict()
            jobs.update_current_list()

            pool_dict_str = "%s," % (time.time())
            for cloud_name, instance_count in curr_dict.iteritems():
                pool_dict_str += "%s:%d," % (cloud_name,instance_count)
            pool_dict_str = pool_dict_str[:-1]
            filelog(self.config.worker_pool_log, pool_dict_str)

            diff_dict = {}

            for cloud_name in curr_dict:
                up_diff =  self.desired_dict[cloud_name] - curr_dict[cloud_name]
                diff_dict[cloud_name] = up_diff

            for cloud_name in curr_dict:
                if curr_dict[cloud_name] > self.desired_dict[cloud_name]:
                    LOG.info("Downscaling in %s" % (cloud_name))
                    down_diff = - diff_dict[cloud_name]
                    candidates = self.get_cloud_instances_by_runtime_inc(cloud_name, jobs)
                    termination_list = self.select_from_candidates(cloud_name, candidates, down_diff)
                    for atuple in termination_list:
                        instance_id = atuple[0]
                        running = atuple[1]
                        instance_info = atuple[2]

                        dns = instance_info['public_dns']

                        LOG.info("AD terminated instance %s in %s" % (cloud_name, instance_id))
                        filelog(self.config.discarded_work_log, "DISCARDED,%s,%s,%s" % (cloud_name, dns, running))
                        filelog(self.config.node_log, "TERMINATED WORKER cloud: %s, instance: %s, dns: %s"
                                                      % (cloud_name, instance_id, dns))

                        LOG.info("Desired capacity (before termination) is %d" % (self.phantom_client.asg.desired_capacity))
                        Worker(self.config, instance_id, instance_info).terminate_condor(self.master.dns)
                        self.phantom_client.terminate_instance(instance_id)
                        LOG.info("Desired capacity (after termination) is %d" % (self.phantom_client.asg.desired_capacity))

                        # figure out where to up scale

                        # sort the diff dict to find cloud with max number of lack isntances ( up scale )
                        # [('c', 10), ('a', 3), ('b', 1)]
                        # we sort dict by value, it returns list of tuples
                        sorted_diff_dict = sorted(diff_dict.iteritems(), key=operator.itemgetter(1), reverse=True)
                        if sorted_diff_dict[0][1] > 0:
                            cloud_to_upscale = sorted_diff_dict[0][0]


                            if cloud_to_upscale != cloud_name:
                                # create new tag :
                                current_cloud_tag = self.phantom_client.cloud_list.split(",")
                                new_cloud_tag = ""
                                new_cloud_count = 0
                                LOG.info("Current cloud tag is %s" % (self.phantom_client.cloud_list))
                                LOG.info("Current dict is %s" % (str(curr_dict)))
                                LOG.info("Diff dict is %s" % (str(diff_dict)))
                                for each_cloud in current_cloud_tag:
                                    tmp_cloud_name = each_cloud.split(":")[0]
                                    tmp_cloud_count = int(each_cloud.split(":")[1])
                                    if tmp_cloud_name == cloud_to_upscale:
                                        new_cloud_tag += "%s:%d," % (tmp_cloud_name, tmp_cloud_count +1)
                                        curr_dict[tmp_cloud_name] += 1
                                        diff_dict[tmp_cloud_name] -= 1
                                    elif tmp_cloud_name == cloud_name:
                                        new_cloud_tag += "%s:%d," % (tmp_cloud_name, tmp_cloud_count - 1)
                                        curr_dict[tmp_cloud_name] -= 1
                                        diff_dict[tmp_cloud_name] += 1
                                    else:
                                        new_cloud_tag += "%s:%d," % (tmp_cloud_name, tmp_cloud_count)
                                    new_cloud_count += curr_dict[tmp_cloud_name]

                                new_cloud_tag_no_comma = new_cloud_tag[:-1]
                                LOG.info("New cloud tag is %s" % (new_cloud_tag_no_comma))
                                LOG.info("New Current dict is %s" % (str(curr_dict)))
                                LOG.info("New Diff dict is %s" % (str(diff_dict)))
                                LOG.info("New Desired capacity (after recounting) is %d" % (new_cloud_count))

                                self.phantom_client.update_tags(new_cloud_tag_no_comma, new_cloud_count)
                                self.phantom_client.cloud_list = new_cloud_tag_no_comma
                                self.phantom_client.asg.set_capacity(new_cloud_count)
                            else:
                                LOG.info("Trying to upscale and downscale in the same cloud .. STOPPED")
Exemplo n.º 4
0
    def run(self):

        LOG.info("Activating OI. Sleep period: %d sec" % (self.interval))
        jobs = Jobs(self.config, self.master.dns)
        while(not self.stop_event.is_set()):
            self.stop_event.wait(self.interval)

            curr_dict = self.get_current_dict()
            jobs.update_current_list()


            pool_dict_str = "%s," % (time.time())
            for cloud_name, instance_count in curr_dict.iteritems():
                pool_dict_str += "%s:%d," % (cloud_name,instance_count)
            pool_dict_str = pool_dict_str[:-1]
            filelog(self.config.worker_pool_log, pool_dict_str)

            diff_dict = {}

            for cloud_name in curr_dict:
                up_diff =  self.desired_dict[cloud_name] - curr_dict[cloud_name]
                diff_dict[cloud_name] = up_diff

            for cloud_name in curr_dict:
                if curr_dict[cloud_name] > self.desired_dict[cloud_name]:

                    down_diff = - diff_dict[cloud_name]
                    candidates = self.get_idle_instances(cloud_name, jobs)

                    # Only terminate as many as needed
                    termination_list = candidates[:down_diff]
                    if termination_list:
                        LOG.info("Downscaling in %s" % (cloud_name))
                    else:
                        LOG.info("Not Downscaling because no idle instances found in %s" % (cloud_name))

                    for instance_tuple in termination_list:
                        instance_id = instance_tuple[0]
                        instance_info = instance_tuple[1]
                        dns = instance_info['public_dns']
                        LOG.info("OI terminated instance %s in %s" % (cloud_name, instance_id))
                        filelog(self.config.discarded_work_log, "DISCARDED,%s,%s,%d" % (cloud_name, dns, 0))
                        filelog(self.config.node_log, "TERMINATED WORKER cloud: %s, instance: %s, dns: %s"
                                                      % (cloud_name, instance_id, dns))

                        LOG.info("Desired capacity (before termination) is %d" % (self.phantom_client.asg.desired_capacity))

                        Worker(self.config, instance_id, instance_info).terminate_condor(self.master.dns)
                        self.phantom_client.terminate_instance(instance_id)

                        LOG.info("Desired capacity (after termination) is %d" % (self.phantom_client.asg.desired_capacity))

                        # upscale

                        sorted_diff_dict = sorted(diff_dict.iteritems(), key=operator.itemgetter(1), reverse=True)
                        if sorted_diff_dict[0][1] > 0:
                            cloud_to_upscale = sorted_diff_dict[0][0]
                            if cloud_to_upscale != cloud_name:
                            # create new tag :
                                current_cloud_tag = self.phantom_client.cloud_list.split(",")
                                new_cloud_tag = ""
                                new_cloud_count = 0
                                LOG.info("Current cloud tag is %s" % (self.phantom_client.cloud_list))
                                LOG.info("Current dict is %s" % (str(curr_dict)))
                                LOG.info("Diff dict is %s" % (str(diff_dict)))
                                for each_cloud in current_cloud_tag:
                                    tmp_cloud_name = each_cloud.split(":")[0]
                                    tmp_cloud_count = int(each_cloud.split(":")[1])
                                    if tmp_cloud_name == cloud_to_upscale:
                                        new_cloud_tag += "%s:%d," % (tmp_cloud_name, tmp_cloud_count +1)
                                        curr_dict[tmp_cloud_name] += 1
                                        diff_dict[tmp_cloud_name] -= 1
                                    elif tmp_cloud_name == cloud_name:
                                        new_cloud_tag += "%s:%d," % (tmp_cloud_name, tmp_cloud_count - 1)
                                        curr_dict[tmp_cloud_name] -= 1
                                        diff_dict[tmp_cloud_name] += 1
                                    else:
                                        new_cloud_tag += "%s:%d," % (tmp_cloud_name, tmp_cloud_count)
                                    new_cloud_count += curr_dict[tmp_cloud_name]

                                new_cloud_tag_no_comma = new_cloud_tag[:-1]
                                LOG.info("New cloud tag is %s" % (new_cloud_tag_no_comma))
                                LOG.info("New Current dict is %s" % (str(curr_dict)))
                                LOG.info("New Diff dict is %s" % (str(diff_dict)))
                                LOG.info("New Desired capacity (after recounting) is %d" % (new_cloud_count))

                                self.phantom_client.update_tags(new_cloud_tag_no_comma, new_cloud_count)
                                self.phantom_client.cloud_list = new_cloud_tag_no_comma
                                self.phantom_client.asg.set_capacity(new_cloud_count)
                            else:
                                LOG.info("Trying to upscale and downscale in the same cloud .. STOPPED")