def run(self):
batch_index = 0
for batch in self.batch_files:
last_line = os.popen("tail -n 1 %s" % batch).read()
# if sleep time is specified
if ("SLEEP" in last_line) or ("sleep" in last_line):
# last item in the line
sleep_time = int(last_line.split()[-1:][0])
else:
sleep_time = 0
# Copy the batch file to the log directory
copy_string = "cp %s %s/" % (batch, self.config.log_dir)
copy_cmd = Command(copy_string)
code = copy_cmd.execute()
if code == 0:
#LOG.info("Workload %s file has been copied successfully to the log directory" % (batch))
pass
# Scp this file to the master
scp_string = "scp %s %s@%s:~/%s" % (batch, self.config.workload.user, self.master.dns, self.config.workload.submit_remote)
scp_cmd = Command(scp_string)
code = scp_cmd.execute()
if code == 0:
#LOG.info("Batch file %s has been copied to the master node" % (batch))
pass
else:
LOG.error("Error occurred during copying batch file %s to the master node" % (batch))
# Send this batch to the work queue
exec_cmd = RemoteCommand(
config = self.config,
hostname = self.master.dns,
ssh_private_key = self.config.globals.priv_path,
user = self.config.workload.user,
command = 'condor_submit %s' % (self.config.workload.submit_remote))
code = exec_cmd.execute()
if code == 0:
#LOG.info("Batch file %s has been submitted to the work queue" % (batch))
pass
else:
LOG.error("Error occurred during submission of batch file %s" % (batch))
LOG.info("%s" % (batch))
batch_index += 1
# Sleep for a while if this is specified in the batch file
time.sleep(sleep_time)
# Periodic log saving, every 100 jobs
if batch_index == 100:
self.scp_log_back()
batch_index = 0
# To give it enough time so the jobs are scheduled; unless specified otherwise
if sleep_time == 0:
time.sleep(60)
# After this for loop, go into monitor mode (run while there are jobs in the queue)
LOG.info("Workload turns into monitor mode: this thread will stop when there are no more jobs in the queue. Sleep interval: %d" % (self.interval))
jobs = Jobs(self.config, self.master.dns)
count = jobs.get_current_number()
print "Initial job count: %d" % (count)
counter = 0
while count > 0:
time.sleep(self.interval)
count = jobs.get_current_number()
print "Job count: %d" % (count)
counter += 1
# Periodic log saving, every 20 iterations
if counter == 20:
self.scp_log_back()
counter = 0
# check that the queue is empty one more time
time.sleep(120)
LOG.info("Checking that the queue is empty one more time after 120 seconds")
count = jobs.get_current_number()
while count > 0:
time.sleep(self.interval)
count = jobs.get_current_number()
print "Job count: %d" % (count)
LOG.info("Workload completed")
def run(self):
LOG.info("Activating OO. Sleep period: %d sec" % (self.interval))
jobs = Jobs(self.config, self.master.dns)
while(not self.stop_event.is_set()):
self.stop_event.wait(self.interval)
# Figure out what type of iteration this is:
# - either when the counter equals the limit -- then try marking nodes offline
# - any other iteration -- then only terminate idle instances if any in the marked_offline_list
self.check_within_interval_counter += 1
if self.check_within_interval_counter == self.self.check_within_interval_limit:
allow_marking_offline = True
# Rest and go back to the beginning of the cycle
self.check_within_interval_counter = 0
LOG.info("OI's iteration with marking nodes offline and termination of idle instances")
else:
allow_marking_offline = False
LOG.info("OI's iteration with termination of previously marked instances")
curr_dict = self.get_current_dict()
jobs.update_current_list()
pool_dict_str = "%s," % (time.time())
for cloud_name, instance_count in curr_dict.iteritems():
pool_dict_str += "%s:%d," % (cloud_name,instance_count)
pool_dict_str = pool_dict_str[:-1]
filelog(self.config.worker_pool_log, pool_dict_str)
diff_dict = {}
for cloud_name in curr_dict:
up_diff = self.desired_dict[cloud_name] - curr_dict[cloud_name]
diff_dict[cloud_name] = up_diff
for cloud_name in curr_dict:
if curr_dict[cloud_name] > self.desired_dict[cloud_name]:
LOG.info("Downscaling in %s" % (cloud_name))
down_diff = - diff_dict[cloud_name]
if not allow_marking_offline:
# give me all idle_instances that are in self.marked_offline_list
# only these instances are allowed to be terminated
if self.marked_offline_list:
candidates = self.get_candidates(cloud_name, jobs, down_diff, return_only_all_idle=True)
idle_candidates = []
for cand in candidates:
ins_id = cand[0]
if tuple_id in self.marked_offline_list:
idle_candidates.append(ins_id)
LOG.info("Selecting idle offline instance for termination: %s" % (ins_id))
self.marked_offline_list.remove(ins_id)
else:
idle_candidates = []
else:
idle_candidates, nonidle_candidates = self.get_candidates(cloud_name, jobs, down_diff, return_only_all_idle=False)
for instance_tuple in idle_candidates:
instance_id = instance_tuple[0]
instance_info = instance_tuple[1]
dns = instance_info['public_dns']
LOG.info("OO terminated idle instance %s in %s" % (cloud_name, instance_id))
filelog(self.config.discarded_work_log, "DISCARDED,%s,%s,%d" % (cloud_name, dns, 0))
filelog(self.config.node_log, "TERMINATED WORKER cloud: %s, instance: %s, dns: %s"
% (cloud_name, instance_id, dns))
LOG.info("Desired capacity (before termination) is %d" % (self.phantom_client.asg.desired_capacity))
Worker(self.config, instance_id, instance_info).terminate_condor(self.master.dns)
self.phantom_client.terminate_instance(instance_id)
LOG.info("Desired capacity (after termination) is %d" % (self.phantom_client.asg.desired_capacity))
# upscale
sorted_diff_dict = sorted(diff_dict.iteritems(), key=operator.itemgetter(1), reverse=True)
if sorted_diff_dict[0][1] > 0:
cloud_to_upscale = sorted_diff_dict[0][0]
if cloud_to_upscale != cloud_name:
# create new tag :
current_cloud_tag = self.phantom_client.cloud_list.split(",")
new_cloud_tag = ""
new_cloud_count = 0
LOG.info("Current cloud tag is %s" % (self.phantom_client.cloud_list))
LOG.info("Current dict is %s" % (str(curr_dict)))
LOG.info("Diff dict is %s" % (str(diff_dict)))
for each_cloud in current_cloud_tag:
tmp_cloud_name = each_cloud.split(":")[0]
tmp_cloud_count = int(each_cloud.split(":")[1])
if tmp_cloud_name == cloud_to_upscale:
new_cloud_tag += "%s:%d," % (tmp_cloud_name, tmp_cloud_count +1)
curr_dict[tmp_cloud_name] += 1
diff_dict[tmp_cloud_name] -= 1
elif tmp_cloud_name == cloud_name:
new_cloud_tag += "%s:%d," % (tmp_cloud_name, tmp_cloud_count - 1)
curr_dict[tmp_cloud_name] -= 1
diff_dict[tmp_cloud_name] += 1
else:
new_cloud_tag += "%s:%d," % (tmp_cloud_name, tmp_cloud_count)
new_cloud_count += curr_dict[tmp_cloud_name]
new_cloud_tag_no_comma = new_cloud_tag[:-1]
LOG.info("New cloud tag is %s" % (new_cloud_tag_no_comma))
LOG.info("New Current dict is %s" % (str(curr_dict)))
LOG.info("New Diff dict is %s" % (str(diff_dict)))
LOG.info("New Desired capacity (after recounting) is %d" % (new_cloud_count))
self.phantom_client.update_tags(new_cloud_tag_no_comma, new_cloud_count)
self.phantom_client.cloud_list = new_cloud_tag_no_comma
self.phantom_client.asg.set_capacity(new_cloud_count)
else:
LOG.info("Trying to upscale and downscale in the same cloud .. STOPPED")
if allow_marking_offline:
for instance_tuple in nonidle_candidates:
instance_id = instance_tuple[0]
instance_info = instance_tuple[1]
dns = instance_info['public_dns']
LOG.info("OO marked instance offline %s in %s" % (cloud_name, instance_id))
filelog(self.config.node_log, "OFFLINED WORKER cloud: %s, instance: %s, dns: %s"
% (cloud_name, instance_id, dns))
filelog(self.config.discarded_work_log, "OFFLINE,%s,%s,%d" % (cloud_name, dns, 0))
worker = Worker(self.config, instance_id, instance_info)
worker.offline(self.master.dns) # marks node offline (it later becomes idle and get terminated)
self.marked_offline_list.append(instance_id)
def run(self):
LOG.info("Activating AD. Sleep period: %d sec" % (self.interval))
jobs = Jobs(self.config, self.master.dns)
while(not self.stop_event.is_set()):
self.stop_event.wait(self.interval)
curr_dict = self.get_current_dict()
jobs.update_current_list()
pool_dict_str = "%s," % (time.time())
for cloud_name, instance_count in curr_dict.iteritems():
pool_dict_str += "%s:%d," % (cloud_name,instance_count)
pool_dict_str = pool_dict_str[:-1]
filelog(self.config.worker_pool_log, pool_dict_str)
diff_dict = {}
for cloud_name in curr_dict:
up_diff = self.desired_dict[cloud_name] - curr_dict[cloud_name]
diff_dict[cloud_name] = up_diff
for cloud_name in curr_dict:
if curr_dict[cloud_name] > self.desired_dict[cloud_name]:
LOG.info("Downscaling in %s" % (cloud_name))
down_diff = - diff_dict[cloud_name]
candidates = self.get_cloud_instances_by_runtime_inc(cloud_name, jobs)
termination_list = self.select_from_candidates(cloud_name, candidates, down_diff)
for atuple in termination_list:
instance_id = atuple[0]
running = atuple[1]
instance_info = atuple[2]
dns = instance_info['public_dns']
LOG.info("AD terminated instance %s in %s" % (cloud_name, instance_id))
filelog(self.config.discarded_work_log, "DISCARDED,%s,%s,%s" % (cloud_name, dns, running))
filelog(self.config.node_log, "TERMINATED WORKER cloud: %s, instance: %s, dns: %s"
% (cloud_name, instance_id, dns))
LOG.info("Desired capacity (before termination) is %d" % (self.phantom_client.asg.desired_capacity))
Worker(self.config, instance_id, instance_info).terminate_condor(self.master.dns)
self.phantom_client.terminate_instance(instance_id)
LOG.info("Desired capacity (after termination) is %d" % (self.phantom_client.asg.desired_capacity))
# figure out where to up scale
# sort the diff dict to find cloud with max number of lack isntances ( up scale )
# [('c', 10), ('a', 3), ('b', 1)]
# we sort dict by value, it returns list of tuples
sorted_diff_dict = sorted(diff_dict.iteritems(), key=operator.itemgetter(1), reverse=True)
if sorted_diff_dict[0][1] > 0:
cloud_to_upscale = sorted_diff_dict[0][0]
if cloud_to_upscale != cloud_name:
# create new tag :
current_cloud_tag = self.phantom_client.cloud_list.split(",")
new_cloud_tag = ""
new_cloud_count = 0
LOG.info("Current cloud tag is %s" % (self.phantom_client.cloud_list))
LOG.info("Current dict is %s" % (str(curr_dict)))
LOG.info("Diff dict is %s" % (str(diff_dict)))
for each_cloud in current_cloud_tag:
tmp_cloud_name = each_cloud.split(":")[0]
tmp_cloud_count = int(each_cloud.split(":")[1])
if tmp_cloud_name == cloud_to_upscale:
new_cloud_tag += "%s:%d," % (tmp_cloud_name, tmp_cloud_count +1)
curr_dict[tmp_cloud_name] += 1
diff_dict[tmp_cloud_name] -= 1
elif tmp_cloud_name == cloud_name:
new_cloud_tag += "%s:%d," % (tmp_cloud_name, tmp_cloud_count - 1)
curr_dict[tmp_cloud_name] -= 1
diff_dict[tmp_cloud_name] += 1
else:
new_cloud_tag += "%s:%d," % (tmp_cloud_name, tmp_cloud_count)
new_cloud_count += curr_dict[tmp_cloud_name]
new_cloud_tag_no_comma = new_cloud_tag[:-1]
LOG.info("New cloud tag is %s" % (new_cloud_tag_no_comma))
LOG.info("New Current dict is %s" % (str(curr_dict)))
LOG.info("New Diff dict is %s" % (str(diff_dict)))
LOG.info("New Desired capacity (after recounting) is %d" % (new_cloud_count))
self.phantom_client.update_tags(new_cloud_tag_no_comma, new_cloud_count)
self.phantom_client.cloud_list = new_cloud_tag_no_comma
self.phantom_client.asg.set_capacity(new_cloud_count)
else:
LOG.info("Trying to upscale and downscale in the same cloud .. STOPPED")
def run(self):
LOG.info("Activating OI. Sleep period: %d sec" % (self.interval))
jobs = Jobs(self.config, self.master.dns)
while(not self.stop_event.is_set()):
self.stop_event.wait(self.interval)
curr_dict = self.get_current_dict()
jobs.update_current_list()
pool_dict_str = "%s," % (time.time())
for cloud_name, instance_count in curr_dict.iteritems():
pool_dict_str += "%s:%d," % (cloud_name,instance_count)
pool_dict_str = pool_dict_str[:-1]
filelog(self.config.worker_pool_log, pool_dict_str)
diff_dict = {}
for cloud_name in curr_dict:
up_diff = self.desired_dict[cloud_name] - curr_dict[cloud_name]
diff_dict[cloud_name] = up_diff
for cloud_name in curr_dict:
if curr_dict[cloud_name] > self.desired_dict[cloud_name]:
down_diff = - diff_dict[cloud_name]
candidates = self.get_idle_instances(cloud_name, jobs)
# Only terminate as many as needed
termination_list = candidates[:down_diff]
if termination_list:
LOG.info("Downscaling in %s" % (cloud_name))
else:
LOG.info("Not Downscaling because no idle instances found in %s" % (cloud_name))
for instance_tuple in termination_list:
instance_id = instance_tuple[0]
instance_info = instance_tuple[1]
dns = instance_info['public_dns']
LOG.info("OI terminated instance %s in %s" % (cloud_name, instance_id))
filelog(self.config.discarded_work_log, "DISCARDED,%s,%s,%d" % (cloud_name, dns, 0))
filelog(self.config.node_log, "TERMINATED WORKER cloud: %s, instance: %s, dns: %s"
% (cloud_name, instance_id, dns))
LOG.info("Desired capacity (before termination) is %d" % (self.phantom_client.asg.desired_capacity))
Worker(self.config, instance_id, instance_info).terminate_condor(self.master.dns)
self.phantom_client.terminate_instance(instance_id)
LOG.info("Desired capacity (after termination) is %d" % (self.phantom_client.asg.desired_capacity))
# upscale
sorted_diff_dict = sorted(diff_dict.iteritems(), key=operator.itemgetter(1), reverse=True)
if sorted_diff_dict[0][1] > 0:
cloud_to_upscale = sorted_diff_dict[0][0]
if cloud_to_upscale != cloud_name:
# create new tag :
current_cloud_tag = self.phantom_client.cloud_list.split(",")
new_cloud_tag = ""
new_cloud_count = 0
LOG.info("Current cloud tag is %s" % (self.phantom_client.cloud_list))
LOG.info("Current dict is %s" % (str(curr_dict)))
LOG.info("Diff dict is %s" % (str(diff_dict)))
for each_cloud in current_cloud_tag:
tmp_cloud_name = each_cloud.split(":")[0]
tmp_cloud_count = int(each_cloud.split(":")[1])
if tmp_cloud_name == cloud_to_upscale:
new_cloud_tag += "%s:%d," % (tmp_cloud_name, tmp_cloud_count +1)
curr_dict[tmp_cloud_name] += 1
diff_dict[tmp_cloud_name] -= 1
elif tmp_cloud_name == cloud_name:
new_cloud_tag += "%s:%d," % (tmp_cloud_name, tmp_cloud_count - 1)
curr_dict[tmp_cloud_name] -= 1
diff_dict[tmp_cloud_name] += 1
else:
new_cloud_tag += "%s:%d," % (tmp_cloud_name, tmp_cloud_count)
new_cloud_count += curr_dict[tmp_cloud_name]
new_cloud_tag_no_comma = new_cloud_tag[:-1]
LOG.info("New cloud tag is %s" % (new_cloud_tag_no_comma))
LOG.info("New Current dict is %s" % (str(curr_dict)))
LOG.info("New Diff dict is %s" % (str(diff_dict)))
LOG.info("New Desired capacity (after recounting) is %d" % (new_cloud_count))
self.phantom_client.update_tags(new_cloud_tag_no_comma, new_cloud_count)
self.phantom_client.cloud_list = new_cloud_tag_no_comma
self.phantom_client.asg.set_capacity(new_cloud_count)
else:
LOG.info("Trying to upscale and downscale in the same cloud .. STOPPED")