def __init__(self, config):
threading.Thread.__init__(self)
self.config = config
self.timeout = self.config.timeout
self.is_reporting = False
self.report_interval = self.config.report_interval
self.instanceMetric = ProvisionInstanceMetric()
class ProvisionReporter(threading.Thread): ''' The class is a thread to report the state of provision ''' def __init__(self, config): threading.Thread.__init__(self) self.config = config self.timeout = self.config.timeout self.is_reporting = False self.report_interval = self.config.report_interval self.instanceMetric=ProvisionInstanceMetric() def run(self): self.collect_statistic() def start_reporter(self): self.start() def terminate_reporter(self): self.is_reporting=False self.join() def collect_statistic(self): timeout = self.timeout start_time=time.time() self.is_reporting=True #while(timeout > 0): while(self.is_reporting): sleep(self.report_interval) print self.instanceMetric._collect_statistic()
def __init__(self, config):
threading.Thread.__init__(self)
self.config = config
self.instanceMetric = ProvisionInstanceMetric()
self.cloud = get_cloud_api(self.config)
self.running = True
self.waiting_for_all_expired = False
class ProvisionReporter(threading.Thread):
''' The class is a thread to report the state of provision '''
def __init__(self, config):
threading.Thread.__init__(self)
self.config = config
self.timeout = self.config.timeout
self.is_reporting = False
self.report_interval = self.config.report_interval
self.instanceMetric = ProvisionInstanceMetric()
def run(self):
self.collect_statistic()
def start_reporter(self):
self.start()
def terminate_reporter(self):
self.is_reporting = False
self.join()
def collect_statistic(self):
timeout = self.timeout
start_time = time.time()
self.is_reporting = True
#while(timeout > 0):
while (self.is_reporting):
sleep(self.report_interval)
print self.instanceMetric._collect_statistic()
def __init__(self, config):
threading.Thread.__init__(self)
self.config = config
self.instanceMetric = ProvisionInstanceMetric()
self.cloud = get_cloud_api(self.config)
self.running = True
self.instance_queue = Queue.Queue()
self.instances_last_update_time = time.time()
self.instances_next_update_time = time.time()
self.instances_update_flag = True
def _terminate_instance(self, cloud_api, instance):
if instance._is_running():
if cloud_api.terminate_instance(instance.id):
ProvisionInstanceMetric()._update_instance(
instance, "DELETING", time.time())
self.is_terminating_instances += 1
else:
ProvisionInstanceMetric()._update_instance(
instance, "DELETED", time.time())
self.is_terminating_instances += 1
print "#######one instance is deleting#########"
return True
class InstanceManager(threading.Thread):
''' The class is a thread to monitor all the provisions '''
def __init__(self, config):
threading.Thread.__init__(self)
self.config = config
self.instanceMetric = ProvisionInstanceMetric()
self.cloud = get_cloud_api(self.config)
self.running = True
self.instance_queue = Queue.Queue()
self.instances_last_update_time = time.time()
self.instances_next_update_time = time.time()
self.instances_update_flag = True
#self.get_host = not self.config.disable_host
def get_instance_queue(self):
return self.instance_queue
def terminateIM(self):
self.running = False
self.join()
def run(self):
while (self.running):
self.collect_instance()
if not self.config.disable_host:
self._update_instances_info()
sleep(1)
self.collect_instance()
self._wait_for_all_instances_get_host()
def collect_instance(self):
while not self.instance_queue.empty():
instance = self.instance_queue.get()
self.instanceMetric._append_instance(instance)
def _wait_for_all_instances_get_host(self):
timeout = 120
while not self.instanceMetric._wait_for_all_instances_get_host(
self.cloud) and timeout > 0:
time.sleep(20)
timeout = timeout - 20
def _update_instances_info(self):
if self.instances_update_flag and (
time.time() - self.instances_next_update_time > 0):
if self.instanceMetric._update_instances_info(self.cloud):
self.instances_next_update_time = time.time(
) + 2 * self.config.min_lifetime / 3
self.instances_last_update_time = time.time()
elif time.time(
) - self.instances_last_update_time > 5 * self.config.client_interval:
self.instances_update_flag = False
class InstanceManager(threading.Thread): ''' The class is a thread to monitor all the provisions ''' def __init__(self, config): threading.Thread.__init__(self) self.config = config self.instanceMetric=ProvisionInstanceMetric() self.cloud = get_cloud_api(self.config) self.running=True self.instance_queue = Queue.Queue() self.instances_last_update_time = time.time() self.instances_next_update_time = time.time() self.instances_update_flag = True #self.get_host = not self.config.disable_host def get_instance_queue(self): return self.instance_queue def terminateIM(self): self.running=False self.join() def run(self): while(self.running): self.collect_instance() if not self.config.disable_host: self._update_instances_info() sleep(1) self.collect_instance() self._wait_for_all_instances_get_host() def collect_instance(self): while not self.instance_queue.empty(): instance = self.instance_queue.get() self.instanceMetric._append_instance(instance) def _wait_for_all_instances_get_host(self): timeout = 120 while not self.instanceMetric._wait_for_all_instances_get_host(self.cloud) and timeout > 0: time.sleep(20) timeout = timeout - 20 def _update_instances_info(self): if self.instances_update_flag and (time.time() - self.instances_next_update_time > 0): if self.instanceMetric._update_instances_info(self.cloud): self.instances_next_update_time = time.time() + 2*self.config.min_lifetime/3 self.instances_last_update_time = time.time() elif time.time() - self.instances_last_update_time > 5*self.config.client_interval: self.instances_update_flag = False
def _terminate_instance_by_id(self, cloud_api, instance_id):
for instance in self.instance_list:
if (instance.id == instance_id) and (not instance.delete_request):
break
if instance._is_running():
if cloud_api.terminate_instance(instance_id):
ProvisionInstanceMetric()._update_instance(
instance, "DELETING", time.time())
self.is_terminating_instances += 1
else:
ProvisionInstanceMetric()._update_instance(
instance, "DELETED", time.time())
self.is_terminating_instances += 1
print "#######one instance is deleting#########"
return True
class InstanceGC(threading.Thread):
''' The class is a thread to monitor all the provisions '''
def __init__(self, config):
threading.Thread.__init__(self)
self.config = config
self.instanceMetric = ProvisionInstanceMetric()
self.cloud = get_cloud_api(self.config)
self.running = True
self.waiting_for_all_expired = False
def join(self):
self.waiting_for_all_expired = True
threading.Thread.join(self)
def terminateGC(self):
self.running = False
self.waiting_for_all_expired = True
self.join()
def run(self):
while (self.running):
if self.GC():
break
sleep(1)
def GC(self):
exp, total = self.instanceMetric._instance_GC(self.cloud)
print 'deleted %d instances from total %d instances' % (exp, total)
if exp == total and self.waiting_for_all_expired:
return True
return False
def __init__(self, config): threading.Thread.__init__(self) self.config = config self.timeout = self.config.timeout self.is_reporting = False self.report_interval = self.config.report_interval self.instanceMetric=ProvisionInstanceMetric()
class InstanceGC(threading.Thread): ''' The class is a thread to monitor all the provisions ''' def __init__(self, config): threading.Thread.__init__(self) self.config = config self.instanceMetric=ProvisionInstanceMetric() self.cloud = get_cloud_api(self.config) self.running=True self.waiting_for_all_expired = False def join(self): self.waiting_for_all_expired = True threading.Thread.join(self) def terminateGC(self): self.running=False self.waiting_for_all_expired = True self.join() def run(self): while(self.running): if self.GC(): break sleep(1) def GC(self): exp, total = self.instanceMetric._instance_GC(self.cloud) print 'deleted %d instances from total %d instances' % (exp, total) if exp == total and self.waiting_for_all_expired: return True return False
def __init__(self, config): threading.Thread.__init__(self) self.config = config self.instanceMetric=ProvisionInstanceMetric() self.cloud = get_cloud_api(self.config) self.running=True self.waiting_for_all_expired = False
def __init__(self):
threading.Thread.__init__(self)
self.instance_metric = ProvisionInstanceMetric()
self.terminate_flag = 0
self.metaConfigure = MetaConfigure()
self.host_ip = self.metaConfigure._get_provision_host_ip() or '0.0.0.0'
self.host_port = self.metaConfigure._get_provision_host_port() or 8675
self.proxy_port = self.metaConfigure._get_proxy_port() or 8821
self.report_online = self.metaConfigure._is_report_online()
self.self_stopped = False
self.httpd = make_server('', self.host_port,
self.provision_monitor_app)
self.httpd.timeout = self.metaConfigure._get_http_timeout()
self.proxy_data = ProvisionProxyMonitorData(host_port=self.host_port)
def __init__(self, config): threading.Thread.__init__(self) self.config = config self.instanceMetric=ProvisionInstanceMetric() self.cloud = get_cloud_api(self.config) self.running=True self.instance_queue = Queue.Queue() self.instances_last_update_time = time.time() self.instances_next_update_time = time.time() self.instances_update_flag = True
def __init__(self):
threading.Thread.__init__(self)
self.instance_metric = ProvisionInstanceMetric()
self.terminate_flag = 0
self.metaConfigure = MetaConfigure()
self.host_ip = self.metaConfigure._get_provision_host_ip() or '0.0.0.0'
self.host_port = self.metaConfigure._get_provision_host_port() or 8675
self.proxy_port = self.metaConfigure._get_proxy_port() or 8821
self.report_online =self.metaConfigure._is_report_online()
self.self_stopped = False
self.httpd = make_server('', self.host_port, self.provision_monitor_app)
self.httpd.timeout = self.metaConfigure._get_http_timeout()
self.proxy_data = ProvisionProxyMonitorData(host_port = self.host_port)
class ProvisionMonitor(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.instance_metric = ProvisionInstanceMetric()
self.terminate_flag = 0
self.metaConfigure = MetaConfigure()
self.host_ip = self.metaConfigure._get_provision_host_ip() or '0.0.0.0'
self.host_port = self.metaConfigure._get_provision_host_port() or 8675
self.proxy_port = self.metaConfigure._get_proxy_port() or 8821
self.report_online =self.metaConfigure._is_report_online()
self.self_stopped = False
self.httpd = make_server('', self.host_port, self.provision_monitor_app)
self.httpd.timeout = self.metaConfigure._get_http_timeout()
self.proxy_data = ProvisionProxyMonitorData(host_port = self.host_port)
def terminate_monitor(self):
self.self_stopped = True
self.join()
def run(self):
self.serve()
def serve(self):
print "Serving on port %d..." % self.host_port
while not self.self_stopped:
self.httpd.handle_request()
if self.metaConfigure._is_debug():
print 'handle one request or timeout ...'
print "Stop monitor server..."
#self.httpd.serve_forever()
def provision_monitor_app(self, environ, start_response):
print 'one request coming'
status = '200 OK' # HTTP Status
headers = [('Content-type', 'text/plain')] # HTTP Headers
start_response(status, headers)
proxy_req = urlparse.parse_qs(environ['QUERY_STRING'])
print proxy_req.keys()
if 'VMType' in proxy_req.keys():
if proxy_req['VMType'] == ['proxy'] and 'PublicIP' in proxy_req.keys() and 'CurTime' in proxy_req.keys():
print "proxy private ip: %s" % self.proxy_data.priv_ip
#if proxy_req['PrivateIP'][0] != self.proxy_data.priv_ip :
if proxy_req['Key'][0] != self.proxy_data.key :
print "recieve old proxy vm connection: %s" % proxy_req
return ["Error: old provison proxy connection"]
self.proxy_data.connected_with_proxy=True
#if 'PrivateIP' in proxy_req.keys():
# self.proxy_data._set_pub_priv_ip(proxy_req['PublicIP'][0], proxy_req['PrivateIP'][0])
#else:
# self.proxy_data._set_pub_ip(proxy_req['PublicIP'][0])
monitor_base = time.time()
self.proxy_data._sync_time(proxy_req['CurTime'][0], monitor_base)
print self.proxy_data._get_info()
print "hahhah, proxy connection is coming!!!!"
return ["200 OK!!!%s" % monitor_base]
#if proxy_req['VMType'] == ['instance'] and 'VMStatus' in proxy_req.keys() and 'VMName' in proxy_req.keys() and 'VMIP' in proxy_req.keys() and 'Timestamp' in proxy_req.keys():
if proxy_req['VMType'] == ['instance'] and 'VMStatus' in proxy_req.keys() and 'VMName' in proxy_req.keys() and 'Timestamp' in proxy_req.keys():
instance = self.instance_metric._get_instance_by_name(proxy_req['VMName'][0])
if instance is None:
print 'can not find instance from metric'
print proxy_req['VMName']
return ["no found"]
if proxy_req['VMStatus'][0] == 'BOOTED' :
ip_lease = 0
if 'IPLease' in proxy_req.keys():
ip_lease = proxy_req['IPLease'][0]
ProvisionInstanceMetric()._update_instance(instance, "RUNNING", proxy_req['Timestamp'][0], ip_lease) #time.time())
print "instance[%s] has finished booted, booted time[%s], dhcp time:%s" % (proxy_req['VMName'], proxy_req['Timestamp'], str(proxy_req['IPLease'][0]))
elif proxy_req['VMStatus'][0] == 'DELETED' :
ProvisionInstanceMetric()._update_instance(instance, "DELETED", proxy_req['Timestamp'][0]) #time.time())
print "instance[%s] has been deleted, deleted time[%s]" % (proxy_req['VMName'], proxy_req['Timestamp'])
return ["200 OK"]
return ["Error:(%s) do not include \'hostname\', \'ip\' and \'status[boot|terminate]\'" % environ['QUERY_STRING']]
def __init__(self):
threading.Thread.__init__(self)
self.instance_metric = ProvisionInstanceMetric()
self.self_stopped = 0
self.metaConfigure = MetaConfigure()
class ProvisionMonitor(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.instance_metric = ProvisionInstanceMetric()
self.terminate_flag = 0
self.metaConfigure = MetaConfigure()
self.host_ip = self.metaConfigure._get_provision_host_ip() or '0.0.0.0'
self.host_port = self.metaConfigure._get_provision_host_port() or 8675
self.proxy_port = self.metaConfigure._get_proxy_port() or 8821
self.report_online = self.metaConfigure._is_report_online()
self.self_stopped = False
self.httpd = make_server('', self.host_port,
self.provision_monitor_app)
self.httpd.timeout = self.metaConfigure._get_http_timeout()
self.proxy_data = ProvisionProxyMonitorData(host_port=self.host_port)
def terminate_monitor(self):
self.self_stopped = True
self.join()
def run(self):
self.serve()
def serve(self):
print "Serving on port %d..." % self.host_port
while not self.self_stopped:
self.httpd.handle_request()
if self.metaConfigure._is_debug():
print 'handle one request or timeout ...'
print "Stop monitor server..."
#self.httpd.serve_forever()
def provision_monitor_app(self, environ, start_response):
print 'one request coming'
status = '200 OK' # HTTP Status
headers = [('Content-type', 'text/plain')] # HTTP Headers
start_response(status, headers)
proxy_req = urlparse.parse_qs(environ['QUERY_STRING'])
print proxy_req.keys()
if 'VMType' in proxy_req.keys():
if proxy_req['VMType'] == ['proxy'
] and 'PublicIP' in proxy_req.keys(
) and 'CurTime' in proxy_req.keys():
print "proxy private ip: %s" % self.proxy_data.priv_ip
#if proxy_req['PrivateIP'][0] != self.proxy_data.priv_ip :
if proxy_req['Key'][0] != self.proxy_data.key:
print "recieve old proxy vm connection: %s" % proxy_req
return ["Error: old provison proxy connection"]
self.proxy_data.connected_with_proxy = True
#if 'PrivateIP' in proxy_req.keys():
# self.proxy_data._set_pub_priv_ip(proxy_req['PublicIP'][0], proxy_req['PrivateIP'][0])
#else:
# self.proxy_data._set_pub_ip(proxy_req['PublicIP'][0])
monitor_base = time.time()
self.proxy_data._sync_time(proxy_req['CurTime'][0],
monitor_base)
print self.proxy_data._get_info()
print "hahhah, proxy connection is coming!!!!"
return ["200 OK!!!%s" % monitor_base]
#if proxy_req['VMType'] == ['instance'] and 'VMStatus' in proxy_req.keys() and 'VMName' in proxy_req.keys() and 'VMIP' in proxy_req.keys() and 'Timestamp' in proxy_req.keys():
if proxy_req['VMType'] == ['instance'
] and 'VMStatus' in proxy_req.keys(
) and 'VMName' in proxy_req.keys(
) and 'Timestamp' in proxy_req.keys():
instance = self.instance_metric._get_instance_by_name(
proxy_req['VMName'][0])
if instance is None:
print 'can not find instance from metric'
print proxy_req['VMName']
return ["no found"]
if proxy_req['VMStatus'][0] == 'BOOTED':
ip_lease = 0
if 'IPLease' in proxy_req.keys():
ip_lease = proxy_req['IPLease'][0]
ProvisionInstanceMetric()._update_instance(
instance, "RUNNING", proxy_req['Timestamp'][0],
ip_lease) #time.time())
print "instance[%s] has finished booted, booted time[%s], dhcp time:%s" % (
proxy_req['VMName'], proxy_req['Timestamp'],
str(proxy_req['IPLease'][0]))
elif proxy_req['VMStatus'][0] == 'DELETED':
ProvisionInstanceMetric()._update_instance(
instance, "DELETED",
proxy_req['Timestamp'][0]) #time.time())
print "instance[%s] has been deleted, deleted time[%s]" % (
proxy_req['VMName'], proxy_req['Timestamp'])
return ["200 OK"]
return [
"Error:(%s) do not include \'hostname\', \'ip\' and \'status[boot|terminate]\'"
% environ['QUERY_STRING']
]
def provision_monitor_app(self, environ, start_response):
print 'one request coming'
status = '200 OK' # HTTP Status
headers = [('Content-type', 'text/plain')] # HTTP Headers
start_response(status, headers)
proxy_req = urlparse.parse_qs(environ['QUERY_STRING'])
print proxy_req.keys()
if 'VMType' in proxy_req.keys():
if proxy_req['VMType'] == ['proxy'
] and 'PublicIP' in proxy_req.keys(
) and 'CurTime' in proxy_req.keys():
print "proxy private ip: %s" % self.proxy_data.priv_ip
#if proxy_req['PrivateIP'][0] != self.proxy_data.priv_ip :
if proxy_req['Key'][0] != self.proxy_data.key:
print "recieve old proxy vm connection: %s" % proxy_req
return ["Error: old provison proxy connection"]
self.proxy_data.connected_with_proxy = True
#if 'PrivateIP' in proxy_req.keys():
# self.proxy_data._set_pub_priv_ip(proxy_req['PublicIP'][0], proxy_req['PrivateIP'][0])
#else:
# self.proxy_data._set_pub_ip(proxy_req['PublicIP'][0])
monitor_base = time.time()
self.proxy_data._sync_time(proxy_req['CurTime'][0],
monitor_base)
print self.proxy_data._get_info()
print "hahhah, proxy connection is coming!!!!"
return ["200 OK!!!%s" % monitor_base]
#if proxy_req['VMType'] == ['instance'] and 'VMStatus' in proxy_req.keys() and 'VMName' in proxy_req.keys() and 'VMIP' in proxy_req.keys() and 'Timestamp' in proxy_req.keys():
if proxy_req['VMType'] == ['instance'
] and 'VMStatus' in proxy_req.keys(
) and 'VMName' in proxy_req.keys(
) and 'Timestamp' in proxy_req.keys():
instance = self.instance_metric._get_instance_by_name(
proxy_req['VMName'][0])
if instance is None:
print 'can not find instance from metric'
print proxy_req['VMName']
return ["no found"]
if proxy_req['VMStatus'][0] == 'BOOTED':
ip_lease = 0
if 'IPLease' in proxy_req.keys():
ip_lease = proxy_req['IPLease'][0]
ProvisionInstanceMetric()._update_instance(
instance, "RUNNING", proxy_req['Timestamp'][0],
ip_lease) #time.time())
print "instance[%s] has finished booted, booted time[%s], dhcp time:%s" % (
proxy_req['VMName'], proxy_req['Timestamp'],
str(proxy_req['IPLease'][0]))
elif proxy_req['VMStatus'][0] == 'DELETED':
ProvisionInstanceMetric()._update_instance(
instance, "DELETED",
proxy_req['Timestamp'][0]) #time.time())
print "instance[%s] has been deleted, deleted time[%s]" % (
proxy_req['VMName'], proxy_req['Timestamp'])
return ["200 OK"]
return [
"Error:(%s) do not include \'hostname\', \'ip\' and \'status[boot|terminate]\'"
% environ['QUERY_STRING']
]
def start_workload(self):
''' a serial of concurrency client behaviors
'''
monitor, proxy_instance = self._prepare_monitor_proxy()
if not proxy_instance:
print 'error while creating proxy instance...please check it'
return
reporter = ProvisionReporter(self.config)
reporter.start_reporter()
instanceGC = InstanceGC(self.config)
instanceGC.start()
clientGC = ClientGC(self.config)
clientGC.startGC()
client_queue = clientGC.get_client_queue()
instanceManager = InstanceManager(self.config)
instanceManager.start()
instance_queue = instanceManager.get_instance_queue()
begin_time = time.time()
if self.config.client_generate == "fixed":
#for i in range(self.config.client_number):
# pth = self.new_provision(client_iter, timeout)
# pth.start()
# thread_list.append(gth)
print 'not implement yet'
pass
else:
timeout = self.timeout
client_iter = 0
while (timeout > 0 and client_iter < self.config.client_number):
pth = self.new_provision(client_iter, timeout, instance_queue)
pth.start()
client_queue.put(pth)
#thread_list.append(pth)
client_iter += 1
sleep(self.config.client_interval)
timeout -= self.config.client_interval
clientGC.joinGC()
print "provision threads have terminated"
instanceManager.terminateIM()
print "instance mannage thread has terminate"
instanceGC.join()
print "instance GC thread has terminate"
reporter.terminate_reporter()
print "reporter thread has terminated"
monitor.terminate_monitor()
print "monitor thread has terminated"
total_time = time.time() - begin_time
print "total time: %d" % total_time
mtable = ProvisionInstanceMetric()._pretty_metric()
print mtable
filename = 'seq' + str(self.config.client_interval) + '_num' + str(
self.config.client_number) + '_life' + str(
self.config.min_lifetime) + ':' + str(
self.config.max_lifetime) + '_provision.res'
with open(filename, 'w') as f:
f.write(str(mtable))
f.close()
ip = ProvisionProxyMonitorData()._get_info()['pub']
proxy_instance.remove_floating_ip(ip)
proxy_instance.delete()