def clean_method(self):
timeout = 300
instances = getattr(self, 'instances', [])
keypair = getattr(self, '__keypair', None)
errors =[]
try:
if instances:
ins_ids = [str(x.id) for x in instances]
self.user.ec2.connection.terminate_instances(ins_ids)
self.user.ec2.terminate_instances(instances)
except Exception as E:
errors.append(E)
self.log.error('{0}\n{1}'.format(get_traceback(), E))
try:
if keypair:
self.user.ec2.delete_keypair(self.keypair)
except Exception as E:
errors.append(E)
self.log.error('{0}\n{1}'.format(get_traceback(), E))
try:
res_dict = self.global_xml_check(instances, timeout=timeout, present=False)
if 'FAILED' in res_dict.values():
raise RuntimeError('Test Failed. XML was still found for terminated instances using'
'Timeout:{0}. See table in test output'.format(timeout))
except Exception as E:
errors.append(E)
self.log.error('{0}\n{1}'.format(get_traceback(), E))
if errors:
ebuf = 'Errors during cleanup...\n'
for E in errors:
ebuf += "{0}\n".format(E)
raise RuntimeError(ebuf)
def tag(self):
try:
if getattr(self, '__tag', None) is None:
if getattr(self, '__xml', None) is not None:
self._tag = self.xml.tag
return self._tag
except Exception as E:
print '{0}\nFailed to fetch tag, err:{1}'.format(
get_traceback(), E)
self.log.error('{0}\nFailed to fetch tag, err:{1}'.format(
get_traceback(), E))
def name(self):
try:
if getattr(self, '__name', None) is None:
if self.xml is not None:
if 'name' in self.xml.attrib:
self._name = self.xml.attrib.get('name')
return self._name
except Exception as E:
print '{0}\nFailed to fetch name, err:{1}'.format(
get_traceback(), E)
self.log.error('{0}\nFailed to fetch name, err:{1}'.format(
get_traceback(), E))
def do_ssh(self, q, lock, name, command):
empty = False
q = q or None
while not empty:
try:
ssh = None
logger = None
self.logger.debug('Thread: {0}, in Q loop...'.format(name))
host = None
try:
host = q.get(timeout=self.maxwait)
except Empty:
empty = True
break
start = time.time()
try:
self.logger.debug('Connecting to new host:' + str(host))
logger = Eulogger(str(host))
ssh = SshConnection(host=host, username=self.username, password=self.password,
keypath=self.keypath, debug_connect=True,
timeout=self.args.timeout, verbose=True, logger=logger)
logger.debug('host: {0} running command:{1} '.format(host, command))
out = ssh.cmd(str(command), listformat=True, timeout=self.args.timeout,
get_pty=not(self.args.no_pty))
logger.debug('Done with host: {0}'.format(host))
with lock:
self.results[host] = {'status': out.get('status'),
'output': out.get('output'),
'elapsed': int(time.time() - start)}
except Exception as E:
err = "{0}\n{1}".format(get_traceback(), E)
with lock:
self.results[host] = {'status': -1,
'output': [err],
'elapsed': int(time.time() - start)}
finally:
logger.debug('Closing ssh to host: {0}'.format(host))
if ssh:
ssh.connection.close()
logger.debug('Closed ssh to host: {0}'.format(host))
try:
if logger:
logger.close()
except:
pass
except Exception as SE:
self.logger.error('{0}\nError in do_ssh:{0}'.format(get_traceback(), SE))
finally:
if q is not None and not empty:
q.task_done()
self.logger.debug('Finished task in thread:{0}'.format(name))
self.logger.debug('{0}: Done with thread'.format(name))
def __repr__(self):
try:
attrs = [self.__class__.__name__]
if self.tag:
attrs.append(str(self.tag))
if self.name:
attrs.append(str(self.name))
return ":".join(attrs)
except Exception as E:
print '{0}\nFailed to create repr, err:{1}'.format(
get_traceback(), E)
self.log.error('{0}\nFailed to create repr, err:{1}'.format(
get_traceback(), E))
def user_info(self, value):
if value is not None or not isinstance(value, dict):
msg = "user_info must be of type None or dict"
self.log.error('{0}\n{1}'.format(get_traceback(), msg))
raise ValueError(msg)
else:
self._user_info = value
def update(self, new_prop=None, silent=True):
"""
Updates this property obj
:params silent: bool, if True will not raise Exceptions found during lookup, will instead
write errors to self.connection.err_method()
:returns : self upon successful update, otherwise returns None
"""
errmsg = ""
if not new_prop:
try:
new_prop = self.connection.get_property(self)
except Exception as LE:
if silent:
errmsg = "{0}\n{1}\n".format(get_traceback(), str(LE))
self.connection.err_method('{0}Update failed for property:{1}'
.format(errmsg, self.name))
return None
else:
raise
if not isinstance(new_prop, EucaProperty):
raise ValueError('"{0}" update error. Non EucaProperty type for new_prop. Found: '
'"{1}/{2}"'.format(self.name, new_prop, type(new_prop)))
if new_prop:
self.__dict__.update(new_prop.__dict__)
return self
def update(self, new_service=None, silent=True):
"""
Updates this service obj
:params silent: bool, if True will not raise Exceptions found during lookup, will instead
write errors to self.connection.err_method()
:returns : self upon successful update, otherwise returns None
"""
errmsg = ""
if not new_service:
if not self.name:
raise ValueError(
'Must set "name" before using update(). Name:{0}'.format(
self.name))
try:
new_service = self.connection.get_services(
service_type=self.type, service_names=self.name)[0]
except Exception as LE:
if silent:
errmsg = "{0}\n{1}\n".format(get_traceback(), str(LE))
self.connection.err_method(
'{0}Update failed. Service: {1} not found'.format(
errmsg, self.name))
return None
else:
raise
if not isinstance(new_service, EucaService):
raise ValueError(
'"{0}" update error. Non EucaService type for new_prop. Found: '
'"{1}/{2}"'.format(self.name, new_service, type(new_service)))
if new_service:
self.__dict__.update(new_service.__dict__)
return self
def get_proxy_instance(self, zone):
if not zone:
raise ValueError('Must provide zone for get_proxy_instance. Got:"{0}"'.format(zone))
proxy_instances = getattr(self, '_proxy_instances', {})
pi = proxy_instances.get(zone, None)
if pi:
try:
pi.update()
if pi.status != "running":
try:
pi.terminate()
except:
pass
pi = None
except Exception as E:
self.log.debug('{0}\nIgnoring error caught while fetching proxy instance '
'status:"{1}'.format(get_traceback(), E))
pi = None
if not pi:
subnet = self.user.ec2.get_default_subnets(zone=zone)
if not subnet:
raise ValueError('No default subnet for zone:{0} to create proxy instance in'
.format(zone))
subnet = subnet[0]
pi = self.user.ec2.run_image(image=self.emi, keypair=self.keypair, group=self.group,
subnet_id = subnet.id, zone=zone,
type=self.args.proxy_vmtype,
systemconnection=self.tc.sysadmin)[0]
proxy_instances[zone] = pi
self._proxy_instances = proxy_instances
return pi
def update(self, new_account=None, silent=True):
"""
Base update method for updating component service objs
:params new_account: optional new_account object to be used to update this account
:params silent: bool, if True will not raise Exceptions found during lookup, will instead
write errors to self.connection.err_method()
:returns : self upon successful update, otherwise returns None
"""
errmsg = ""
if not new_account:
try:
if self.id:
new_account = self.connection.get_account_by_id(account_id=self.id)
else:
new_account = self.connection.get_account_by_name(account_name=self.name)
except Exception as LE:
if silent:
errmsg = "{0}\n{1}\n".format(get_traceback(), str(LE))
self.connection.log.error('{0}Update failed. IamAccount: {1} not found'
.format(errmsg, self.name))
return None
else:
raise
if not isinstance(new_account, self.__class__):
raise ValueError('"{0}" update error. Non {1} type for new_account. '
'Found: "{2}/{3}"'.format(self.name,
self.__class__.__name__,
new_account,
type(new_account)))
if new_account:
self.__dict__.update(new_account.__dict__)
return self
def update(self, new_prop=None, silent=True):
"""
Updates this property obj
:params silent: bool, if True will not raise Exceptions found during lookup, will instead
write errors to self.connection.err_method()
:returns : self upon successful update, otherwise returns None
"""
errmsg = ""
if not new_prop:
try:
new_prop = self.connection.get_property(self)
except Exception as LE:
if silent:
errmsg = "{0}\n{1}\n".format(get_traceback(), str(LE))
self.connection.err_method('{0}Update failed for property:{1}'
.format(errmsg, self.name))
return None
else:
raise
if not isinstance(new_prop, EucaProperty):
raise ValueError('"{0}" update error. Non EucaProperty type for new_prop. Found: '
'"{1}/{2}"'.format(self.name, new_prop, type(new_prop)))
if new_prop:
self.__dict__.update(new_prop.__dict__)
return self
def get_hosts_for_node_controllers(self, partition=None, instanceid=None):
if instanceid is not None and not isinstance(instanceid, basestring):
raise ValueError('Instance id not of string type, got:"{0}"/"{1}"'
.format(instanceid, type(instanceid)))
ncs = self.get_hosts_by_service_type(servicetype='node')
if not partition and not instanceid:
return ncs
retlist = []
if instanceid:
try:
reservation = self.ec2_connection.get_all_instances(instance_ids=[instanceid])
except:
self.log.error('{0}\nFailed to find instance:"{1}" on system'
.format(get_traceback(), instanceid))
return []
if reservation:
instance = reservation[0].instances[0]
node_addr = instance.tags.get('euca:node')
if node_addr:
for nc in ncs:
if nc.hostname == node_addr:
return [nc]
if partition and partition in nc.partitions:
retlist.append(nc)
return retlist
def get_hosts_for_node_controllers(self, partition=None, instanceid=None):
if instanceid is not None and not isinstance(instanceid, basestring):
raise ValueError(
'Instance id not of string type, got:"{0}"/"{1}"'.format(
instanceid, type(instanceid)))
ncs = self.get_hosts_by_service_type(servicetype='node')
if not partition and not instanceid:
return ncs
retlist = []
if instanceid:
try:
reservation = self.ec2_connection.get_all_instances(
instance_ids=['verbose', instanceid])
except:
self.log.error(
'{0}\nFailed to find instance:"{1}" on system'.format(
get_traceback(), instanceid))
return []
if reservation:
instance = reservation[0].instances[0]
node_addr = instance.tags.get('euca:node')
if node_addr:
for nc in ncs:
if nc.hostname == node_addr:
return [nc]
if partition and partition in nc.partitions:
retlist.append(nc)
return retlist
def complete(self, text, state):
"""Return the next possible completion for 'text'.
If a command has not been entered, then complete against command list.
Otherwise try to call complete_<command> to get list of completions.
"""
try:
self.dprint('{0}.complete(text="{1}", state="{2}")'.format(
self.name, text, state))
origline = readline.get_line_buffer() or ""
try:
if state == 0:
line = origline.lstrip()
stripped = len(origline) - len(line)
begidx = readline.get_begidx() - stripped
endidx = readline.get_endidx() - stripped
menu = self.get_submenu_completer_for_text(origline)
readline.set_completion_display_matches_hook(
menu._completer_display)
self.dprint(
'Complete(): text:{0}, origline:{1}, begidx:{2}, endidz:{3}'
.format(text, line, begidx, endidx))
if begidx >= 0:
#cmd, args, foo = self.parseline(line)
cmd, args, foo = menu.parseline(text)
compfunc = menu.completedefault
if cmd and hasattr(
menu,
'complete_' + cmd,
):
compfunc = getattr(menu, 'complete_' + cmd)
menu.dprint('Complete(): got method complete_' +
str(cmd))
else:
menu.dprint('Complete(): non-zero state sending to '
'completenames(), state:{0}'.format(state))
compfunc = menu.completenames
try:
self.completion_matches = compfunc(
text, line, begidx, endidx)
except:
print_exc()
raise
try:
self.dprint(
'Returning {0}.complete(text={1}, state={2}) = "{3}"'.
format(self.name, text, state,
self.completion_matches[state]))
return self.completion_matches[state]
except IndexError:
return None
finally:
readline.set_completion_display_matches_hook(
self._completer_display)
except Exception as E:
# readline will often fail silently, and not always show/raise errors
self.stderr.write('{0}\nError in complete: "{1}"'.format(
get_traceback(), E))
raise
def update(self, new_account=None, silent=True):
"""
Base update method for updating component service objs
:params new_account: optional new_account object to be used to update this account
:params silent: bool, if True will not raise Exceptions found during lookup, will instead
write errors to self.connection.err_method()
:returns : self upon successful update, otherwise returns None
"""
errmsg = ""
if not new_account:
try:
if self.id:
new_account = self.connection.get_account_by_id(
account_id=self.id)
else:
new_account = self.connection.get_account_by_name(
account_name=self.name)
except Exception as LE:
if silent:
errmsg = "{0}\n{1}\n".format(get_traceback(), str(LE))
self.connection.log.error(
'{0}Update failed. IamAccount: {1} not found'.format(
errmsg, self.name))
return None
else:
raise
if not isinstance(new_account, self.__class__):
raise ValueError(
'"{0}" update error. Non {1} type for new_account. '
'Found: "{2}/{3}"'.format(self.name, self.__class__.__name__,
new_account, type(new_account)))
if new_account:
self.__dict__.update(new_account.__dict__)
return self
def user_info(self, value):
if value is not None or not isinstance(value, dict):
msg = "user_info must be of type None or dict"
self.log.error('{0}\n{1}'.format(get_traceback(), msg))
raise ValueError(msg)
else:
self._user_info = value
def run(self):
if self.verbose:
self.debug('Starting Sender...\n')
if self.ssh is None:
send_packet(destip=self.dest_ip,
proto=self.proto,
dstport=self.port,
count=self.count,
interval=interval,
verbose=self.verbose)
else:
try:
self.result = remote_sender(ssh=self.ssh,
dst_addr=self.dest_ip,
proto=self.proto,
port=self.port,
srcport=self.srcport,
count=self.count,
interval=interval,
verbose=self.verbose,
socktimeout=self.socktimeout,
cb=self.packet_test_cb,
cbargs=[verbose])
except Exception as E:
self.error = "{0}\nERROR IN PACKET SENDER:{1}".format(
get_traceback(), str(E))
self.done_time = time.time()
def update(self, new_service=None, silent=True):
"""
Updates this service obj
:params silent: bool, if True will not raise Exceptions found during lookup, will instead
write errors to self.connection.err_method()
:returns : self upon successful update, otherwise returns None
"""
errmsg = ""
if not new_service:
if not self.name:
raise ValueError('Must set "name" before using update(). Name:{0}'.format(self.name))
try:
new_service = self.connection.get_services(service_type=self.type, service_names=self.name)[0]
except Exception as LE:
if silent:
errmsg = "{0}\n{1}\n".format(get_traceback(), str(LE))
self.connection.err_method("{0}Update failed. Service: {1} not found".format(errmsg, self.name))
return None
else:
raise
if not isinstance(new_service, EucaService):
raise ValueError(
'"{0}" update error. Non EucaService type for new_prop. Found: '
'"{1}/{2}"'.format(self.name, new_service, type(new_service))
)
if new_service:
self.__dict__.update(new_service.__dict__)
return self
def cred_depot(self):
if not self._cred_depot and self._cred_depot_connection_info.get('hostname'):
try:
self._cred_depot = Machine(**self._cred_depot_connection_info)
except Exception as E:
self.log.error('{0}\nError connecting to cred depot machine:"{1}"'
.format(get_traceback(), E))
raise E
return self._cred_depot
def get_last_capacity_status(self):
"""
Attempts to find and parse the last reported status line from nc.log
Sample line:
returning status=enabled cores=30/32 mem=7538/8050 disk=47/57
"""
last = getattr(self, '__capacity_status', None)
if last:
if (time.time() - last.get('updated')) <= 5:
return last
ret = {
"status": None,
'updated': 0,
'cores': None,
'cores_total': None,
'mem': None,
'mem_total': None,
'disk': None,
'disk_total': None
}
if str(self.eucahost.eucalyptus_conf.LOGLEVEL).lower() not in [
'debug', 'trace'
]:
self.log.debug(
'Cant fetch capacity status from node with loglevel: DEBUG < "{0}"'
.format(self.eucahost.eucalyptus_conf.LOGLEVEL))
return ret
euca_path = self.eucahost.eucalyptus_home or ""
nclog = os.path.join(euca_path, 'var/log/eucalyptus/nc.log')
try:
out = self.sys(
'tac {0} | grep -m1 "returning status="'.format(nclog),
code=0,
listformat=False)
if out:
timestamp = re.search("^(\d+-\d+-\d+\s+\d+:\d+:\d+)\s", out)
if timestamp:
ret['timestamp'] = timestamp.group(1)
else:
ret['timestamp'] = 'unknown'
for val in ['status', 'cores', 'mem', 'disk']:
search = "{0}=(\S+)".format(val)
grp = re.search(search, out)
if grp:
if val == 'status':
ret[val] = grp.group(1)
else:
cur, tot = grp.group(1).split('/')
ret[val] = int(cur)
ret[val + "_total"] = int(tot)
ret['updated'] = time.time()
setattr(self, '__capacity_status', ret)
except CommandExitCodeException as CE:
self.log.warn('{0}\nError fetching nc status:"{1}"'.format(
get_traceback(), str(CE)))
return ret
def packet_test_scenario(
self,
zone1,
zone2,
sec_group1,
sec_group_2,
vpc1,
vpc2,
subnet1,
subnet2,
use_private,
protocol,
pkt_count=5,
retries=2,
verbose=None,
):
vpc = self.default_vpc
results = {}
start = time.time()
if verbose is None:
if self.args.log_level == "DEBUG":
verbose = 2
else:
verbose = 0
sec_group = self.get_test_security_groups(vpc=vpc, count=1, rules=self.DEFAULT_RULES)[0]
try:
for zone in self.zones:
self.log.info('STARTING PACKET TEST AGAINST ZONE:"{0}"'.format(zone))
ins1, ins2 = self.get_test_instances(zone=zone, group_id=sec_group.id, count=2)
for retry in xrange(1, retries + 1):
try:
pkt_dict = packet_test(
sender_ssh=ins1.ssh, receiver_ssh=ins2.ssh, protocol=1, count=pkt_count, verbose=verbose
)
if pkt_dict.get("error", None) or (pkt_dict.get("count") != pkt_count):
raise RuntimeError("Packet test failed, results:{0}".format(pkt_dict))
self.log.debug("Results for Zone: {0}\n{1}".format(zone, pkt_dict))
results[zone] = pkt_dict
except Exception as PE:
self.log.error(
"{0}\nPacket Test for zone: {1} failed attempt:{2}/{3}".format(
get_traceback(), zone, retry, retries
)
)
wait = 30 - int(time.time() - start)
if wait > 0:
self.log.debug('Waiting "{0}" seconds to retry packet test'.format(wait))
time.sleep(wait)
if zone not in results:
raise RuntimeError("Failed packet test for zone: {0}".format(zone))
finally:
for zone, pkt_dict in results.iteritems():
self.show_packet_test_results(
pkt_dict, header="test2_icmp_packet_test_same_az_and_sg. " "Zone:{0}".format(zone)
)
self.log.info("test2_icmp_packet_test_same_az_and_sg passed")
def validate_instance_dns(self=self):
try:
for instance in instances:
if not re.search("internal", instance.private_dns_name):
self.user.ec2.debug(
"Did not find instance DNS enabled, skipping test")
self.set_instances(instances)
return instances
self.log.debug(
'\n'
'# Test to see if Dynamic DNS has been configured \n'
'# Per AWS standard, resolution should have private hostname or '
'private IP as a valid response\n'
'# Perform DNS resolution against public IP and public DNS name\n'
'# Perform DNS resolution against private IP and private DNS name\n'
'# Check to see if nslookup was able to resolve\n')
assert isinstance(instance, EuInstance)
install_bind_utils_on_instance(instance)
self.log.debug(
'Check nslookup to resolve public DNS Name to local-ipv4 address'
)
self.assertTrue(
instance.found("nslookup " + instance.public_dns_name,
instance.private_ip_address),
"Incorrect DNS resolution for hostname.")
self.log.debug('Check nslookup to resolve public-ipv4 '
'address to public DNS name')
if self.managed_network:
self.assertTrue(
instance.found("nslookup " + instance.ip_address,
instance.public_dns_name),
"Incorrect DNS resolution for public IP address")
self.log.debug(
'Check nslookup to resolve private DNS Name to local-ipv4 address'
)
if self.managed_network:
self.assertTrue(
instance.found(
"nslookup " + instance.private_dns_name,
instance.private_ip_address),
"Incorrect DNS resolution for private hostname.")
self.log.debug(
'Check nslookup to resolve local-ipv4 address to private DNS name'
)
self.assertTrue(
instance.found(
"nslookup " + instance.private_ip_address,
instance.private_dns_name),
"Incorrect DNS resolution for private IP address")
self.log.debug('Attempt to ping instance public_dns_name')
self.assertTrue(ping(instance.public_dns_name))
return True
except Exception, e:
self.log.error('{0}\nValidate_instance_dns error:"{1}"'.format(
get_traceback(), e))
return False
def cleanup(user):
try:
user.ec2.terminate_instances()
for key in user.ec2.test_resources["keypairs"]:
user.ec2.delete_keypair(key)
remove(key.name + ".pem")
except Exception as E:
my_error = {"user": str(user), "error": "{0}\nError:{1}".format(get_traceback(), E)}
with userlock:
errors.append(my_error)
def try_assume_admin_on_clc(self):
if not self.aws_secret_key and not self.aws_access_key:
try:
self.assume_role_on_remote_clc()
res = try_serviceconnection(self)
return res
except Exception as AE:
self.debug('{0}\nFailed to update creds using '
'"clcadmin-assume-system-credentials", err:"{1}"'
.format(get_traceback(), str(AE)))
def test2_monitor_instances_to_running_and_connect(self, instances=None, timeout=300):
ins = instances or getattr(self, 'instances', None)
if ins is None:
raise ValueError('Instances were not found to monitor or connect. '
'Run test1 first or provide a list of euinstances to this test')
try:
self.user.ec2.monitor_euinstances_to_running(ins, timeout=timeout)
except Exception as E:
self.log.error("{0}\nError(s) during monitor_instances_to_running: {1}"
.format(get_traceback(), E))
def cleanup(user):
try:
user.ec2.terminate_instances()
for key in user.ec2.test_resources['keypairs']:
user.ec2.delete_keypair(key)
remove(key.name + ".pem")
except Exception as E:
my_error = {'user':str(user), 'error':'{0}\nError:{1}'.format(get_traceback(), E)}
with userlock:
errors.append(my_error)
def get_all_vm_type_info(self):
vm_types = []
get_zone = [str(self.name)]
get_zone.append('verbose')
found = False
try:
myzone = self.connection.get_all_zones(zones=get_zone)
except Exception, e:
tb = get_traceback()
raise Exception(str(tb) + '\n Could not get zone:' + str(self.name) + "\n" + str(e))
def subnet(self):
if self._subnet is None:
if self.args.subnet:
try:
self._subnet = self.user.ec2.get_subnet(self.args.subnet)
except Exception as E:
self.log.error(
red('{0}\nFailed to fetch CLI provided subnet:"{1}", ERR:"{2}"'
.format(get_traceback(), self.args.subnet, E)))
return self._subnet
def sysadmin(self):
if not self._sysadmin:
try:
self._sysadmin = SystemConnection(**self._system_connection_info)
except Exception as TE:
self.log.error('{0}\nCould not create sysadmin interface, timed out: "{1}"'
.format(get_traceback(), TE))
raise TE
return self._sysadmin
def user(self):
if not self._test_user:
try:
self._test_user = self.create_user_using_cloudadmin(
**self._test_user_connection_info)
except Exception as E:
self.log.error('{0}\nError Creating test user, error:"{1}"'
.format(get_traceback(), E))
raise E
return self._test_user
def boto2(self):
if not self._b2_connection:
try:
self._b2_connection = self.boto2_connect(
verbose=self._connection_kwargs.get('verbose'),
conn_kwargs=self._connection_kwargs)
except Exception as CE:
self.log.error(red('{0}\nFailed to create boto2 "{1}" connection. Err:"{2}"'
.format(get_traceback(), self.__class__.__name__, CE)))
raise
return self._b2_connection
def sysadmin(self):
if not self._sysadmin:
if not self._system_connection_info.get('hostname', None):
return None
try:
self._sysadmin = SystemConnection(**self._system_connection_info)
except Exception as TE:
self.log.error('{0}\nCould not create sysadmin interface, timed out: "{1}"'
.format(get_traceback(), TE))
raise TE
return self._sysadmin
def boto3(self):
if not self._b3_connection:
try:
self._b3_connection = B3Session(ops=self,
connection_kwargs=self._connection_kwargs,
verbose=self._connection_kwargs.get('verbose'))
except Exception as CE:
self.log.error(red('{0}\nFailed to create boto3 "{1}" session. Err:"{2}"'
.format(get_traceback(), self.__class__.__name__, CE)))
raise
return self._b3_connection
def clean_method(self):
if self.args.no_clean_on_exit:
self.log.info('"no_clean_on_exit" set. Skipping Clean method')
return
task = self.current_task
err_buf = ""
if not task:
return
try:
if task.instance:
self.user.ec2.terminate_single_instance(task.instance)
except Exception as E:
msg = 'Error terminating task instance, err: {0}'.format(E)
err_buf += msg + "\n"
self.log.error("{0}\n{1}".format(get_traceback(), msg))
try:
if task.volumes:
self.user.ec2.delete_volumes(task.volumes)
except Exception as E:
msg = 'Error deleting task volumes, err: {0}'.format(E)
err_buf += msg + "\n"
self.log.error("{0}\n{1}".format(get_traceback(), msg))
try:
for keypair in self._created_keypairs:
self.user.ec2.delete_keypair(keypair)
except Exception as E:
msg = 'Error deleting keypair, err: {0}'.format(E)
err_buf += msg + "\n"
self.log.error("{0}\n{1}".format(get_traceback(), msg))
try:
# Delete the security group only if this test created it
if self._group and getattr(self, '_created_group', False):
self.user.ec2.delete_group(self._group)
except Exception as E:
msg = 'Error deleting security group, err: {0}'.format(E)
err_buf += msg + "\n"
self.log.error("{0}\n{1}".format(get_traceback(), msg))
if err_buf:
raise RuntimeError(err_buf)
else:
self.log.info('Clean method completed')
def clean_method(self):
errors = []
try:
if self.instances:
self.user.ec2.terminate_instances(self.instances)
except Exception as E:
self.log.error(red(get_traceback()))
errors.append(E)
try:
if self.volumes:
delete = []
for volume in self.volumes:
try:
volume.update()
if volume.status != 'deleted':
delete.append(volume)
except EC2ResponseError as ER:
if ER.status == 400 and ER.error_code == 'InvalidVolume.NotFound':
pass
if delete:
self.user.ec2.delete_volumes(delete)
except Exception as E:
self.log.error(red(get_traceback()))
errors.append(E)
try:
if self._keypair:
self.user.ec2.delete_keypair(self.keypair)
except Exception as E:
self.log.error(red(get_traceback()))
errors.append(E)
try:
if self._group:
self.user.ec2.delete_group(self.group)
except Exception as E:
self.log.error(red(get_traceback()))
errors.append(E)
if errors:
buf = "The following errors occurred during test cleanup:"
for error in errors:
buf += "\n{0}".format(error)
raise RuntimeError(buf)
def clean_method(self):
errors = []
try:
if self.instances:
self.user.ec2.terminate_instances(self.instances)
except Exception as E:
self.log.error(red(get_traceback()))
errors.append(E)
try:
if self.volumes:
delete = []
for volume in self.volumes:
try:
volume.update()
if volume.status != 'deleted':
delete.append(volume)
except EC2ResponseError as ER:
if ER.status == 400 and ER.error_code == 'InvalidVolume.NotFound':
pass
if delete:
self.user.ec2.delete_volumes(delete)
except Exception as E:
self.log.error(red(get_traceback()))
errors.append(E)
try:
if self._keypair:
self.user.ec2.delete_keypair(self.keypair)
except Exception as E:
self.log.error(red(get_traceback()))
errors.append(E)
try:
if self._group:
self.user.ec2.delete_group(self.group)
except Exception as E:
self.log.error(red(get_traceback()))
errors.append(E)
if errors:
buf = "The following errors occurred during test cleanup:"
for error in errors:
buf += "\n{0}".format(error)
raise RuntimeError(buf)
def clean_method(self):
if self.args.no_clean_on_exit:
self.log.info('"no_clean_on_exit" set. Skipping Clean method')
return
task = self.current_task
err_buf = ""
if not task:
return
try:
if task.instance:
self.user.ec2.terminate_single_instance(task.instance)
except Exception as E:
msg = 'Error terminating task instance, err: {0}'.format(E)
err_buf += msg + "\n"
self.log.error("{0}\n{1}".format(get_traceback(), msg))
try:
if task.volumes:
self.user.ec2.delete_volumes(task.volumes)
except Exception as E:
msg = 'Error deleting task volumes, err: {0}'.format(E)
err_buf += msg + "\n"
self.log.error("{0}\n{1}".format(get_traceback(), msg))
try:
for keypair in self._created_keypairs:
self.user.ec2.delete_keypair(keypair)
except Exception as E:
msg = 'Error deleting keypair, err: {0}'.format(E)
err_buf += msg + "\n"
self.log.error("{0}\n{1}".format(get_traceback(), msg))
try:
# Delete the security group only if this test created it
if self._group and getattr(self, '_created_group', False):
self.user.ec2.delete_group(self._group)
except Exception as E:
msg = 'Error deleting security group, err: {0}'.format(E)
err_buf += msg + "\n"
self.log.error("{0}\n{1}".format(get_traceback(), msg))
if err_buf:
raise RuntimeError(err_buf)
else:
self.log.info('Clean method completed')
def machine(self):
if not self._machine:
try:
if self.host:
self._machine = Machine(hostname=self.host,
password=self.password,
keypath=self.keypath)
except Exception as E:
self.log.warning(
'{0}\nFailed to create machine object to host:"{1}", error:"{2}"'
.format(get_traceback(), self.host, E))
return self._machine
def global_xml(self):
try:
if not self._global_xml:
self._global_xml = GlobalXML(
xml_element=self._get_global_xml_root(),
eucanetd=self,
log_level=self.log.stdout_level)
return self._global_xml
except Exception as E:
self.log.error(
"{0}\nFailed to create global xml element. Error:{1}".format(
get_traceback(), E))
def get_all_vm_type_info(self):
vm_types = []
get_zone = [str(self.name)]
get_zone.append('verbose')
found = False
try:
myzone = self.connection.get_all_zones(zones=get_zone)
except Exception, e:
tb = get_traceback()
raise Exception(
str(tb) + '\n Could not get zone:' + str(self.name) + "\n" +
str(e))
def emi(self):
emi = getattr(self, '__emi', None)
if not emi:
try:
if self.args.emi:
emi = self.user.ec2.get_emi(emi=self.args.emi)
else:
emi = self.user.ec2.get_emi()
setattr(self, '__emi', emi)
except Exception as E:
self.log.error("{0}\nFailed to fetch EMI:{1}".format(get_traceback(), E))
return emi
def boto2(self):
if not self._b2_connection:
try:
self._b2_connection = self.boto2_connect(
verbose=self._connection_kwargs.get('verbose'),
conn_kwargs=self._connection_kwargs)
except Exception as CE:
self.log.error(
red('{0}\nFailed to create boto2 "{1}" connection. Err:"{2}"'
.format(get_traceback(), self.__class__.__name__, CE)))
raise
return self._b2_connection
def user(self):
try:
if not self._user:
if self.args.access_key and self.args.secret_key and self.args.region:
self._user = UserContext(aws_access_key=self.args.access_key,
aws_secret_key=self.args.secret_key,
region=self.args.region)
if (self.args.clc or self.args.environment_file) and self.tc:
self._user = self.tc.user
except Exception as UE:
self.log.error('{0}\nFailed to create user: {1}'.format(get_traceback(), UE))
return self._user
def domain(self):
if self._domain is None:
prop_name = 'system.dns.dnsdomain'
try:
# First try from the cloud property...
region_prop = self.sysadmin.get_property(prop_name)
if region_prop.value:
self._domain = region_prop.value
except Exception as E:
self.log.error('{0}\nError fetching cloud property:"{1}". Error:"{2}"'
.format(get_traceback(), prop_name, E))
return self._domain or self.region
def try_serviceconnection(self):
if self.aws_secret_key and self.aws_access_key and self._clc_ip:
self._connect_services()
try:
res = self.update_attrs_from_cloud_services()
if res:
self.debug('Derived creds from serviceconnection')
return res
except RuntimeError as RE:
self.debug('{0}\nFailed to update creds using serviceconnection, err:"{1}"'
.format(get_traceback(), str(RE)))
self._close_adminpi()
def get_location(url, depth, maxdepth):
if depth > maxdepth:
raise ValueError(
'Max redirects limit has been reached:{0}/{1}'.format(
depth, maxdepth))
conn = None
try:
url = url.replace('http://', '')
host = url.split('/')[0]
path = url.replace(host, '')
res = None
err = None
retries = 5
for retry in xrange(0, retries):
try:
debug('HTTP HEAD request for: {0}, attempt:{1}/{2}'.
format(url, retry, retries))
conn = httplib.HTTPConnection(host)
conn.request("HEAD", path)
res = conn.getresponse()
break
except Exception as HE:
err = '{0}\nError attempting to fetch url:{1}, attempt:{2}/{3}, ' \
'error:{4}'.format(get_traceback(), url, retry, retries, HE)
debug(err)
time.sleep(retry)
if not res:
err = err or "Error retrieving url:{0}".format(url)
raise RuntimeError(err)
location = res.getheader('location')
if location and location != url:
depth += 1
debug('Redirecting: depth:{0}, url:{1}'.format(
depth, location))
return get_location(location,
depth=depth,
maxdepth=maxdepth)
else:
content_length = res.getheader('content-length')
if content_length is None:
raise ValueError(
'No content-length header found for url:{0}'.
format(url))
fbytes = int(content_length)
return fbytes
except Exception as HE:
debug('Failed to fetch content-length header from url:{0}'.
format(url))
raise HE
finally:
if conn:
conn.close()
def boto3(self):
if not self._b3_connection:
try:
self._b3_connection = B3Session(
ops=self,
connection_kwargs=self._connection_kwargs,
verbose=self._connection_kwargs.get('verbose'))
except Exception as CE:
self.log.error(
red('{0}\nFailed to create boto3 "{1}" session. Err:"{2}"'.
format(get_traceback(), self.__class__.__name__, CE)))
raise
return self._b3_connection
def dump_conn_debug(self, info):
"""
Helper method to format and display the connection info contained in a specific dict.
Example: self.dump_conn_debug(self._system_connection_info)
:param info: connection dict.
"""
try:
self.log.debug(
'Connection info:\n{0}'
.format("\n".join("{0}:{1}".format(x, y) for x,y in info.iteritems())))
except Exception as doh:
self.log.error('{0}\nError attempting to dump connection info:{1}'
.format(get_traceback(), doh))
def _start_session(self, connection_kwargs=None):
if self._ops._user_context:
self._session = self._ops._user_context.session
else:
try:
region = self._region
self._session = Session(aws_access_key_id=self._access_key,
aws_secret_access_key=self._secret_key,
region_name=region)
except Exception as SE:
self._log.error(red('{0}\nError creating boto3 {1} session. Error:{2}'
.format(get_traceback(), self.__class__.__name__, SE)))
raise
return self._session
def group(self, group):
if group is None:
self._group = group
else:
if isinstance(group, basestring):
group_name = group
else:
group_name = group.name
try:
self._group = self._get_security_group(group_name=group_name)
except Exception as E:
self.log.error("{0}\nError setting up security group, err:{1}"
.format(get_traceback(), E))
raise E
def cloudformation(self):
ops_class = CFNops
name = self.CLASS_MAP[ops_class.__name__]
if not self._connections.get(name, None):
if getattr(self, ops_class.EUCARC_URL_NAME, None):
try:
self._connections[name] = ops_class(**self._connection_kwargs)
except Exception as CE:
self.log.error(red('{0}\nFailed to created "{1}" interface.\n'
'Connection kwargs:\n{2}\nError:{3}'
.format(get_traceback(),
ops_class.__name__,
self._connection_kwargs,
CE)))
return self._connections.get(name, None)
def wait_for_mido_metadata_nat_rule(instances, addtime=0, interval=2, timeout=300):
if networkmode != "VPCMIDO":
return {}
start = time.time()
elapsed = 0
ret_dict = {}
errbuff = ""
waiting = copy.copy(instances)
while waiting and elapsed < timeout:
errbuff = ""
for instance in instances:
if instance in waiting:
instance.update()
rule = None
try:
rule = md.get_instance_bridge_port_metadata_nat_rule(instance.id)
except Exception as E:
errbuff += (
"{0}\nERROR fetching meta nat rule, elapsed:{1}, instance_state:{"
"2}, Error:{3}".format(get_traceback(), elapsed, instance.state, E)
)
if not int(elapsed) % 10:
md.log.error(
"ERROR fetching meta nat rule, elapsed:{0}, "
"instance_state: {1}, Error:{2}".format(elapsed, instance.state, E)
)
if rule:
tc.log.info("Got MD rule for instance:{0} after elapsed{1}".format(instance.id, elapsed))
try:
md.show_rules(rule)
except:
pass
elapsed = time.time() - start
ret_dict[instance.id] = elapsed + addtime
waiting.remove(instance)
else:
if instance.state == "terminated":
waiting.remove(instance)
elapsed = time.time() - start
ret_dict[instance.id] = "instance terminated after:{0}".format(elapsed)
elapsed = time.time() - start
if elapsed < timeout and waiting:
time.sleep(interval)
if errbuff:
tc.log.error(errbuff)
for instance in waiting:
ret_dict[instance.id] = "Timed out waiting for MD after: {0}".format(int(elapsed))
return ret_dict
def get_last_capacity_status(self):
"""
Attempts to find and parse the last reported status line from nc.log
Sample line:
returning status=enabled cores=30/32 mem=7538/8050 disk=47/57
"""
last = getattr(self, '__capacity_status', None)
if last:
if (time.time() - last.get('updated')) <= 5:
return last
ret = {"status": None,
'updated': 0,
'cores': None,
'cores_total': None,
'mem': None,
'mem_total': None,
'disk': None,
'disk_total': None}
if str(self.eucahost.eucalyptus_conf.LOGLEVEL).lower() not in ['debug', 'trace']:
self.log.debug('Cant fetch capacity status from node with loglevel: DEBUG < "{0}"'
.format(self.eucahost.eucalyptus_conf.LOG_LEVEL))
return ret
euca_path = self.eucahost.get_eucalyptus_home() or ""
nclog = os.path.join(euca_path, 'var/log/eucalyptus/nc.log')
try:
out = self.sys('tac {0} | grep -m1 "returning status="'.format(nclog), code=0,
listformat=False)
if out:
timestamp = re.search("^(\d+-\d+-\d+\s+\d+:\d+:\d+)\s", out)
if timestamp:
ret['timestamp'] = timestamp.group(1)
else:
ret['timestamp'] = 'unknown'
for val in ['status', 'cores', 'mem', 'disk']:
search = "{0}=(\S+)".format(val)
grp = re.search(search, out)
if grp:
if val == 'status':
ret[val] = grp.group(1)
else:
cur, tot = grp.group(1).split('/')
ret[val] = int(cur)
ret[val + "_total"] = int(tot)
ret['updated'] = time.time()
setattr(self, '__capacity_status', ret)
except CommandExitCodeException as CE:
self.log.warn('{0}\nError fetching nc status:"{1}"'.format(get_traceback(), str(CE)))
return ret
