def clean_method(self): timeout = 300 instances = getattr(self, 'instances', []) keypair = getattr(self, '__keypair', None) errors =[] try: if instances: ins_ids = [str(x.id) for x in instances] self.user.ec2.connection.terminate_instances(ins_ids) self.user.ec2.terminate_instances(instances) except Exception as E: errors.append(E) self.log.error('{0}\n{1}'.format(get_traceback(), E)) try: if keypair: self.user.ec2.delete_keypair(self.keypair) except Exception as E: errors.append(E) self.log.error('{0}\n{1}'.format(get_traceback(), E)) try: res_dict = self.global_xml_check(instances, timeout=timeout, present=False) if 'FAILED' in res_dict.values(): raise RuntimeError('Test Failed. XML was still found for terminated instances using' 'Timeout:{0}. See table in test output'.format(timeout)) except Exception as E: errors.append(E) self.log.error('{0}\n{1}'.format(get_traceback(), E)) if errors: ebuf = 'Errors during cleanup...\n' for E in errors: ebuf += "{0}\n".format(E) raise RuntimeError(ebuf)
def tag(self): try: if getattr(self, '__tag', None) is None: if getattr(self, '__xml', None) is not None: self._tag = self.xml.tag return self._tag except Exception as E: print '{0}\nFailed to fetch tag, err:{1}'.format( get_traceback(), E) self.log.error('{0}\nFailed to fetch tag, err:{1}'.format( get_traceback(), E))
def name(self): try: if getattr(self, '__name', None) is None: if self.xml is not None: if 'name' in self.xml.attrib: self._name = self.xml.attrib.get('name') return self._name except Exception as E: print '{0}\nFailed to fetch name, err:{1}'.format( get_traceback(), E) self.log.error('{0}\nFailed to fetch name, err:{1}'.format( get_traceback(), E))
def do_ssh(self, q, lock, name, command): empty = False q = q or None while not empty: try: ssh = None logger = None self.logger.debug('Thread: {0}, in Q loop...'.format(name)) host = None try: host = q.get(timeout=self.maxwait) except Empty: empty = True break start = time.time() try: self.logger.debug('Connecting to new host:' + str(host)) logger = Eulogger(str(host)) ssh = SshConnection(host=host, username=self.username, password=self.password, keypath=self.keypath, debug_connect=True, timeout=self.args.timeout, verbose=True, logger=logger) logger.debug('host: {0} running command:{1} '.format(host, command)) out = ssh.cmd(str(command), listformat=True, timeout=self.args.timeout, get_pty=not(self.args.no_pty)) logger.debug('Done with host: {0}'.format(host)) with lock: self.results[host] = {'status': out.get('status'), 'output': out.get('output'), 'elapsed': int(time.time() - start)} except Exception as E: err = "{0}\n{1}".format(get_traceback(), E) with lock: self.results[host] = {'status': -1, 'output': [err], 'elapsed': int(time.time() - start)} finally: logger.debug('Closing ssh to host: {0}'.format(host)) if ssh: ssh.connection.close() logger.debug('Closed ssh to host: {0}'.format(host)) try: if logger: logger.close() except: pass except Exception as SE: self.logger.error('{0}\nError in do_ssh:{0}'.format(get_traceback(), SE)) finally: if q is not None and not empty: q.task_done() self.logger.debug('Finished task in thread:{0}'.format(name)) self.logger.debug('{0}: Done with thread'.format(name))
def __repr__(self): try: attrs = [self.__class__.__name__] if self.tag: attrs.append(str(self.tag)) if self.name: attrs.append(str(self.name)) return ":".join(attrs) except Exception as E: print '{0}\nFailed to create repr, err:{1}'.format( get_traceback(), E) self.log.error('{0}\nFailed to create repr, err:{1}'.format( get_traceback(), E))
def user_info(self, value): if value is not None or not isinstance(value, dict): msg = "user_info must be of type None or dict" self.log.error('{0}\n{1}'.format(get_traceback(), msg)) raise ValueError(msg) else: self._user_info = value
def update(self, new_prop=None, silent=True): """ Updates this property obj :params silent: bool, if True will not raise Exceptions found during lookup, will instead write errors to self.connection.err_method() :returns : self upon successful update, otherwise returns None """ errmsg = "" if not new_prop: try: new_prop = self.connection.get_property(self) except Exception as LE: if silent: errmsg = "{0}\n{1}\n".format(get_traceback(), str(LE)) self.connection.err_method('{0}Update failed for property:{1}' .format(errmsg, self.name)) return None else: raise if not isinstance(new_prop, EucaProperty): raise ValueError('"{0}" update error. Non EucaProperty type for new_prop. Found: ' '"{1}/{2}"'.format(self.name, new_prop, type(new_prop))) if new_prop: self.__dict__.update(new_prop.__dict__) return self
def update(self, new_service=None, silent=True): """ Updates this service obj :params silent: bool, if True will not raise Exceptions found during lookup, will instead write errors to self.connection.err_method() :returns : self upon successful update, otherwise returns None """ errmsg = "" if not new_service: if not self.name: raise ValueError( 'Must set "name" before using update(). Name:{0}'.format( self.name)) try: new_service = self.connection.get_services( service_type=self.type, service_names=self.name)[0] except Exception as LE: if silent: errmsg = "{0}\n{1}\n".format(get_traceback(), str(LE)) self.connection.err_method( '{0}Update failed. Service: {1} not found'.format( errmsg, self.name)) return None else: raise if not isinstance(new_service, EucaService): raise ValueError( '"{0}" update error. Non EucaService type for new_prop. Found: ' '"{1}/{2}"'.format(self.name, new_service, type(new_service))) if new_service: self.__dict__.update(new_service.__dict__) return self
def get_proxy_instance(self, zone): if not zone: raise ValueError('Must provide zone for get_proxy_instance. Got:"{0}"'.format(zone)) proxy_instances = getattr(self, '_proxy_instances', {}) pi = proxy_instances.get(zone, None) if pi: try: pi.update() if pi.status != "running": try: pi.terminate() except: pass pi = None except Exception as E: self.log.debug('{0}\nIgnoring error caught while fetching proxy instance ' 'status:"{1}'.format(get_traceback(), E)) pi = None if not pi: subnet = self.user.ec2.get_default_subnets(zone=zone) if not subnet: raise ValueError('No default subnet for zone:{0} to create proxy instance in' .format(zone)) subnet = subnet[0] pi = self.user.ec2.run_image(image=self.emi, keypair=self.keypair, group=self.group, subnet_id = subnet.id, zone=zone, type=self.args.proxy_vmtype, systemconnection=self.tc.sysadmin)[0] proxy_instances[zone] = pi self._proxy_instances = proxy_instances return pi
def update(self, new_account=None, silent=True): """ Base update method for updating component service objs :params new_account: optional new_account object to be used to update this account :params silent: bool, if True will not raise Exceptions found during lookup, will instead write errors to self.connection.err_method() :returns : self upon successful update, otherwise returns None """ errmsg = "" if not new_account: try: if self.id: new_account = self.connection.get_account_by_id(account_id=self.id) else: new_account = self.connection.get_account_by_name(account_name=self.name) except Exception as LE: if silent: errmsg = "{0}\n{1}\n".format(get_traceback(), str(LE)) self.connection.log.error('{0}Update failed. IamAccount: {1} not found' .format(errmsg, self.name)) return None else: raise if not isinstance(new_account, self.__class__): raise ValueError('"{0}" update error. Non {1} type for new_account. ' 'Found: "{2}/{3}"'.format(self.name, self.__class__.__name__, new_account, type(new_account))) if new_account: self.__dict__.update(new_account.__dict__) return self
def get_hosts_for_node_controllers(self, partition=None, instanceid=None): if instanceid is not None and not isinstance(instanceid, basestring): raise ValueError('Instance id not of string type, got:"{0}"/"{1}"' .format(instanceid, type(instanceid))) ncs = self.get_hosts_by_service_type(servicetype='node') if not partition and not instanceid: return ncs retlist = [] if instanceid: try: reservation = self.ec2_connection.get_all_instances(instance_ids=[instanceid]) except: self.log.error('{0}\nFailed to find instance:"{1}" on system' .format(get_traceback(), instanceid)) return [] if reservation: instance = reservation[0].instances[0] node_addr = instance.tags.get('euca:node') if node_addr: for nc in ncs: if nc.hostname == node_addr: return [nc] if partition and partition in nc.partitions: retlist.append(nc) return retlist
def get_hosts_for_node_controllers(self, partition=None, instanceid=None): if instanceid is not None and not isinstance(instanceid, basestring): raise ValueError( 'Instance id not of string type, got:"{0}"/"{1}"'.format( instanceid, type(instanceid))) ncs = self.get_hosts_by_service_type(servicetype='node') if not partition and not instanceid: return ncs retlist = [] if instanceid: try: reservation = self.ec2_connection.get_all_instances( instance_ids=['verbose', instanceid]) except: self.log.error( '{0}\nFailed to find instance:"{1}" on system'.format( get_traceback(), instanceid)) return [] if reservation: instance = reservation[0].instances[0] node_addr = instance.tags.get('euca:node') if node_addr: for nc in ncs: if nc.hostname == node_addr: return [nc] if partition and partition in nc.partitions: retlist.append(nc) return retlist
def complete(self, text, state): """Return the next possible completion for 'text'. If a command has not been entered, then complete against command list. Otherwise try to call complete_<command> to get list of completions. """ try: self.dprint('{0}.complete(text="{1}", state="{2}")'.format( self.name, text, state)) origline = readline.get_line_buffer() or "" try: if state == 0: line = origline.lstrip() stripped = len(origline) - len(line) begidx = readline.get_begidx() - stripped endidx = readline.get_endidx() - stripped menu = self.get_submenu_completer_for_text(origline) readline.set_completion_display_matches_hook( menu._completer_display) self.dprint( 'Complete(): text:{0}, origline:{1}, begidx:{2}, endidz:{3}' .format(text, line, begidx, endidx)) if begidx >= 0: #cmd, args, foo = self.parseline(line) cmd, args, foo = menu.parseline(text) compfunc = menu.completedefault if cmd and hasattr( menu, 'complete_' + cmd, ): compfunc = getattr(menu, 'complete_' + cmd) menu.dprint('Complete(): got method complete_' + str(cmd)) else: menu.dprint('Complete(): non-zero state sending to ' 'completenames(), state:{0}'.format(state)) compfunc = menu.completenames try: self.completion_matches = compfunc( text, line, begidx, endidx) except: print_exc() raise try: self.dprint( 'Returning {0}.complete(text={1}, state={2}) = "{3}"'. format(self.name, text, state, self.completion_matches[state])) return self.completion_matches[state] except IndexError: return None finally: readline.set_completion_display_matches_hook( self._completer_display) except Exception as E: # readline will often fail silently, and not always show/raise errors self.stderr.write('{0}\nError in complete: "{1}"'.format( get_traceback(), E)) raise
def update(self, new_account=None, silent=True): """ Base update method for updating component service objs :params new_account: optional new_account object to be used to update this account :params silent: bool, if True will not raise Exceptions found during lookup, will instead write errors to self.connection.err_method() :returns : self upon successful update, otherwise returns None """ errmsg = "" if not new_account: try: if self.id: new_account = self.connection.get_account_by_id( account_id=self.id) else: new_account = self.connection.get_account_by_name( account_name=self.name) except Exception as LE: if silent: errmsg = "{0}\n{1}\n".format(get_traceback(), str(LE)) self.connection.log.error( '{0}Update failed. IamAccount: {1} not found'.format( errmsg, self.name)) return None else: raise if not isinstance(new_account, self.__class__): raise ValueError( '"{0}" update error. Non {1} type for new_account. ' 'Found: "{2}/{3}"'.format(self.name, self.__class__.__name__, new_account, type(new_account))) if new_account: self.__dict__.update(new_account.__dict__) return self
def run(self): if self.verbose: self.debug('Starting Sender...\n') if self.ssh is None: send_packet(destip=self.dest_ip, proto=self.proto, dstport=self.port, count=self.count, interval=interval, verbose=self.verbose) else: try: self.result = remote_sender(ssh=self.ssh, dst_addr=self.dest_ip, proto=self.proto, port=self.port, srcport=self.srcport, count=self.count, interval=interval, verbose=self.verbose, socktimeout=self.socktimeout, cb=self.packet_test_cb, cbargs=[verbose]) except Exception as E: self.error = "{0}\nERROR IN PACKET SENDER:{1}".format( get_traceback(), str(E)) self.done_time = time.time()
def update(self, new_service=None, silent=True): """ Updates this service obj :params silent: bool, if True will not raise Exceptions found during lookup, will instead write errors to self.connection.err_method() :returns : self upon successful update, otherwise returns None """ errmsg = "" if not new_service: if not self.name: raise ValueError('Must set "name" before using update(). Name:{0}'.format(self.name)) try: new_service = self.connection.get_services(service_type=self.type, service_names=self.name)[0] except Exception as LE: if silent: errmsg = "{0}\n{1}\n".format(get_traceback(), str(LE)) self.connection.err_method("{0}Update failed. Service: {1} not found".format(errmsg, self.name)) return None else: raise if not isinstance(new_service, EucaService): raise ValueError( '"{0}" update error. Non EucaService type for new_prop. Found: ' '"{1}/{2}"'.format(self.name, new_service, type(new_service)) ) if new_service: self.__dict__.update(new_service.__dict__) return self
def cred_depot(self): if not self._cred_depot and self._cred_depot_connection_info.get('hostname'): try: self._cred_depot = Machine(**self._cred_depot_connection_info) except Exception as E: self.log.error('{0}\nError connecting to cred depot machine:"{1}"' .format(get_traceback(), E)) raise E return self._cred_depot
def get_last_capacity_status(self): """ Attempts to find and parse the last reported status line from nc.log Sample line: returning status=enabled cores=30/32 mem=7538/8050 disk=47/57 """ last = getattr(self, '__capacity_status', None) if last: if (time.time() - last.get('updated')) <= 5: return last ret = { "status": None, 'updated': 0, 'cores': None, 'cores_total': None, 'mem': None, 'mem_total': None, 'disk': None, 'disk_total': None } if str(self.eucahost.eucalyptus_conf.LOGLEVEL).lower() not in [ 'debug', 'trace' ]: self.log.debug( 'Cant fetch capacity status from node with loglevel: DEBUG < "{0}"' .format(self.eucahost.eucalyptus_conf.LOGLEVEL)) return ret euca_path = self.eucahost.eucalyptus_home or "" nclog = os.path.join(euca_path, 'var/log/eucalyptus/nc.log') try: out = self.sys( 'tac {0} | grep -m1 "returning status="'.format(nclog), code=0, listformat=False) if out: timestamp = re.search("^(\d+-\d+-\d+\s+\d+:\d+:\d+)\s", out) if timestamp: ret['timestamp'] = timestamp.group(1) else: ret['timestamp'] = 'unknown' for val in ['status', 'cores', 'mem', 'disk']: search = "{0}=(\S+)".format(val) grp = re.search(search, out) if grp: if val == 'status': ret[val] = grp.group(1) else: cur, tot = grp.group(1).split('/') ret[val] = int(cur) ret[val + "_total"] = int(tot) ret['updated'] = time.time() setattr(self, '__capacity_status', ret) except CommandExitCodeException as CE: self.log.warn('{0}\nError fetching nc status:"{1}"'.format( get_traceback(), str(CE))) return ret
def packet_test_scenario( self, zone1, zone2, sec_group1, sec_group_2, vpc1, vpc2, subnet1, subnet2, use_private, protocol, pkt_count=5, retries=2, verbose=None, ): vpc = self.default_vpc results = {} start = time.time() if verbose is None: if self.args.log_level == "DEBUG": verbose = 2 else: verbose = 0 sec_group = self.get_test_security_groups(vpc=vpc, count=1, rules=self.DEFAULT_RULES)[0] try: for zone in self.zones: self.log.info('STARTING PACKET TEST AGAINST ZONE:"{0}"'.format(zone)) ins1, ins2 = self.get_test_instances(zone=zone, group_id=sec_group.id, count=2) for retry in xrange(1, retries + 1): try: pkt_dict = packet_test( sender_ssh=ins1.ssh, receiver_ssh=ins2.ssh, protocol=1, count=pkt_count, verbose=verbose ) if pkt_dict.get("error", None) or (pkt_dict.get("count") != pkt_count): raise RuntimeError("Packet test failed, results:{0}".format(pkt_dict)) self.log.debug("Results for Zone: {0}\n{1}".format(zone, pkt_dict)) results[zone] = pkt_dict except Exception as PE: self.log.error( "{0}\nPacket Test for zone: {1} failed attempt:{2}/{3}".format( get_traceback(), zone, retry, retries ) ) wait = 30 - int(time.time() - start) if wait > 0: self.log.debug('Waiting "{0}" seconds to retry packet test'.format(wait)) time.sleep(wait) if zone not in results: raise RuntimeError("Failed packet test for zone: {0}".format(zone)) finally: for zone, pkt_dict in results.iteritems(): self.show_packet_test_results( pkt_dict, header="test2_icmp_packet_test_same_az_and_sg. " "Zone:{0}".format(zone) ) self.log.info("test2_icmp_packet_test_same_az_and_sg passed")
def validate_instance_dns(self=self): try: for instance in instances: if not re.search("internal", instance.private_dns_name): self.user.ec2.debug( "Did not find instance DNS enabled, skipping test") self.set_instances(instances) return instances self.log.debug( '\n' '# Test to see if Dynamic DNS has been configured \n' '# Per AWS standard, resolution should have private hostname or ' 'private IP as a valid response\n' '# Perform DNS resolution against public IP and public DNS name\n' '# Perform DNS resolution against private IP and private DNS name\n' '# Check to see if nslookup was able to resolve\n') assert isinstance(instance, EuInstance) install_bind_utils_on_instance(instance) self.log.debug( 'Check nslookup to resolve public DNS Name to local-ipv4 address' ) self.assertTrue( instance.found("nslookup " + instance.public_dns_name, instance.private_ip_address), "Incorrect DNS resolution for hostname.") self.log.debug('Check nslookup to resolve public-ipv4 ' 'address to public DNS name') if self.managed_network: self.assertTrue( instance.found("nslookup " + instance.ip_address, instance.public_dns_name), "Incorrect DNS resolution for public IP address") self.log.debug( 'Check nslookup to resolve private DNS Name to local-ipv4 address' ) if self.managed_network: self.assertTrue( instance.found( "nslookup " + instance.private_dns_name, instance.private_ip_address), "Incorrect DNS resolution for private hostname.") self.log.debug( 'Check nslookup to resolve local-ipv4 address to private DNS name' ) self.assertTrue( instance.found( "nslookup " + instance.private_ip_address, instance.private_dns_name), "Incorrect DNS resolution for private IP address") self.log.debug('Attempt to ping instance public_dns_name') self.assertTrue(ping(instance.public_dns_name)) return True except Exception, e: self.log.error('{0}\nValidate_instance_dns error:"{1}"'.format( get_traceback(), e)) return False
def cleanup(user): try: user.ec2.terminate_instances() for key in user.ec2.test_resources["keypairs"]: user.ec2.delete_keypair(key) remove(key.name + ".pem") except Exception as E: my_error = {"user": str(user), "error": "{0}\nError:{1}".format(get_traceback(), E)} with userlock: errors.append(my_error)
def try_assume_admin_on_clc(self): if not self.aws_secret_key and not self.aws_access_key: try: self.assume_role_on_remote_clc() res = try_serviceconnection(self) return res except Exception as AE: self.debug('{0}\nFailed to update creds using ' '"clcadmin-assume-system-credentials", err:"{1}"' .format(get_traceback(), str(AE)))
def test2_monitor_instances_to_running_and_connect(self, instances=None, timeout=300): ins = instances or getattr(self, 'instances', None) if ins is None: raise ValueError('Instances were not found to monitor or connect. ' 'Run test1 first or provide a list of euinstances to this test') try: self.user.ec2.monitor_euinstances_to_running(ins, timeout=timeout) except Exception as E: self.log.error("{0}\nError(s) during monitor_instances_to_running: {1}" .format(get_traceback(), E))
def cleanup(user): try: user.ec2.terminate_instances() for key in user.ec2.test_resources['keypairs']: user.ec2.delete_keypair(key) remove(key.name + ".pem") except Exception as E: my_error = {'user':str(user), 'error':'{0}\nError:{1}'.format(get_traceback(), E)} with userlock: errors.append(my_error)
def get_all_vm_type_info(self): vm_types = [] get_zone = [str(self.name)] get_zone.append('verbose') found = False try: myzone = self.connection.get_all_zones(zones=get_zone) except Exception, e: tb = get_traceback() raise Exception(str(tb) + '\n Could not get zone:' + str(self.name) + "\n" + str(e))
def subnet(self): if self._subnet is None: if self.args.subnet: try: self._subnet = self.user.ec2.get_subnet(self.args.subnet) except Exception as E: self.log.error( red('{0}\nFailed to fetch CLI provided subnet:"{1}", ERR:"{2}"' .format(get_traceback(), self.args.subnet, E))) return self._subnet
def sysadmin(self): if not self._sysadmin: try: self._sysadmin = SystemConnection(**self._system_connection_info) except Exception as TE: self.log.error('{0}\nCould not create sysadmin interface, timed out: "{1}"' .format(get_traceback(), TE)) raise TE return self._sysadmin
def user(self): if not self._test_user: try: self._test_user = self.create_user_using_cloudadmin( **self._test_user_connection_info) except Exception as E: self.log.error('{0}\nError Creating test user, error:"{1}"' .format(get_traceback(), E)) raise E return self._test_user
def boto2(self): if not self._b2_connection: try: self._b2_connection = self.boto2_connect( verbose=self._connection_kwargs.get('verbose'), conn_kwargs=self._connection_kwargs) except Exception as CE: self.log.error(red('{0}\nFailed to create boto2 "{1}" connection. Err:"{2}"' .format(get_traceback(), self.__class__.__name__, CE))) raise return self._b2_connection
def sysadmin(self): if not self._sysadmin: if not self._system_connection_info.get('hostname', None): return None try: self._sysadmin = SystemConnection(**self._system_connection_info) except Exception as TE: self.log.error('{0}\nCould not create sysadmin interface, timed out: "{1}"' .format(get_traceback(), TE)) raise TE return self._sysadmin
def boto3(self): if not self._b3_connection: try: self._b3_connection = B3Session(ops=self, connection_kwargs=self._connection_kwargs, verbose=self._connection_kwargs.get('verbose')) except Exception as CE: self.log.error(red('{0}\nFailed to create boto3 "{1}" session. Err:"{2}"' .format(get_traceback(), self.__class__.__name__, CE))) raise return self._b3_connection
def clean_method(self): if self.args.no_clean_on_exit: self.log.info('"no_clean_on_exit" set. Skipping Clean method') return task = self.current_task err_buf = "" if not task: return try: if task.instance: self.user.ec2.terminate_single_instance(task.instance) except Exception as E: msg = 'Error terminating task instance, err: {0}'.format(E) err_buf += msg + "\n" self.log.error("{0}\n{1}".format(get_traceback(), msg)) try: if task.volumes: self.user.ec2.delete_volumes(task.volumes) except Exception as E: msg = 'Error deleting task volumes, err: {0}'.format(E) err_buf += msg + "\n" self.log.error("{0}\n{1}".format(get_traceback(), msg)) try: for keypair in self._created_keypairs: self.user.ec2.delete_keypair(keypair) except Exception as E: msg = 'Error deleting keypair, err: {0}'.format(E) err_buf += msg + "\n" self.log.error("{0}\n{1}".format(get_traceback(), msg)) try: # Delete the security group only if this test created it if self._group and getattr(self, '_created_group', False): self.user.ec2.delete_group(self._group) except Exception as E: msg = 'Error deleting security group, err: {0}'.format(E) err_buf += msg + "\n" self.log.error("{0}\n{1}".format(get_traceback(), msg)) if err_buf: raise RuntimeError(err_buf) else: self.log.info('Clean method completed')
def clean_method(self): errors = [] try: if self.instances: self.user.ec2.terminate_instances(self.instances) except Exception as E: self.log.error(red(get_traceback())) errors.append(E) try: if self.volumes: delete = [] for volume in self.volumes: try: volume.update() if volume.status != 'deleted': delete.append(volume) except EC2ResponseError as ER: if ER.status == 400 and ER.error_code == 'InvalidVolume.NotFound': pass if delete: self.user.ec2.delete_volumes(delete) except Exception as E: self.log.error(red(get_traceback())) errors.append(E) try: if self._keypair: self.user.ec2.delete_keypair(self.keypair) except Exception as E: self.log.error(red(get_traceback())) errors.append(E) try: if self._group: self.user.ec2.delete_group(self.group) except Exception as E: self.log.error(red(get_traceback())) errors.append(E) if errors: buf = "The following errors occurred during test cleanup:" for error in errors: buf += "\n{0}".format(error) raise RuntimeError(buf)
def machine(self): if not self._machine: try: if self.host: self._machine = Machine(hostname=self.host, password=self.password, keypath=self.keypath) except Exception as E: self.log.warning( '{0}\nFailed to create machine object to host:"{1}", error:"{2}"' .format(get_traceback(), self.host, E)) return self._machine
def global_xml(self): try: if not self._global_xml: self._global_xml = GlobalXML( xml_element=self._get_global_xml_root(), eucanetd=self, log_level=self.log.stdout_level) return self._global_xml except Exception as E: self.log.error( "{0}\nFailed to create global xml element. Error:{1}".format( get_traceback(), E))
def get_all_vm_type_info(self): vm_types = [] get_zone = [str(self.name)] get_zone.append('verbose') found = False try: myzone = self.connection.get_all_zones(zones=get_zone) except Exception, e: tb = get_traceback() raise Exception( str(tb) + '\n Could not get zone:' + str(self.name) + "\n" + str(e))
def emi(self): emi = getattr(self, '__emi', None) if not emi: try: if self.args.emi: emi = self.user.ec2.get_emi(emi=self.args.emi) else: emi = self.user.ec2.get_emi() setattr(self, '__emi', emi) except Exception as E: self.log.error("{0}\nFailed to fetch EMI:{1}".format(get_traceback(), E)) return emi
def boto2(self): if not self._b2_connection: try: self._b2_connection = self.boto2_connect( verbose=self._connection_kwargs.get('verbose'), conn_kwargs=self._connection_kwargs) except Exception as CE: self.log.error( red('{0}\nFailed to create boto2 "{1}" connection. Err:"{2}"' .format(get_traceback(), self.__class__.__name__, CE))) raise return self._b2_connection
def user(self): try: if not self._user: if self.args.access_key and self.args.secret_key and self.args.region: self._user = UserContext(aws_access_key=self.args.access_key, aws_secret_key=self.args.secret_key, region=self.args.region) if (self.args.clc or self.args.environment_file) and self.tc: self._user = self.tc.user except Exception as UE: self.log.error('{0}\nFailed to create user: {1}'.format(get_traceback(), UE)) return self._user
def domain(self): if self._domain is None: prop_name = 'system.dns.dnsdomain' try: # First try from the cloud property... region_prop = self.sysadmin.get_property(prop_name) if region_prop.value: self._domain = region_prop.value except Exception as E: self.log.error('{0}\nError fetching cloud property:"{1}". Error:"{2}"' .format(get_traceback(), prop_name, E)) return self._domain or self.region
def try_serviceconnection(self): if self.aws_secret_key and self.aws_access_key and self._clc_ip: self._connect_services() try: res = self.update_attrs_from_cloud_services() if res: self.debug('Derived creds from serviceconnection') return res except RuntimeError as RE: self.debug('{0}\nFailed to update creds using serviceconnection, err:"{1}"' .format(get_traceback(), str(RE))) self._close_adminpi()
def get_location(url, depth, maxdepth): if depth > maxdepth: raise ValueError( 'Max redirects limit has been reached:{0}/{1}'.format( depth, maxdepth)) conn = None try: url = url.replace('http://', '') host = url.split('/')[0] path = url.replace(host, '') res = None err = None retries = 5 for retry in xrange(0, retries): try: debug('HTTP HEAD request for: {0}, attempt:{1}/{2}'. format(url, retry, retries)) conn = httplib.HTTPConnection(host) conn.request("HEAD", path) res = conn.getresponse() break except Exception as HE: err = '{0}\nError attempting to fetch url:{1}, attempt:{2}/{3}, ' \ 'error:{4}'.format(get_traceback(), url, retry, retries, HE) debug(err) time.sleep(retry) if not res: err = err or "Error retrieving url:{0}".format(url) raise RuntimeError(err) location = res.getheader('location') if location and location != url: depth += 1 debug('Redirecting: depth:{0}, url:{1}'.format( depth, location)) return get_location(location, depth=depth, maxdepth=maxdepth) else: content_length = res.getheader('content-length') if content_length is None: raise ValueError( 'No content-length header found for url:{0}'. format(url)) fbytes = int(content_length) return fbytes except Exception as HE: debug('Failed to fetch content-length header from url:{0}'. format(url)) raise HE finally: if conn: conn.close()
def boto3(self): if not self._b3_connection: try: self._b3_connection = B3Session( ops=self, connection_kwargs=self._connection_kwargs, verbose=self._connection_kwargs.get('verbose')) except Exception as CE: self.log.error( red('{0}\nFailed to create boto3 "{1}" session. Err:"{2}"'. format(get_traceback(), self.__class__.__name__, CE))) raise return self._b3_connection
def dump_conn_debug(self, info): """ Helper method to format and display the connection info contained in a specific dict. Example: self.dump_conn_debug(self._system_connection_info) :param info: connection dict. """ try: self.log.debug( 'Connection info:\n{0}' .format("\n".join("{0}:{1}".format(x, y) for x,y in info.iteritems()))) except Exception as doh: self.log.error('{0}\nError attempting to dump connection info:{1}' .format(get_traceback(), doh))
def _start_session(self, connection_kwargs=None): if self._ops._user_context: self._session = self._ops._user_context.session else: try: region = self._region self._session = Session(aws_access_key_id=self._access_key, aws_secret_access_key=self._secret_key, region_name=region) except Exception as SE: self._log.error(red('{0}\nError creating boto3 {1} session. Error:{2}' .format(get_traceback(), self.__class__.__name__, SE))) raise return self._session
def group(self, group): if group is None: self._group = group else: if isinstance(group, basestring): group_name = group else: group_name = group.name try: self._group = self._get_security_group(group_name=group_name) except Exception as E: self.log.error("{0}\nError setting up security group, err:{1}" .format(get_traceback(), E)) raise E
def cloudformation(self): ops_class = CFNops name = self.CLASS_MAP[ops_class.__name__] if not self._connections.get(name, None): if getattr(self, ops_class.EUCARC_URL_NAME, None): try: self._connections[name] = ops_class(**self._connection_kwargs) except Exception as CE: self.log.error(red('{0}\nFailed to created "{1}" interface.\n' 'Connection kwargs:\n{2}\nError:{3}' .format(get_traceback(), ops_class.__name__, self._connection_kwargs, CE))) return self._connections.get(name, None)
def wait_for_mido_metadata_nat_rule(instances, addtime=0, interval=2, timeout=300): if networkmode != "VPCMIDO": return {} start = time.time() elapsed = 0 ret_dict = {} errbuff = "" waiting = copy.copy(instances) while waiting and elapsed < timeout: errbuff = "" for instance in instances: if instance in waiting: instance.update() rule = None try: rule = md.get_instance_bridge_port_metadata_nat_rule(instance.id) except Exception as E: errbuff += ( "{0}\nERROR fetching meta nat rule, elapsed:{1}, instance_state:{" "2}, Error:{3}".format(get_traceback(), elapsed, instance.state, E) ) if not int(elapsed) % 10: md.log.error( "ERROR fetching meta nat rule, elapsed:{0}, " "instance_state: {1}, Error:{2}".format(elapsed, instance.state, E) ) if rule: tc.log.info("Got MD rule for instance:{0} after elapsed{1}".format(instance.id, elapsed)) try: md.show_rules(rule) except: pass elapsed = time.time() - start ret_dict[instance.id] = elapsed + addtime waiting.remove(instance) else: if instance.state == "terminated": waiting.remove(instance) elapsed = time.time() - start ret_dict[instance.id] = "instance terminated after:{0}".format(elapsed) elapsed = time.time() - start if elapsed < timeout and waiting: time.sleep(interval) if errbuff: tc.log.error(errbuff) for instance in waiting: ret_dict[instance.id] = "Timed out waiting for MD after: {0}".format(int(elapsed)) return ret_dict
def get_last_capacity_status(self): """ Attempts to find and parse the last reported status line from nc.log Sample line: returning status=enabled cores=30/32 mem=7538/8050 disk=47/57 """ last = getattr(self, '__capacity_status', None) if last: if (time.time() - last.get('updated')) <= 5: return last ret = {"status": None, 'updated': 0, 'cores': None, 'cores_total': None, 'mem': None, 'mem_total': None, 'disk': None, 'disk_total': None} if str(self.eucahost.eucalyptus_conf.LOGLEVEL).lower() not in ['debug', 'trace']: self.log.debug('Cant fetch capacity status from node with loglevel: DEBUG < "{0}"' .format(self.eucahost.eucalyptus_conf.LOG_LEVEL)) return ret euca_path = self.eucahost.get_eucalyptus_home() or "" nclog = os.path.join(euca_path, 'var/log/eucalyptus/nc.log') try: out = self.sys('tac {0} | grep -m1 "returning status="'.format(nclog), code=0, listformat=False) if out: timestamp = re.search("^(\d+-\d+-\d+\s+\d+:\d+:\d+)\s", out) if timestamp: ret['timestamp'] = timestamp.group(1) else: ret['timestamp'] = 'unknown' for val in ['status', 'cores', 'mem', 'disk']: search = "{0}=(\S+)".format(val) grp = re.search(search, out) if grp: if val == 'status': ret[val] = grp.group(1) else: cur, tot = grp.group(1).split('/') ret[val] = int(cur) ret[val + "_total"] = int(tot) ret['updated'] = time.time() setattr(self, '__capacity_status', ret) except CommandExitCodeException as CE: self.log.warn('{0}\nError fetching nc status:"{1}"'.format(get_traceback(), str(CE))) return ret