def send_email(bug, bug_template): """Send email to the owner and cc's to notify the TestBug. @param bug: TestBug instance. @param bug_template: A template dictionary specifying the default bug filing options for failures in this suite. """ autotest_stats.Counter(EMAIL_COUNT_KEY % 'total').increment() to_set = set(bug.cc) if bug.cc else set() if bug.owner: to_set.add(bug.owner) if bug_template.get('cc'): to_set = to_set.union(bug_template.get('cc')) if bug_template.get('owner'): to_set.add(bug_template.get('owner')) recipients = ', '.join(to_set) try: gmail_lib.send_email( recipients, bug.title(), bug.summary(), retry=False, creds_path=site_utils.get_creds_abspath(EMAIL_CREDS_FILE)) except Exception: autotest_stats.Counter(EMAIL_COUNT_KEY % 'fail').increment() raise
def log_devserver_match_stats(dut_hostname, devserver_url): """Log stats whether host and devserver are in the same subnet. @param dut_hostname: Hostname of the dut. @param devserver_url: Url to the devserver. """ try: devserver_name = dev_server.ImageServer.get_server_name( devserver_url) devserver_ip = socket.gethostbyname(devserver_name) dut_ip = socket.gethostbyname(dut_hostname) except socket.gaierror as e: logging.error('Failed to get IP address, error: %s', e) return # Take the first 2 octets as the indicator of subnet. devserver_subnet = '_'.join(devserver_ip.split('.')[0:2]) dut_subnet = '_'.join(dut_ip.split('.')[0:2]) if not utils.is_in_same_subnet(devserver_ip, dut_ip, 19): counter = ('devserver_mismatch.%s_to_%s' % (devserver_subnet, dut_subnet)) autotest_stats.Counter(counter).increment() counter = 'devserver_mismatch.%s' % devserver_subnet else: counter = 'devserver_match.%s' % devserver_subnet autotest_stats.Counter(counter).increment()
def symbolicate_minidump_with_devserver(minidump_path, resultdir): """ Generates a stack trace for the specified minidump by consulting devserver. This function assumes the debug symbols have been staged on the devserver. @param minidump_path: absolute path to minidump to by symbolicated. @param resultdir: server job's result directory. @raise DevServerException upon failure, HTTP or otherwise. """ # First, look up what build we tested. If we can't find this, we can't # get the right debug symbols, so we might as well give up right now. keyvals = client_utils.read_keyval(resultdir) if JOB_BUILD_KEY not in keyvals: raise dev_server.DevServerException( 'Cannot determine build being tested.') crashserver_name = dev_server.get_least_loaded_devserver( devserver_type=dev_server.CrashServer) if not crashserver_name: autotest_stats.Counter(CRASH_SERVER_OVERLOAD).increment() raise dev_server.DevServerException( 'No crash server has the capacity to symbolicate the dump.') else: autotest_stats.Counter(CRASH_SERVER_FOUND).increment() devserver = dev_server.CrashServer(crashserver_name) trace_text = devserver.symbolicate_dump(minidump_path, keyvals[JOB_BUILD_KEY]) if not trace_text: raise dev_server.DevServerException('Unknown error!!') with open(minidump_path + '.txt', 'w') as trace_file: trace_file.write(trace_text)
def main(): try: autotest_stats.Counter(STATS_KEY).increment('starts') main_without_exception_handling() except Exception as e: message = 'Uncaught exception. Terminating shard_client.' email_manager.manager.log_stacktrace(message) logging.exception(message) autotest_stats.Counter(STATS_KEY).increment('uncaught_exceptions') raise finally: email_manager.manager.send_queued_emails()
def dispatchRequest(self, request): """ Invoke a json RPC call from a decoded json request. @param request: a decoded json_request @returns a dictionary with keys id, result, err and err_traceback """ results = self.blank_result_dict() try: results['id'] = self._getRequestId(request) methName = request['method'] args = request['params'] except KeyError: raise BadServiceRequest(request) autotest_stats.Counter('rpc').increment(methName) metadata = request.copy() metadata['_type'] = 'rpc' timer = autotest_stats.Timer('rpc', metadata=metadata) try: timer.start() meth = self.findServiceEndpoint(methName) results['result'] = self.invokeServiceEndpoint(meth, args) except Exception, err: results['err_traceback'] = traceback.format_exc() results['err'] = err
def run_with_retry(self, function, *args, **dargs): """Call function(*args, **dargs) until either it passes without an operational error, or a timeout is reached. This will re-connect to the database, so it is NOT safe to use this inside of a database transaction. It can be safely used with transactions, but the transaction start & end must be completely contained within the call to 'function'.""" OperationalError = _get_error_class("OperationalError") success = False start_time = time.time() while not success: try: result = function(*args, **dargs) except OperationalError, e: self._log_operational_error(e) stop_time = time.time() elapsed_time = stop_time - start_time if elapsed_time > self.query_timeout: raise else: try: self._random_delay() self._init_db() autotest_stats.Counter('tko_db_error').increment() except OperationalError, e: self._log_operational_error(e)
def _handle_pidfile_error(self, error, message=''): metadata = {'_type': 'scheduler_error', 'error': 'autoserv died without writing exit code', 'process': str(self._state.process), 'pidfile_id': str(self.pidfile_id)} autotest_stats.Counter('autoserv_died_without_writing_exit_code', metadata=metadata).increment() self.on_lost_process(self._state.process)
def find_and_generate_minidump_stacktraces(host_resultdir): """ Finds all minidump files and generates a stack trace for each. Enumerates all files under the test results directory (recursively) and generates a stack trace file for the minidumps. Minidump files are identified as files with .dmp extension. The stack trace filename is composed by appending the .txt extension to the minidump filename. @param host_resultdir: Directory to walk looking for dmp files. @returns The list of generated minidumps. """ minidumps = [] for dir, subdirs, files in os.walk(host_resultdir): for file in files: if not file.endswith('.dmp'): continue minidump = os.path.join(dir, file) # First, try to symbolicate locally. try: generate_minidump_stacktrace(minidump) logging.info('Generated stack trace for dump %s', minidump) minidumps.append(minidump) continue except client_utils.error.CmdError as err: logging.warning( 'Failed to generate stack trace locally for ' 'dump %s (rc=%d):\n%r', minidump, err.result_obj.exit_status, err) # If that did not succeed, try to symbolicate using the dev server. try: logging.info('Generating stack trace for %s', minidump) minidumps.append(minidump) is_timeout, _ = retry.timeout( symbolicate_minidump_with_devserver, args=(minidump, host_resultdir), timeout_sec=600) if is_timeout: logging.warn( 'Generating stack trace is timed out for dump ' '%s', minidump) autotest_stats.Counter(SYMBOLICATE_TIMEDOUT).increment() else: logging.info('Generated stack trace for dump %s', minidump) continue except dev_server.DevServerException as e: logging.warning( 'Failed to generate stack trace on devserver for ' 'dump %s:\n%r', minidump, e) return minidumps
def _check_paired_results_exist(self): if not self._paired_with_monitor().has_process(): metadata = { '_type': 'scheduler_error', 'error': 'No paired results in task', 'task': str(self), 'pidfile_id': str(self._paired_with_monitor().pidfile_id) } autotest_stats.Counter('no_paired_results_in_task', metadata=metadata).increment() self.finished(False) return False return True
def _init_db(self): # make sure we clean up any existing connection if self.con: self.con.close() self.con = None try: # create the db connection and cursor self.con = self.connect(self.host, self.database, self.user, self.password, self.port) except: autotest_stats.Counter('tko_db_con_error').increment() raise self.cur = self.con.cursor()
def _find_installable_dir(cls, host): client_autodir_paths = cls.get_client_autodir_paths(host) for path in client_autodir_paths: try: host.run('mkdir -p %s' % utils.sh_escape(path)) host.run('test -w %s' % utils.sh_escape(path)) return path except error.AutoservRunError: logging.debug('Failed to create %s', path) metadata = {'_type': 'AutoservInstallError', 'hostname': host.hostname} autotest_stats.Counter('AutoservInstallError', metadata=metadata).increment() raise error.AutoservInstallError( 'Unable to find a place to install Autotest; tried %s' % ', '.join(client_autodir_paths))
def _deserialize_many(self, serialized_list, djmodel, message): """Deserialize data in JSON format to database. Deserialize a list of JSON-formatted data to database using Django. @param serialized_list: A list of JSON-formatted data. @param djmodel: Django model type. @param message: A string to be used in a logging message. """ for serialized in serialized_list: with transaction.commit_on_success(): try: djmodel.deserialize(serialized) except Exception as e: logging.error('Deserializing a %s fails: %s, Error: %s', message, serialized, e) autotest_stats.Counter(STATS_KEY).increment( 'deserialization_failures')
def wait_for_machine_to_recover(host, hours_to_wait=HOURS_TO_WAIT): """Wait for a machine (possibly down) to become accessible again. @param host: A RemoteHost instance to wait on @param hours_to_wait: Number of hours to wait before giving up @returns: True if the machine comes back up, False otherwise """ current_time = time.strftime("%b %d %H:%M:%S", time.localtime()) if host.is_up(): logging.info("%s already up, collecting crash info", host.hostname) return True logging.info("Waiting %s hours for %s to come up (%s)", hours_to_wait, host.hostname, current_time) if not host.wait_up(timeout=hours_to_wait * 3600): autotest_stats.Counter('collect_crashinfo_timeout').increment() logging.warning("%s down, unable to collect crash info", host.hostname) return False else: logging.info("%s is back up, collecting crash info", host.hostname) return True
def _run(self, command, timeout, ignore_status, stdout, stderr, connect_timeout, env, options, stdin, args, ignore_timeout): """Helper function for run().""" ssh_cmd = self.ssh_command(connect_timeout, options) if not env.strip(): env = "" else: env = "export %s;" % env for arg in args: command += ' "%s"' % utils.sh_escape(arg) full_cmd = '%s "%s %s"' % (ssh_cmd, env, utils.sh_escape(command)) # TODO(jrbarnette): crbug.com/484726 - When we're in an SSP # container, sometimes shortly after reboot we will see DNS # resolution errors on ssh commands; the problem never # occurs more than once in a row. This especially affects # the autoupdate_Rollback test, but other cases have been # affected, too. # # We work around it by detecting the first DNS resolution error # and retrying exactly one time. dns_retry_count = 2 while True: result = utils.run(full_cmd, timeout, True, stdout, stderr, verbose=False, stdin=stdin, stderr_is_expected=ignore_status, ignore_timeout=ignore_timeout) dns_retry_count -= 1 if (result and result.exit_status == 255 and re.search( r'^ssh: .*: Name or service not known', result.stderr)): if dns_retry_count: logging.debug('Retrying because of DNS failure') continue logging.debug('Retry failed.') autotest_stats.Counter('dns_retry_hack.fail').increment() elif not dns_retry_count: logging.debug('Retry succeeded.') autotest_stats.Counter('dns_retry_hack.pass').increment() break if ignore_timeout and not result: return None # The error messages will show up in band (indistinguishable # from stuff sent through the SSH connection), so we have the # remote computer echo the message "Connected." before running # any command. Since the following 2 errors have to do with # connecting, it's safe to do these checks. if result.exit_status == 255: if re.search( r'^ssh: connect to host .* port .*: ' r'Connection timed out\r$', result.stderr): raise error.AutoservSSHTimeout("ssh timed out", result) if "Permission denied." in result.stderr: msg = "ssh permission denied" raise error.AutoservSshPermissionDeniedError(msg, result) if not ignore_status and result.exit_status > 0: raise error.AutoservRunError("command execution error", result) return result
def offload_dir(dir_entry, dest_path): """Offload the specified directory entry to Google storage. @param dir_entry: Directory entry to offload. @param dest_path: Location in google storage where we will offload the directory. """ try: counter = autotest_stats.Counter(STATS_KEY) counter.increment('jobs_offload_started') sanitize_dir(dir_entry) if LIMIT_FILE_COUNT: limit_file_count(dir_entry) error = False stdout_file = tempfile.TemporaryFile('w+') stderr_file = tempfile.TemporaryFile('w+') process = None signal.alarm(OFFLOAD_TIMEOUT_SECS) gs_path = '%s%s' % (gs_uri, dest_path) process = subprocess.Popen(get_cmd_list(multiprocessing, dir_entry, gs_path), stdout=stdout_file, stderr=stderr_file) process.wait() signal.alarm(0) if process.returncode == 0: dir_size = get_directory_size_kibibytes(dir_entry) counter.increment('kibibytes_transferred_total', dir_size) metadata = { '_type': METADATA_TYPE, 'size_KB': dir_size, 'result_dir': dir_entry, 'drone': socket.gethostname().replace('.', '_') } autotest_stats.Gauge(STATS_KEY, metadata=metadata).send( 'kibibytes_transferred', dir_size) counter.increment('jobs_offloaded') shutil.rmtree(dir_entry) else: error = True except TimeoutException: # If we finished the call to Popen(), we may need to # terminate the child process. We don't bother calling # process.poll(); that inherently races because the child # can die any time it wants. if process: try: process.terminate() except OSError: # We don't expect any error other than "No such # process". pass logging.error( 'Offloading %s timed out after waiting %d ' 'seconds.', dir_entry, OFFLOAD_TIMEOUT_SECS) error = True except OSError as e: # The wrong file permission can lead call # `shutil.rmtree(dir_entry)` to raise OSError with message # 'Permission denied'. Details can be found in # crbug.com/536151 if e.errno == errno.EACCES: logging.warn('Try to correct file permission of %s.', dir_entry) correct_results_folder_permission(dir_entry) finally: signal.alarm(0) if error: # Rewind the log files for stdout and stderr and log # their contents. stdout_file.seek(0) stderr_file.seek(0) stderr_content = stderr_file.read() logging.error('Error occurred when offloading %s:', dir_entry) logging.error('Stdout:\n%s \nStderr:\n%s', stdout_file.read(), stderr_content) # Some result files may have wrong file permission. Try # to correct such error so later try can success. # TODO(dshi): The code is added to correct result files # with wrong file permission caused by bug 511778. After # this code is pushed to lab and run for a while to # clean up these files, following code and function # correct_results_folder_permission can be deleted. if 'CommandException: Error opening file' in stderr_content: logging.warn('Try to correct file permission of %s.', dir_entry) correct_results_folder_permission(dir_entry) stdout_file.close() stderr_file.close()
def _heartbeat_failure(self, log_message): logging.error("Heartbeat failed. %s", log_message) autotest_stats.Counter(STATS_KEY).increment('heartbeat_failures')