def get_child_jobs_info(suite_job_id, num_child_jobs, sanity_check):
"""
Gets information about child jobs of a suite.
@param suite_job_id: Job id of a suite.
@param num_child_jobs: Number of child jobs of the suite.
@param sanity_check: Do sanity check if True.
@return: A tuple of (dictionary, list). For dictionary, the key is
a DUT's hostname and the value is a list of jobs that ran on
the DUT. List is the list of all jobs of the suite.
"""
results = autotest_es.query(fields_returned=['job_id', 'hostname'],
equality_constraints=[
('_type', 'host_history'),
('parent_job_id', suite_job_id),
('status', 'Running'),
])
dut_jobs_dict = {}
job_filter = set()
for hit in results.hits:
job_id = hit['job_id']
dut = hit['hostname']
if job_id in job_filter:
continue
job_list = dut_jobs_dict.setdefault(dut, [])
job_list.append(job_id)
job_filter.add(job_id)
if sanity_check and len(job_filter) != num_child_jobs:
print(
'WARNING: Mismatch number of child jobs of a suite (%d): '
'%d != %d' % (suite_job_id, len(job_filter), num_child_jobs))
return dut_jobs_dict, list(job_filter)
def find_most_recent_entry_before(t, type_str, hostname, fields):
"""Returns the fields of the most recent entry before t.
@param t: time we are interested in.
@param type_str: _type in esdb, such as 'host_history' (string)
@param hostname: hostname of DUT (string)
@param fields: list of fields we are interested in
@returns: time, field_value of the latest entry.
"""
# History older than 90 days are ignored. This helps the ES query faster.
t_epoch = time_utils.to_epoch_time(t)
result = autotest_es.query(fields_returned=fields,
equality_constraints=[('_type', type_str),
('hostname', hostname)],
range_constraints=[
('time_recorded', t_epoch -
3600 * 24 * _MAX_DAYS_FOR_HISTORY, t_epoch)
],
size=1,
sort_specs=[{
'time_recorded': 'desc'
}])
if result.total > 0:
return result.hits[0]
return {}
def get_tasks_runtime(task_list, dut, t_start, job_id, job_info_dict):
"""
Get sum of durations for special tasks.
job_info_dict will be modified in this function to store the duration
for each special task.
@param task_list: List of task id.
@param dut: Hostname of a DUT that the tasks ran on.
@param t_start: Beginning timestamp.
@param job_id: The job id that is related to the tasks.
This is used only for debugging purpose.
@param job_info_dict: Dictionary that has information for jobs.
@return: Sum of durations of the tasks.
"""
t_start_epoch = time_utils.to_epoch_time(t_start)
results = autotest_es.query(
fields_returned=['status', 'task_id', 'duration'],
equality_constraints=[('_type', 'job_time_breakdown'),
('hostname', dut)],
range_constraints=[('time_recorded', t_start_epoch, None)],
batch_constraints=[('task_id', task_list)])
sum = 0
for hit in results.hits:
sum += float(hit['duration'])
job_info_dict[job_id][hit['status']] = float(hit['duration'])
print_verbose('Task %s for Job %s took %s', hit['task_id'], job_id,
hit['duration'])
return sum
def get_summary(start_time, end_time, top=None, report_stat=False):
"""Get the summary of total size of test results for given period.
@param start_time: Start time of the test results to search for.
@param end_time: End time of the test results to search for.
@param top: Number of top hits with the largest size of test results.
@param report_stat: True to report the total test results size to statsd.
"""
fields_returned = ['size_KB', 'time_recorded']
if top > 0:
fields_returned.append('result_dir')
records = autotest_es.query(
fields_returned=fields_returned,
equality_constraints=[('_type', 'result_dir_size'),],
range_constraints=[('time_recorded', start_time, end_time)],
sort_specs=[{'time_recorded': 'asc'}])
total_GB = sum([e['size_KB'] for e in records.hits]) / 1000000.0
print 'Number of test result entries found: %d' % len(records.hits)
print 'Total size of test results uploaded: %.2f GB' % total_GB
if top:
hits = sorted(records.hits, key=lambda hit:hit['size_KB'], reverse=True)
for hit in hits[:top]:
print ('%s: \t%.2f MB' %
(hit['result_dir'], hit['size_KB']/1000.0))
def get_intervals_for_host(t_start, t_end, hostname):
"""Gets intervals for the given.
Query metaDB to return all intervals between given start and end time.
Note that intervals found in metaDB may miss the history from t_start to
the first interval found.
@param t_start: beginning of time period we are interested in.
@param t_end: end of time period we are interested in.
@param hosts: A list of hostnames to look for history.
@param board: Name of the board to look for history. Default is None.
@param pool: Name of the pool to look for history. Default is None.
@returns: A dictionary of hostname: intervals.
"""
t_start_epoch = time_utils.to_epoch_time(t_start)
t_end_epoch = time_utils.to_epoch_time(t_end)
host_history_entries = autotest_es.query(
fields_returned=None,
equality_constraints=[('_type', _HOST_HISTORY_TYPE),
('hostname', hostname)],
range_constraints=[('time_recorded', t_start_epoch, t_end_epoch)],
sort_specs=[{
'time_recorded': 'asc'
}])
return host_history_entries.hits
def get_entries(time_start, time_end, gte, lte, size, index, hostname):
"""Gets all entries from es db with the given constraints.
@param time_start: Earliest time entry was recorded
@param time_end: Latest time entry was recorded
@param gte: Lowest reboot_time to return
@param lte: Highest reboot_time to return
@param size: Max number of entries to return
@param index: es db index to get entries for, i.e. 'cautotest'
@param hostname: string representing hostname to query for
@returns: Entries from esdb.
"""
time_start_epoch = time_utils.to_epoch_time(time_start)
time_end_epoch = time_utils.to_epoch_time(time_end)
gte_epoch = time_utils.to_epoch_time(gte)
lte_epoch = time_utils.to_epoch_time(lte)
return autotest_es.query(
index=index,
fields_returned=['hostname', 'time_recorded', 'value'],
equality_constraints=[('_type', 'reboot_total'),
('hostname', hostname)],
range_constraints=[('time_recorded', time_start_epoch, time_end_epoch),
('value', gte_epoch, lte_epoch)],
size=size,
sort_specs=[{
'hostname': 'asc'
}, {
'value': 'desc'
}])
return results
def get_host_labels(days_back=0, hostname=None, labels=None):
"""Get the labels for a given host or all hosts.
@param days_back: Get the label info around that number of days back. The
default is 0, i.e., the latest label information.
@param hostname: Name of the host, if set to None, return labels for all
hosts. Default is None.
@param labels: A list of labels to filter hosts.
@return: A dictionary of host labels, key is the hostname, and value is a
list of labels, e.g.,
{'host1': ['board:daisy', 'pool:bvt']}
"""
# Search for the latest logged labels before the given days_back.
# Default is 0, which means the last time host labels were logged.
t_end = time.time() - days_back*24*3600
results = autotest_es.query(
fields_returned=['time_index'],
equality_constraints=[('_type', _HOST_LABEL_TIME_INDEX_TYPE),],
range_constraints=[('time_index', None, t_end)],
size=1,
sort_specs=[{'time_index': 'desc'}])
t_end_str = time_utils.epoch_time_to_date_string(t_end)
if results.total == 0:
logging.error('No label information was logged before %s.', t_end_str)
return
time_index = results.hits[0]['time_index']
logging.info('Host labels were recorded at %s',
time_utils.epoch_time_to_date_string(time_index))
# Search for labels for a given host or all hosts, at time_index.
equality_constraints=[('_type', _HOST_LABEL_TYPE),
('time_index', time_index),]
if hostname:
equality_constraints.append(('hostname', hostname))
if labels:
for label in labels:
equality_constraints.append(('labels', label))
results = autotest_es.query(
fields_returned=['hostname', 'labels'],
equality_constraints=equality_constraints)
host_labels = {}
for hit in results.hits:
if 'labels' in hit:
host_labels[hit['hostname']] = hit['labels']
return host_labels
def analyze_suites(start_time, end_time):
"""
Calculates timing stats (i.e., suite runtime, scheduling overhead)
for the suites that finished within the timestamps given by parameters.
@param start_time: Beginning timestamp.
@param end_time: Ending timestamp.
"""
print('Analyzing suites from %s to %s...' %
(time_utils.epoch_time_to_date_string(start_time),
time_utils.epoch_time_to_date_string(end_time)))
if _options.bvtonly:
batch_constraints = [('suite_name',
['bvt-inline', 'bvt-cq', 'bvt-perbuild'])]
else:
batch_constraints = []
start_time_epoch = time_utils.to_epoch_time(start_time)
end_time_epoch = time_utils.to_epoch_time(end_time)
results = autotest_es.query(fields_returned=[
'suite_name', 'suite_job_id', 'board', 'build', 'num_child_jobs',
'duration'
],
equality_constraints=[
('_type', job_overhead.SUITE_RUNTIME_KEY),
],
range_constraints=[
('time_recorded', start_time_epoch,
end_time_epoch)
],
sort_specs=[{
'time_recorded': 'asc'
}],
batch_constraints=batch_constraints)
print('Found %d suites' % (results.total))
for hit in results.hits:
suite_job_id = hit['suite_job_id']
try:
suite_name = hit['suite_name']
num_child_jobs = int(hit['num_child_jobs'])
suite_runtime = float(hit['duration'])
print('Suite: %s (%s), Board: %s, Build: %s, Num child jobs: %d' %
(suite_name, suite_job_id, hit['board'], hit['build'],
num_child_jobs))
suite_stats = get_scheduling_overhead(suite_job_id, num_child_jobs)
print('Suite: %s (%s) runtime: %f,' %
(suite_name, suite_job_id, suite_runtime)),
print_suite_stats(suite_stats)
except Exception as e:
print('ERROR: Exception is raised while processing suite %s' %
(suite_job_id))
print e
def get_job_tasks(job_list, job_info_dict):
"""
Get task ids for each job.
job_info_dict will be modified in this function to store the task ids.
@param job_list: List of job ids
@param job_info_dict: Dictionary that task ids for each job will be stored.
"""
results = autotest_es.query(fields_returned=['job_id', 'task_id'],
equality_constraints=[('_type', 'host_history')
],
batch_constraints=[('job_id', job_list)])
for hit in results.hits:
if 'task_id' in hit:
info_dict = job_info_dict.setdefault(hit['job_id'], {})
task_set = info_dict.setdefault('tasks', set())
task_set.add(hit['task_id'])
def get_calls(time_start,
time_end,
artifact_filters=None,
regex_constraints=None,
devserver=None,
size=1e7):
"""Gets all devserver calls from es db with the given constraints.
@param time_start: Earliest time entry was recorded.
@param time_end: Latest time entry was recorded.
@param artifact_filters: A list of names to match artifacts.
@param regex_constraints: A list of regex constraints for ES query.
@param devserver: name of devserver to query for. If it's set to None,
return calls for all devservers. Default is set to None.
@param size: Max number of entries to return, default to 1 million.
@returns: Entries from esdb.
"""
eqs = [('_type', 'devserver')]
if devserver:
eqs.append(('devserver', devserver))
if artifact_filters:
for artifact in artifact_filters:
eqs.append(('artifacts', artifact))
time_start_epoch = time_utils.to_epoch_time(time_start)
time_end_epoch = time_utils.to_epoch_time(time_end)
results = autotest_es.query(fields_returned=None,
equality_constraints=eqs,
range_constraints=[
('time_recorded', time_start_epoch,
time_end_epoch)
],
size=size,
sort_specs=[{
'time_recorded': 'desc'
}],
regex_constraints=regex_constraints)
devserver_calls = []
for hit in results.hits:
devserver_calls.append(devserver_call(hit))
logging.info('Found %d calls.', len(devserver_calls))
return devserver_calls
def get_nontask_runtime(job_id, dut, job_info_dict):
"""
Get sum of durations for "Queued", "Running", "Parsing", and "Gathering"
status records.
job_info_dict will be modified in this function to store the duration
for each status.
@param job_id: The job id of interest.
@param dut: Hostname of a DUT that the job ran on.
@param job_info_dict: Dictionary that has information for jobs.
@return: Tuple of sum of durations and the timestamp for the last
Queued record.
"""
results = autotest_es.query(
fields_returned=['status', 'duration', 'time_recorded'],
equality_constraints=[('_type', 'job_time_breakdown'),
('job_id', job_id), ('hostname', dut)],
sort_specs=[{
'time_recorded': 'desc'
}])
sum = 0
last_queued_timestamp = 0
# There could be multiple "Queued" records.
# Get sum of durations for the records after the last "Queued" records
# (including the last "Queued" record).
# Exploits the fact that "results" are ordered in the descending order
# of time_recorded.
for hit in results.hits:
job_info_dict[job_id][hit['status']] = float(hit['duration'])
if hit['status'] == 'Queued':
# The first Queued record is the last one because of the descending
# order of "results".
last_queued_timestamp = float(hit['time_recorded'])
sum += float(hit['duration'])
break
else:
sum += float(hit['duration'])
return (sum, last_queued_timestamp)
def _run_one_test_metadata(self, test_type, num_entries, keys):
"""Puts many entries into elasticdb, then query it. """
print('preparing to insert %s entries with keys %s into elasticdb...' %
(num_entries, keys))
es_test_utils.sequential_random_insert_ints(
keys=keys,
target_type=test_type,
index=self.index,
host=self.host,
port=self.port,
use_http=autotest_es.ES_USE_HTTP,
udp_port=autotest_es.ES_UDP_PORT,
num_entries=num_entries,
print_interval=num_entries / 5)
# Wait a bit for es to be populated with the metadata entry.
# I set this to 6 seconds because bulk.udp.flush_interval (es server)
# is configured to be 5 seconds.
print 'waiting %s seconds...' % (self.wait_time)
time.sleep(self.wait_time)
result = autotest_es.query(host=self.host,
port=self.port,
index=self.index,
fields_returned=keys,
range_constraints=[('host_id', 0, None)])
if not result:
print('%s test error: Index %s not found.' %
(test_type, self.index))
return
# TODO(michaelliang): Check hits and total are valid keys at each layer.
num_entries_found = result.total
print(' Inserted %s entries, found %s entries.' %
(num_entries, num_entries_found))
if num_entries_found != num_entries:
print '\n\n%s test failed! \n\n' % (test_type)
else:
print '\n\n%s test passed! \n\n' % (test_type)
def get_intervals_for_hosts(t_start, t_end, hosts=None, board=None, pool=None):
"""Gets intervals for given hosts or board/pool.
Query metaDB to return all intervals between given start and end time.
If a list of hosts is provided, the board and pool constraints are ignored.
If hosts is set to None, and board or pool is set, this method will attempt
to search host history with labels for all hosts, to help the search perform
faster.
If hosts, board and pool are all set to None, return intervals for all
hosts.
Note that intervals found in metaDB may miss the history from t_start to
the first interval found.
@param t_start: beginning of time period we are interested in.
@param t_end: end of time period we are interested in.
@param hosts: A list of hostnames to look for history.
@param board: Name of the board to look for history. Default is None.
@param pool: Name of the pool to look for history. Default is None.
@returns: A dictionary of hostname: intervals.
"""
hosts_intervals = {}
if hosts:
for host in hosts:
hosts_intervals[host] = get_intervals_for_host(
t_start, t_end, host)
else:
hosts = get_matched_hosts(board, pool)
if not hosts:
raise NoHostFoundException(
'No host is found for board:%s, pool:%s.' % (board, pool))
equality_constraints = [
('_type', _HOST_HISTORY_TYPE),
]
if board:
equality_constraints.append(('labels', 'board:' + board))
if pool:
equality_constraints.append(('labels', 'pool:' + pool))
t_start_epoch = time_utils.to_epoch_time(t_start)
t_end_epoch = time_utils.to_epoch_time(t_end)
results = autotest_es.query(equality_constraints=equality_constraints,
range_constraints=[
('time_recorded', t_start_epoch,
t_end_epoch)
],
sort_specs=[{
'hostname': 'asc'
}])
results_group_by_host = {}
for hostname, intervals_for_host in groupby(results.hits,
lambda h: h['hostname']):
results_group_by_host[hostname] = intervals_for_host
for host in hosts:
intervals = results_group_by_host.get(host, None)
# In case the host's board or pool label was modified after
# the last status change event was reported, we need to run a
# separate query to get its history. That way the host's
# history won't be shown as blank.
if not intervals:
intervals = get_intervals_for_host(t_start, t_end, host)
hosts_intervals[host] = intervals
return hosts_intervals
def get_host_history_intervals(input):
"""Gets stats for a host.
This method uses intervals found in metaDB to build a full history of the
host. The intervals argument contains a list of metadata from querying ES
for records between t_start and t_end. To get the status from t_start to
the first record logged in ES, we need to look back to the last record
logged in ES before t_start.
@param input: A dictionary of input args, which including following args:
t_start: beginning of time period we are interested in.
t_end: end of time period we are interested in.
hostname: hostname for the host we are interested in (string)
intervals: intervals from ES query.
@returns: dictionary, num_entries_found
dictionary of status: time spent in that status
num_entries_found: number of host history entries
found in [t_start, t_end]
"""
t_start = input['t_start']
t_end = input['t_end']
hostname = input['hostname']
intervals = input['intervals']
lock_history_recent = find_most_recent_entry_before(
t=t_start,
type_str=_LOCK_HISTORY_TYPE,
hostname=hostname,
fields=['time_recorded', 'locked'])
# I use [0] and [None] because lock_history_recent's type is list.
t_lock = lock_history_recent.get('time_recorded', None)
t_lock_val = lock_history_recent.get('locked', None)
t_metadata = find_most_recent_entry_before(t=t_start,
type_str=_HOST_HISTORY_TYPE,
hostname=hostname,
fields=None)
t_host = t_metadata.pop('time_recorded', None)
t_host_stat = t_metadata.pop('status', None)
status_first = t_host_stat if t_host else 'Ready'
t = min([t for t in [t_lock, t_host, t_start] if t])
t_epoch = time_utils.to_epoch_time(t)
t_end_epoch = time_utils.to_epoch_time(t_end)
lock_history_entries = autotest_es.query(
fields_returned=['locked', 'time_recorded'],
equality_constraints=[('_type', _LOCK_HISTORY_TYPE),
('hostname', hostname)],
range_constraints=[('time_recorded', t_epoch, t_end_epoch)],
sort_specs=[{
'time_recorded': 'asc'
}])
# Validate lock history. If an unlock event failed to be recorded in metadb,
# lock history will show the dut being locked while host still has status
# changed over the time. This check tries to remove the lock event in lock
# history if:
# 1. There is only one entry in lock_history_entries (it's a good enough
# assumption to avoid the code being over complicated.
# 2. The host status has changes after the lock history starts as locked.
if (len(lock_history_entries.hits) == 1 and t_lock_val
and len(intervals) > 1):
locked_intervals = None
print('Lock history of dut %s is ignored, the dut may have missing '
'data in lock history in metadb. Try to lock and unlock the dut '
'in AFE will force the lock history to be updated in metadb.' %
hostname)
else:
locked_intervals = lock_history_to_intervals(t_lock_val, t, t_end,
lock_history_entries)
num_entries_found = len(intervals)
t_prev = t_start
status_prev = status_first
metadata_prev = t_metadata
intervals_of_statuses = collections.OrderedDict()
for entry in intervals:
metadata = entry.copy()
t_curr = metadata.pop('time_recorded')
status_curr = metadata.pop('status')
intervals_of_statuses.update(
calculate_status_times(t_prev, t_curr, status_prev, metadata_prev,
locked_intervals))
# Update vars
t_prev = t_curr
status_prev = status_curr
metadata_prev = metadata
# Do final as well.
intervals_of_statuses.update(
calculate_status_times(t_prev, t_end, status_prev, metadata_prev,
locked_intervals))
return hostname, intervals_of_statuses, num_entries_found