def main(argv):
"""Standard main routine.
@param argv Command line arguments, including `sys.argv[0]`.
"""
arguments = _parse_command(argv)
if not arguments:
sys.exit(1)
_configure_logging(arguments)
try:
if arguments.debug_metrics or not arguments.debug:
metrics_file = None if not arguments.debug_metrics else '/dev/null'
with site_utils.SetupTsMonGlobalState('repair_loops',
debug_file=metrics_file,
auto_flush=False):
_perform_inventory_reports(arguments)
metrics.Flush()
else:
_perform_inventory_reports(arguments)
except KeyboardInterrupt:
pass
except EnvironmentError as e:
logging.exception('Unexpected OS error: %s', e)
except Exception as e:
logging.exception('Unexpected exception: %s', e)
def main():
"""Main script."""
with site_utils.SetupTsMonGlobalState('check_slave_db_delay',
indirect=True):
options = parse_options()
log_config = logging_config.LoggingConfig()
if options.logfile:
log_config.add_file_handler(file_path=os.path.abspath(
options.logfile),
level=logging.DEBUG)
db_user = CONFIG.get_config_value('AUTOTEST_WEB', 'user')
db_password = CONFIG.get_config_value('AUTOTEST_WEB', 'password')
global_db_user = CONFIG.get_config_value('AUTOTEST_WEB',
'global_db_user',
default=db_user)
global_db_password = CONFIG.get_config_value('AUTOTEST_WEB',
'global_db_password',
default=db_password)
logging.info('Start checking Seconds_Behind_Master of slave databases')
for replica in options.replicas:
check_delay(replica, global_db_user, global_db_password)
if not options.replicas:
logging.warning('No replicas checked.')
slaves = server_manager_utils.get_servers(role='database_slave',
status='primary')
for slave in slaves:
check_delay(slave.hostname, db_user, db_password)
if not slaves:
logging.warning('No slaves checked.')
logging.info('Finished checking.')
def main():
"""Counts the number of AFE jobs in the last day, then pushes the count to statsd"""
parser = argparse.ArgumentParser(description=(
'A script which records the number of afe jobs run in a time interval.'
))
parser.parse_args(sys.argv[1:])
count = number_of_jobs_since(timedelta(days=1))
with site_utils.SetupTsMonGlobalState('count_jobs', short_lived=True):
# TODO: Reporting a stat for each job created from the afe directly could be better.
# More discussions are needed to decide whether to remove this file.
metrics.Gauge(
'chromeos/autotest/experimental/jobs_rate/afe_daily_count').set(
count)
def main():
"""tko_parse entry point."""
options, args = parse_args()
# We are obliged to use indirect=False, not use the SetupTsMonGlobalState
# context manager, and add a manual flush, because tko/parse is expected to
# be a very short lived (<1 min) script when working effectively, and we
# can't afford to either a) wait for up to 1min for metrics to flush at the
# end or b) drop metrics that were sent within the last minute of execution.
site_utils.SetupTsMonGlobalState('tko_parse', indirect=False,
short_lived=True)
try:
with metrics.SuccessCounter('chromeos/autotest/tko_parse/runs'):
_main_with_options(options, args)
finally:
metrics.Flush()
def main(argv):
"""Standard main routine.
@param argv Command line arguments including `sys.argv[0]`.
"""
arguments = _parse_command(argv)
if arguments.production:
metrics_manager = site_utils.SetupTsMonGlobalState(
'balance_pools',
indirect=False,
auto_flush=False,
)
else:
metrics_manager = site_utils.TrivialContextManager()
with metrics_manager:
end_time = time.time()
start_time = end_time - 24 * 60 * 60
afe = frontend.AFE(server=arguments.web)
def balancer(pool, labels):
"""Balance the specified model.
@param pool: The pool to rebalance for the model.
@param labels: labels to restrict to balancing operations
within.
"""
_balance_model(arguments, afe, pool, labels, start_time, end_time)
_log_message('')
pools = (lab_inventory.CRITICAL_POOLS if arguments.pool
== _ALL_CRITICAL_POOLS else [arguments.pool])
balancer_targets = infer_balancer_targets(afe, arguments, pools)
try:
parallel.RunTasksInProcessPool(
balancer,
balancer_targets,
processes=8,
)
except KeyboardInterrupt:
pass
finally:
metrics.Flush()
def main():
"""Main script. """
with site_utils.SetupTsMonGlobalState('check_hung_proc', short_lived=True):
for p in PROGRAM_TO_CHECK_SET:
check_proc(p, 86400)
def run_autoserv(pid_file_manager, results, parser, ssp_url, use_ssp):
"""Run server job with given options.
@param pid_file_manager: PidFileManager used to monitor the autoserv process
@param results: Folder to store results.
@param parser: Parser for the command line arguments.
@param ssp_url: Url to server-side package.
@param use_ssp: Set to True to run with server-side packaging.
"""
# send stdin to /dev/null
dev_null = os.open(os.devnull, os.O_RDONLY)
os.dup2(dev_null, sys.stdin.fileno())
os.close(dev_null)
# Create separate process group if the process is not a process group
# leader. This allows autoserv process to keep running after the caller
# process (drone manager call) exits.
if os.getpid() != os.getpgid(0):
os.setsid()
# Container name is predefined so the container can be destroyed in
# handle_sigterm.
job_or_task_id = job_directories.get_job_id_or_task_id(
parser.options.results)
container_id = lxc.ContainerId(job_or_task_id, time.time(), os.getpid())
# Implement SIGTERM handler
def handle_sigterm(signum, frame):
logging.debug('Received SIGTERM')
if pid_file_manager:
pid_file_manager.close_file(1, signal.SIGTERM)
logging.debug('Finished writing to pid_file. Killing process.')
# Update results folder's file permission. This needs to be done ASAP
# before the parsing process tries to access the log.
if use_ssp and results:
correct_results_folder_permission(results)
# TODO (sbasi) - remove the time.sleep when crbug.com/302815 is solved.
# This sleep allows the pending output to be logged before the kill
# signal is sent.
time.sleep(.1)
if use_ssp:
logging.debug(
'Destroy container %s before aborting the autoserv '
'process.', container_id)
try:
bucket = lxc.ContainerBucket()
container = bucket.get_container(container_id)
if container:
container.destroy()
else:
logging.debug('Container %s is not found.', container_id)
except:
# Handle any exception so the autoserv process can be aborted.
logging.exception('Failed to destroy container %s.',
container_id)
# Try to correct the result file permission again after the
# container is destroyed, as the container might have created some
# new files in the result folder.
if results:
correct_results_folder_permission(results)
os.killpg(os.getpgrp(), signal.SIGKILL)
# Set signal handler
signal.signal(signal.SIGTERM, handle_sigterm)
# faulthandler is only needed to debug in the Lab and is not avaliable to
# be imported in the chroot as part of VMTest, so Try-Except it.
try:
import faulthandler
faulthandler.register(signal.SIGTERM, all_threads=True, chain=True)
logging.debug('faulthandler registered on SIGTERM.')
except ImportError:
sys.exc_clear()
# Ignore SIGTTOU's generated by output from forked children.
signal.signal(signal.SIGTTOU, signal.SIG_IGN)
# If we received a SIGALARM, let's be loud about it.
signal.signal(signal.SIGALRM, log_alarm)
# Server side tests that call shell scripts often depend on $USER being set
# but depending on how you launch your autotest scheduler it may not be set.
os.environ['USER'] = getpass.getuser()
label = parser.options.label
group_name = parser.options.group_name
user = parser.options.user
client = parser.options.client
server = parser.options.server
verify = parser.options.verify
repair = parser.options.repair
cleanup = parser.options.cleanup
provision = parser.options.provision
reset = parser.options.reset
job_labels = parser.options.job_labels
no_tee = parser.options.no_tee
execution_tag = parser.options.execution_tag
ssh_user = parser.options.ssh_user
ssh_port = parser.options.ssh_port
ssh_pass = parser.options.ssh_pass
collect_crashinfo = parser.options.collect_crashinfo
control_filename = parser.options.control_filename
verify_job_repo_url = parser.options.verify_job_repo_url
skip_crash_collection = parser.options.skip_crash_collection
ssh_verbosity = int(parser.options.ssh_verbosity)
ssh_options = parser.options.ssh_options
no_use_packaging = parser.options.no_use_packaging
in_lab = bool(parser.options.lab)
# can't be both a client and a server side test
if client and server:
parser.parser.error(
"Can not specify a test as both server and client!")
if provision and client:
parser.parser.error("Cannot specify provisioning and client!")
is_special_task = (verify or repair or cleanup or collect_crashinfo
or provision or reset)
use_client_trampoline = False
if parser.options.control_name:
if use_ssp:
# When use_ssp is True, autoserv will be re-executed inside a
# container preserving the --control-name argument. Control file
# will be staged inside the rexecuted autoserv.
control = None
else:
try:
control = _stage_control_file(parser.options.control_name,
results)
except error.AutoservError as e:
logging.info("Using client trampoline because of: %s", e)
control = parser.options.control_name
use_client_trampoline = True
elif parser.args:
control = parser.args[0]
else:
if not is_special_task:
parser.parser.error("Missing argument: control file")
control = None
if ssh_verbosity > 0:
# ssh_verbosity is an integer between 0 and 3, inclusive
ssh_verbosity_flag = '-' + 'v' * ssh_verbosity
else:
ssh_verbosity_flag = ''
machines = _get_machines(parser)
if group_name and len(machines) < 2:
parser.parser.error('-G %r may only be supplied with more than one '
'machine.' % group_name)
job_kwargs = {
'control':
control,
'args':
parser.args[1:],
'resultdir':
results,
'label':
label,
'user':
user,
'machines':
machines,
'machine_dict_list':
server_job.get_machine_dicts(
machine_names=machines,
store_dir=os.path.join(results, parser.options.host_info_subdir),
in_lab=in_lab,
use_shadow_store=not parser.options.local_only_host_info,
host_attributes=parser.options.host_attributes,
),
'client':
client,
'ssh_user':
ssh_user,
'ssh_port':
ssh_port,
'ssh_pass':
ssh_pass,
'ssh_verbosity_flag':
ssh_verbosity_flag,
'ssh_options':
ssh_options,
'group_name':
group_name,
'tag':
execution_tag,
'disable_sysinfo':
parser.options.disable_sysinfo,
'in_lab':
in_lab,
'use_client_trampoline':
use_client_trampoline,
}
if parser.options.parent_job_id:
job_kwargs['parent_job_id'] = int(parser.options.parent_job_id)
if control_filename:
job_kwargs['control_filename'] = control_filename
job = server_job.server_job(**job_kwargs)
job.logging.start_logging()
# perform checks
job.precheck()
# run the job
exit_code = 0
auto_start_servod = _CONFIG.get_config_value('AUTOSERV',
'auto_start_servod',
type=bool,
default=False)
site_utils.SetupTsMonGlobalState('autoserv',
indirect=False,
short_lived=True)
try:
try:
if repair:
if auto_start_servod and len(machines) == 1:
_start_servod(machines[0])
job.repair(job_labels)
elif verify:
job.verify(job_labels)
elif provision:
job.provision(job_labels)
elif reset:
job.reset(job_labels)
elif cleanup:
job.cleanup(job_labels)
else:
if auto_start_servod and len(machines) == 1:
_start_servod(machines[0])
if use_ssp:
try:
_run_with_ssp(job, container_id, job_or_task_id,
results, parser, ssp_url, machines)
finally:
# Update the ownership of files in result folder.
correct_results_folder_permission(results)
else:
if collect_crashinfo:
# Update the ownership of files in result folder. If the
# job to collect crashinfo was running inside container
# (SSP) and crashed before correcting folder permission,
# the result folder might have wrong permission setting.
try:
correct_results_folder_permission(results)
except:
# Ignore any error as the user may not have root
# permission to run sudo command.
pass
metric_name = ('chromeos/autotest/experimental/'
'autoserv_job_run_duration')
f = {
'in_container': utils.is_in_container(),
'success': False
}
with metrics.SecondsTimer(metric_name, fields=f) as c:
job.run(verify_job_repo_url=verify_job_repo_url,
only_collect_crashinfo=collect_crashinfo,
skip_crash_collection=skip_crash_collection,
job_labels=job_labels,
use_packaging=(not no_use_packaging))
c['success'] = True
finally:
job.close()
# Special task doesn't run parse, so result summary needs to be
# built here.
if results and (repair or verify or reset or cleanup or provision):
# Throttle the result on the server side.
try:
result_utils.execute(
results, control_data.DEFAULT_MAX_RESULT_SIZE_KB)
except:
logging.exception(
'Non-critical failure: Failed to throttle results '
'in directory %s.', results)
# Build result view and report metrics for result sizes.
site_utils.collect_result_sizes(results)
except:
exit_code = 1
traceback.print_exc()
finally:
metrics.Flush()
sys.exit(exit_code)
def run_autoserv(pid_file_manager, results, parser, ssp_url, use_ssp):
"""Run server job with given options.
@param pid_file_manager: PidFileManager used to monitor the autoserv process
@param results: Folder to store results.
@param parser: Parser for the command line arguments.
@param ssp_url: Url to server-side package.
@param use_ssp: Set to True to run with server-side packaging.
"""
if parser.options.warn_no_ssp:
# Post a warning in the log.
logging.warn('Autoserv is required to run with server-side packaging. '
'However, no drone is found to support server-side '
'packaging. The test will be executed in a drone without '
'server-side packaging supported.')
# send stdin to /dev/null
dev_null = os.open(os.devnull, os.O_RDONLY)
os.dup2(dev_null, sys.stdin.fileno())
os.close(dev_null)
# Create separate process group if the process is not a process group
# leader. This allows autoserv process to keep running after the caller
# process (drone manager call) exits.
if os.getpid() != os.getpgid(0):
os.setsid()
# Container name is predefined so the container can be destroyed in
# handle_sigterm.
job_or_task_id = job_directories.get_job_id_or_task_id(
parser.options.results)
container_name = (lxc.TEST_CONTAINER_NAME_FMT %
(job_or_task_id, time.time(), os.getpid()))
job_folder = job_directories.get_job_folder_name(parser.options.results)
# Implement SIGTERM handler
def handle_sigterm(signum, frame):
logging.debug('Received SIGTERM')
if pid_file_manager:
pid_file_manager.close_file(1, signal.SIGTERM)
logging.debug('Finished writing to pid_file. Killing process.')
# Update results folder's file permission. This needs to be done ASAP
# before the parsing process tries to access the log.
if use_ssp and results:
correct_results_folder_permission(results)
# TODO (sbasi) - remove the time.sleep when crbug.com/302815 is solved.
# This sleep allows the pending output to be logged before the kill
# signal is sent.
time.sleep(.1)
if use_ssp:
logging.debug(
'Destroy container %s before aborting the autoserv '
'process.', container_name)
metadata = {
'drone': socket.gethostname(),
'job_id': job_or_task_id,
'container_name': container_name,
'action': 'abort',
'success': True
}
try:
bucket = lxc.ContainerBucket()
container = bucket.get(container_name)
if container:
container.destroy()
else:
metadata['success'] = False
metadata['error'] = 'container not found'
logging.debug('Container %s is not found.', container_name)
except:
metadata['success'] = False
metadata['error'] = 'Exception: %s' % str(sys.exc_info())
# Handle any exception so the autoserv process can be aborted.
logging.exception('Failed to destroy container %s.',
container_name)
autotest_es.post(use_http=True,
type_str=lxc.CONTAINER_RUN_TEST_METADB_TYPE,
metadata=metadata)
# Try to correct the result file permission again after the
# container is destroyed, as the container might have created some
# new files in the result folder.
if results:
correct_results_folder_permission(results)
os.killpg(os.getpgrp(), signal.SIGKILL)
# Set signal handler
signal.signal(signal.SIGTERM, handle_sigterm)
# faulthandler is only needed to debug in the Lab and is not avaliable to
# be imported in the chroot as part of VMTest, so Try-Except it.
try:
import faulthandler
faulthandler.register(signal.SIGTERM, all_threads=True, chain=True)
logging.debug('faulthandler registered on SIGTERM.')
except ImportError:
sys.exc_clear()
# Ignore SIGTTOU's generated by output from forked children.
signal.signal(signal.SIGTTOU, signal.SIG_IGN)
# If we received a SIGALARM, let's be loud about it.
signal.signal(signal.SIGALRM, log_alarm)
# Server side tests that call shell scripts often depend on $USER being set
# but depending on how you launch your autotest scheduler it may not be set.
os.environ['USER'] = getpass.getuser()
label = parser.options.label
group_name = parser.options.group_name
user = parser.options.user
client = parser.options.client
server = parser.options.server
install_before = parser.options.install_before
install_after = parser.options.install_after
verify = parser.options.verify
repair = parser.options.repair
cleanup = parser.options.cleanup
provision = parser.options.provision
reset = parser.options.reset
job_labels = parser.options.job_labels
no_tee = parser.options.no_tee
parse_job = parser.options.parse_job
execution_tag = parser.options.execution_tag
if not execution_tag:
execution_tag = parse_job
ssh_user = parser.options.ssh_user
ssh_port = parser.options.ssh_port
ssh_pass = parser.options.ssh_pass
collect_crashinfo = parser.options.collect_crashinfo
control_filename = parser.options.control_filename
test_retry = parser.options.test_retry
verify_job_repo_url = parser.options.verify_job_repo_url
skip_crash_collection = parser.options.skip_crash_collection
ssh_verbosity = int(parser.options.ssh_verbosity)
ssh_options = parser.options.ssh_options
no_use_packaging = parser.options.no_use_packaging
host_attributes = parser.options.host_attributes
in_lab = bool(parser.options.lab)
# can't be both a client and a server side test
if client and server:
parser.parser.error(
"Can not specify a test as both server and client!")
if provision and client:
parser.parser.error("Cannot specify provisioning and client!")
is_special_task = (verify or repair or cleanup or collect_crashinfo
or provision or reset)
if len(parser.args) < 1 and not is_special_task:
parser.parser.error("Missing argument: control file")
if ssh_verbosity > 0:
# ssh_verbosity is an integer between 0 and 3, inclusive
ssh_verbosity_flag = '-' + 'v' * ssh_verbosity
else:
ssh_verbosity_flag = ''
# We have a control file unless it's just a verify/repair/cleanup job
if len(parser.args) > 0:
control = parser.args[0]
else:
control = None
machines = _get_machines(parser)
if group_name and len(machines) < 2:
parser.parser.error('-G %r may only be supplied with more than one '
'machine.' % group_name)
kwargs = {
'group_name': group_name,
'tag': execution_tag,
'disable_sysinfo': parser.options.disable_sysinfo
}
if parser.options.parent_job_id:
kwargs['parent_job_id'] = int(parser.options.parent_job_id)
if control_filename:
kwargs['control_filename'] = control_filename
if host_attributes:
kwargs['host_attributes'] = host_attributes
kwargs['in_lab'] = in_lab
job = server_job.server_job(control, parser.args[1:], results, label, user,
machines, client, parse_job, ssh_user,
ssh_port, ssh_pass, ssh_verbosity_flag,
ssh_options, test_retry, **kwargs)
job.logging.start_logging()
job.init_parser()
# perform checks
job.precheck()
# run the job
exit_code = 0
auto_start_servod = _CONFIG.get_config_value('AUTOSERV',
'auto_start_servod',
type=bool,
default=False)
site_utils.SetupTsMonGlobalState('autoserv',
indirect=False,
short_lived=True)
try:
try:
if repair:
if auto_start_servod and len(machines) == 1:
_start_servod(machines[0])
job.repair(job_labels)
elif verify:
job.verify(job_labels)
elif provision:
job.provision(job_labels)
elif reset:
job.reset(job_labels)
elif cleanup:
job.cleanup(job_labels)
else:
if auto_start_servod and len(machines) == 1:
_start_servod(machines[0])
if use_ssp:
try:
_run_with_ssp(job, container_name, job_or_task_id,
results, parser, ssp_url, job_folder,
machines)
finally:
# Update the ownership of files in result folder.
correct_results_folder_permission(results)
else:
if collect_crashinfo:
# Update the ownership of files in result folder. If the
# job to collect crashinfo was running inside container
# (SSP) and crashed before correcting folder permission,
# the result folder might have wrong permission setting.
try:
correct_results_folder_permission(results)
except:
# Ignore any error as the user may not have root
# permission to run sudo command.
pass
metric_name = ('chromeos/autotest/experimental/'
'autoserv_job_run_duration')
f = {
'in_container': utils.is_in_container(),
'success': False
}
with metrics.SecondsTimer(metric_name, fields=f) as c:
job.run(install_before,
install_after,
verify_job_repo_url=verify_job_repo_url,
only_collect_crashinfo=collect_crashinfo,
skip_crash_collection=skip_crash_collection,
job_labels=job_labels,
use_packaging=(not no_use_packaging))
c['success'] = True
finally:
while job.hosts:
host = job.hosts.pop()
host.close()
except:
exit_code = 1
traceback.print_exc()
finally:
metrics.Flush()
if pid_file_manager:
pid_file_manager.num_tests_failed = job.num_tests_failed
pid_file_manager.close_file(exit_code)
job.cleanup_parser()
sys.exit(exit_code)
parser.add_argument('-p', type=float, dest='probability',
required=False, default=0,
help='(optional) per-addressee probability '
'with which to send email. If not specified '
'all addressees will receive message.')
parser.add_argument('recipients', nargs='*',
help='Email addresses separated by space.')
args = parser.parse_args()
if not args.recipients or not args.subject:
print 'Requires both recipients and subject.'
sys.exit(1)
message_text = sys.stdin.read()
if args.probability:
recipients = []
for r in args.recipients:
if random.random() < args.probability:
recipients.append(r)
if recipients:
print 'Randomly selected recipients %s' % recipients
else:
print 'Random filtering removed all recipients. Sending nothing.'
sys.exit(0)
else:
recipients = args.recipients
with site_utils.SetupTsMonGlobalState('gmail_lib', short_lived=True):
send_email(','.join(recipients), args.subject , message_text)
def main():
"""Main entrance."""
start_time = datetime.datetime.now()
# Record the processed jobs so that
# we can send the duration of parsing to metadata db.
processed_jobs = set()
options, args = parse_args()
if options.detach:
_detach_from_parent_process()
parse_options = _ParseOptions(options.reparse, options.mailit,
options.dry_run, options.suite_report,
options.datastore_creds,
options.export_to_gcloud_path)
results_dir = os.path.abspath(args[0])
assert os.path.exists(results_dir)
site_utils.SetupTsMonGlobalState('tko_parse',
indirect=False,
short_lived=True)
pid_file_manager = pidfile.PidFileManager("parser", results_dir)
if options.write_pidfile:
pid_file_manager.open_file()
try:
# build up the list of job dirs to parse
if options.singledir:
jobs_list = [results_dir]
else:
jobs_list = [
os.path.join(results_dir, subdir)
for subdir in os.listdir(results_dir)
]
# build up the database
db = tko_db.db(autocommit=False,
host=options.db_host,
user=options.db_user,
password=options.db_pass,
database=options.db_name)
# parse all the jobs
for path in jobs_list:
lockfile = open(os.path.join(path, ".parse.lock"), "w")
flags = fcntl.LOCK_EX
if options.noblock:
flags |= fcntl.LOCK_NB
try:
fcntl.flock(lockfile, flags)
except IOError, e:
# lock is not available and nonblock has been requested
if e.errno == errno.EWOULDBLOCK:
lockfile.close()
continue
else:
raise # something unexpected happened
try:
new_jobs = parse_path(db, path, options.level, parse_options)
processed_jobs.update(new_jobs)
finally:
fcntl.flock(lockfile, fcntl.LOCK_UN)
lockfile.close()
except Exception as e:
pid_file_manager.close_file(1)
raise
else:
pid_file_manager.close_file(0)
finally:
metrics.Flush()
duration_secs = (datetime.datetime.now() - start_time).total_seconds()
if options.record_duration:
record_parsing(processed_jobs, duration_secs)