def main(argv):
"""Standard main routine.
@param argv Command line arguments, including `sys.argv[0]`.
"""
arguments = _parse_command(argv)
if not arguments:
sys.exit(1)
_configure_logging(arguments)
try:
if arguments.debug_metrics or not arguments.debug:
metrics_file = None if not arguments.debug_metrics else '/dev/null'
with site_utils.SetupTsMonGlobalState('repair_loops',
debug_file=metrics_file,
auto_flush=False):
_perform_inventory_reports(arguments)
metrics.Flush()
else:
_perform_inventory_reports(arguments)
except KeyboardInterrupt:
pass
except EnvironmentError as e:
logging.exception('Unexpected OS error: %s', e)
except Exception as e:
logging.exception('Unexpected exception: %s', e)
def _WaitToFlush(last_flush, reset_after=()):
"""Sleeps until the next time we can call metrics.Flush(), then flushes.
Args:
last_flush: timestamp of the last flush
reset_after: A list of metrics to reset after the flush.
"""
time_delta = time.time() - last_flush
time.sleep(max(0, FLUSH_INTERVAL - time_delta))
metrics.Flush(reset_after=reset_after)
def collect_metrics(cycles):
system_metrics.get_uptime()
system_metrics.get_cpu_info()
system_metrics.get_disk_info()
system_metrics.get_mem_info()
system_metrics.get_net_info()
system_metrics.get_proc_info()
system_metrics.get_load_avg()
puppet_metrics.get_puppet_summary()
if cycles == 0:
system_metrics.get_os_info()
system_metrics.get_unix_time() # must be just before flush
metrics.Flush()
def _FlushIfReady(self):
"""Call metrics.Flush() if we are ready and have pending metrics.
This allows us to only call flush every FLUSH_INTERVAL seconds.
"""
now = time.time()
time_delta = now - self.last_flush
if time_delta > FLUSH_INTERVAL:
self.last_flush = now
time_delta = 0
metrics.Flush(reset_after=self.reset_after_flush)
self.pending = False
return time_delta
def __call__(self):
"""Collect metrics."""
system_metrics.collect_uptime()
system_metrics.collect_cpu_info()
system_metrics.collect_disk_info()
system_metrics.collect_mem_info()
net_metrics.collect_net_info()
proc_metrics.collect_proc_info()
system_metrics.collect_load_avg()
puppet_metrics.collect_puppet_summary()
git_metrics.collect_git_metrics()
self._collect_osinfo()
system_metrics.collect_unix_time() # must be just before flush
metrics.Flush()
def Main():
"""Sets up logging and runs matchers against stdin"""
args = ParseArgs()
log_daemon_common.SetupLogging(args)
# Set up metrics sending and go.
ts_mon_args = {}
if args.debug_metrics_file:
ts_mon_args['debug_file'] = args.debug_metrics_file
with ts_mon_config.SetupTsMonGlobalState('apache_error_log_metrics',
**ts_mon_args):
log_daemon_common.RunMatchers(sys.stdin, MATCHERS)
metrics.Flush()
def main():
"""tko_parse entry point."""
options, args = parse_args()
# We are obliged to use indirect=False, not use the SetupTsMonGlobalState
# context manager, and add a manual flush, because tko/parse is expected to
# be a very short lived (<1 min) script when working effectively, and we
# can't afford to either a) wait for up to 1min for metrics to flush at the
# end or b) drop metrics that were sent within the last minute of execution.
site_utils.SetupTsMonGlobalState('tko_parse', indirect=False,
short_lived=True)
try:
with metrics.SuccessCounter('chromeos/autotest/tko_parse/runs'):
_main_with_options(options, args)
finally:
metrics.Flush()
def main(argv):
"""Standard main routine.
@param argv Command line arguments including `sys.argv[0]`.
"""
arguments = _parse_command(argv)
if arguments.production:
metrics_manager = site_utils.SetupTsMonGlobalState(
'balance_pools',
indirect=False,
auto_flush=False,
)
else:
metrics_manager = site_utils.TrivialContextManager()
with metrics_manager:
end_time = time.time()
start_time = end_time - 24 * 60 * 60
afe = frontend.AFE(server=arguments.web)
def balancer(pool, labels):
"""Balance the specified model.
@param pool: The pool to rebalance for the model.
@param labels: labels to restrict to balancing operations
within.
"""
_balance_model(arguments, afe, pool, labels, start_time, end_time)
_log_message('')
pools = (lab_inventory.CRITICAL_POOLS if arguments.pool
== _ALL_CRITICAL_POOLS else [arguments.pool])
balancer_targets = infer_balancer_targets(afe, arguments, pools)
try:
parallel.RunTasksInProcessPool(
balancer,
balancer_targets,
processes=8,
)
except KeyboardInterrupt:
pass
finally:
metrics.Flush()
def _FlushIfReady(pending, last_flush, reset_after=()):
"""Call metrics.Flush() if we are ready and have pending metrics.
This allows us to only call flush every FLUSH_INTERVAL seconds.
Args:
pending: bool indicating whether there are pending metrics to flush.
last_flush: time stamp of the last time flush() was called.
reset_after: A list of metrics to reset after the flush.
"""
now = time.time()
time_delta = now - last_flush
if time_delta > FLUSH_INTERVAL and pending:
last_flush = now
time_delta = 0
metrics.Flush(reset_after=reset_after)
pending = False
else:
pending = True
return pending, last_flush, time_delta
def main():
"""Standard main routine."""
parser = argparse.ArgumentParser(
description='Update the stable repair version for all '
'boards')
parser.add_argument('-n',
'--dry-run',
action='store_true',
help='print changes without executing them')
loglib.add_logging_options(parser)
# TODO(crbug/888046) Make these arguments required once puppet is updated to
# pass them in.
parser.add_argument('--web',
default='cautotest',
help='URL to the AFE to update.')
arguments = parser.parse_args()
loglib.configure_logging_with_args(parser, arguments)
tsmon_args = {
'service_name': parser.prog,
'indirect': False,
'auto_flush': False,
}
if arguments.dry_run:
logging.info('DRYRUN: No changes will be made.')
# metrics will be logged to logging stream anyway.
tsmon_args['debug_file'] = '/dev/null'
try:
with ts_mon_config.SetupTsMonGlobalState(**tsmon_args):
with metrics.SuccessCounter(_METRICS_PREFIX + '/tick',
fields={'afe': arguments.web}):
_assign_stable_images(arguments)
finally:
metrics.Flush()
def run_autoserv(pid_file_manager, results, parser, ssp_url, use_ssp):
"""Run server job with given options.
@param pid_file_manager: PidFileManager used to monitor the autoserv process
@param results: Folder to store results.
@param parser: Parser for the command line arguments.
@param ssp_url: Url to server-side package.
@param use_ssp: Set to True to run with server-side packaging.
"""
# send stdin to /dev/null
dev_null = os.open(os.devnull, os.O_RDONLY)
os.dup2(dev_null, sys.stdin.fileno())
os.close(dev_null)
# Create separate process group if the process is not a process group
# leader. This allows autoserv process to keep running after the caller
# process (drone manager call) exits.
if os.getpid() != os.getpgid(0):
os.setsid()
# Container name is predefined so the container can be destroyed in
# handle_sigterm.
job_or_task_id = job_directories.get_job_id_or_task_id(
parser.options.results)
container_id = lxc.ContainerId(job_or_task_id, time.time(), os.getpid())
# Implement SIGTERM handler
def handle_sigterm(signum, frame):
logging.debug('Received SIGTERM')
if pid_file_manager:
pid_file_manager.close_file(1, signal.SIGTERM)
logging.debug('Finished writing to pid_file. Killing process.')
# Update results folder's file permission. This needs to be done ASAP
# before the parsing process tries to access the log.
if use_ssp and results:
correct_results_folder_permission(results)
# TODO (sbasi) - remove the time.sleep when crbug.com/302815 is solved.
# This sleep allows the pending output to be logged before the kill
# signal is sent.
time.sleep(.1)
if use_ssp:
logging.debug(
'Destroy container %s before aborting the autoserv '
'process.', container_id)
try:
bucket = lxc.ContainerBucket()
container = bucket.get_container(container_id)
if container:
container.destroy()
else:
logging.debug('Container %s is not found.', container_id)
except:
# Handle any exception so the autoserv process can be aborted.
logging.exception('Failed to destroy container %s.',
container_id)
# Try to correct the result file permission again after the
# container is destroyed, as the container might have created some
# new files in the result folder.
if results:
correct_results_folder_permission(results)
os.killpg(os.getpgrp(), signal.SIGKILL)
# Set signal handler
signal.signal(signal.SIGTERM, handle_sigterm)
# faulthandler is only needed to debug in the Lab and is not avaliable to
# be imported in the chroot as part of VMTest, so Try-Except it.
try:
import faulthandler
faulthandler.register(signal.SIGTERM, all_threads=True, chain=True)
logging.debug('faulthandler registered on SIGTERM.')
except ImportError:
sys.exc_clear()
# Ignore SIGTTOU's generated by output from forked children.
signal.signal(signal.SIGTTOU, signal.SIG_IGN)
# If we received a SIGALARM, let's be loud about it.
signal.signal(signal.SIGALRM, log_alarm)
# Server side tests that call shell scripts often depend on $USER being set
# but depending on how you launch your autotest scheduler it may not be set.
os.environ['USER'] = getpass.getuser()
label = parser.options.label
group_name = parser.options.group_name
user = parser.options.user
client = parser.options.client
server = parser.options.server
verify = parser.options.verify
repair = parser.options.repair
cleanup = parser.options.cleanup
provision = parser.options.provision
reset = parser.options.reset
job_labels = parser.options.job_labels
no_tee = parser.options.no_tee
execution_tag = parser.options.execution_tag
ssh_user = parser.options.ssh_user
ssh_port = parser.options.ssh_port
ssh_pass = parser.options.ssh_pass
collect_crashinfo = parser.options.collect_crashinfo
control_filename = parser.options.control_filename
verify_job_repo_url = parser.options.verify_job_repo_url
skip_crash_collection = parser.options.skip_crash_collection
ssh_verbosity = int(parser.options.ssh_verbosity)
ssh_options = parser.options.ssh_options
no_use_packaging = parser.options.no_use_packaging
in_lab = bool(parser.options.lab)
# can't be both a client and a server side test
if client and server:
parser.parser.error(
"Can not specify a test as both server and client!")
if provision and client:
parser.parser.error("Cannot specify provisioning and client!")
is_special_task = (verify or repair or cleanup or collect_crashinfo
or provision or reset)
use_client_trampoline = False
if parser.options.control_name:
if use_ssp:
# When use_ssp is True, autoserv will be re-executed inside a
# container preserving the --control-name argument. Control file
# will be staged inside the rexecuted autoserv.
control = None
else:
try:
control = _stage_control_file(parser.options.control_name,
results)
except error.AutoservError as e:
logging.info("Using client trampoline because of: %s", e)
control = parser.options.control_name
use_client_trampoline = True
elif parser.args:
control = parser.args[0]
else:
if not is_special_task:
parser.parser.error("Missing argument: control file")
control = None
if ssh_verbosity > 0:
# ssh_verbosity is an integer between 0 and 3, inclusive
ssh_verbosity_flag = '-' + 'v' * ssh_verbosity
else:
ssh_verbosity_flag = ''
machines = _get_machines(parser)
if group_name and len(machines) < 2:
parser.parser.error('-G %r may only be supplied with more than one '
'machine.' % group_name)
job_kwargs = {
'control':
control,
'args':
parser.args[1:],
'resultdir':
results,
'label':
label,
'user':
user,
'machines':
machines,
'machine_dict_list':
server_job.get_machine_dicts(
machine_names=machines,
store_dir=os.path.join(results, parser.options.host_info_subdir),
in_lab=in_lab,
use_shadow_store=not parser.options.local_only_host_info,
host_attributes=parser.options.host_attributes,
),
'client':
client,
'ssh_user':
ssh_user,
'ssh_port':
ssh_port,
'ssh_pass':
ssh_pass,
'ssh_verbosity_flag':
ssh_verbosity_flag,
'ssh_options':
ssh_options,
'group_name':
group_name,
'tag':
execution_tag,
'disable_sysinfo':
parser.options.disable_sysinfo,
'in_lab':
in_lab,
'use_client_trampoline':
use_client_trampoline,
}
if parser.options.parent_job_id:
job_kwargs['parent_job_id'] = int(parser.options.parent_job_id)
if control_filename:
job_kwargs['control_filename'] = control_filename
job = server_job.server_job(**job_kwargs)
job.logging.start_logging()
# perform checks
job.precheck()
# run the job
exit_code = 0
auto_start_servod = _CONFIG.get_config_value('AUTOSERV',
'auto_start_servod',
type=bool,
default=False)
site_utils.SetupTsMonGlobalState('autoserv',
indirect=False,
short_lived=True)
try:
try:
if repair:
if auto_start_servod and len(machines) == 1:
_start_servod(machines[0])
job.repair(job_labels)
elif verify:
job.verify(job_labels)
elif provision:
job.provision(job_labels)
elif reset:
job.reset(job_labels)
elif cleanup:
job.cleanup(job_labels)
else:
if auto_start_servod and len(machines) == 1:
_start_servod(machines[0])
if use_ssp:
try:
_run_with_ssp(job, container_id, job_or_task_id,
results, parser, ssp_url, machines)
finally:
# Update the ownership of files in result folder.
correct_results_folder_permission(results)
else:
if collect_crashinfo:
# Update the ownership of files in result folder. If the
# job to collect crashinfo was running inside container
# (SSP) and crashed before correcting folder permission,
# the result folder might have wrong permission setting.
try:
correct_results_folder_permission(results)
except:
# Ignore any error as the user may not have root
# permission to run sudo command.
pass
metric_name = ('chromeos/autotest/experimental/'
'autoserv_job_run_duration')
f = {
'in_container': utils.is_in_container(),
'success': False
}
with metrics.SecondsTimer(metric_name, fields=f) as c:
job.run(verify_job_repo_url=verify_job_repo_url,
only_collect_crashinfo=collect_crashinfo,
skip_crash_collection=skip_crash_collection,
job_labels=job_labels,
use_packaging=(not no_use_packaging))
c['success'] = True
finally:
job.close()
# Special task doesn't run parse, so result summary needs to be
# built here.
if results and (repair or verify or reset or cleanup or provision):
# Throttle the result on the server side.
try:
result_utils.execute(
results, control_data.DEFAULT_MAX_RESULT_SIZE_KB)
except:
logging.exception(
'Non-critical failure: Failed to throttle results '
'in directory %s.', results)
# Build result view and report metrics for result sizes.
site_utils.collect_result_sizes(results)
except:
exit_code = 1
traceback.print_exc()
finally:
metrics.Flush()
sys.exit(exit_code)
def run_autoserv(pid_file_manager, results, parser, ssp_url, use_ssp):
"""Run server job with given options.
@param pid_file_manager: PidFileManager used to monitor the autoserv process
@param results: Folder to store results.
@param parser: Parser for the command line arguments.
@param ssp_url: Url to server-side package.
@param use_ssp: Set to True to run with server-side packaging.
"""
if parser.options.warn_no_ssp:
# Post a warning in the log.
logging.warn('Autoserv is required to run with server-side packaging. '
'However, no drone is found to support server-side '
'packaging. The test will be executed in a drone without '
'server-side packaging supported.')
# send stdin to /dev/null
dev_null = os.open(os.devnull, os.O_RDONLY)
os.dup2(dev_null, sys.stdin.fileno())
os.close(dev_null)
# Create separate process group if the process is not a process group
# leader. This allows autoserv process to keep running after the caller
# process (drone manager call) exits.
if os.getpid() != os.getpgid(0):
os.setsid()
# Container name is predefined so the container can be destroyed in
# handle_sigterm.
job_or_task_id = job_directories.get_job_id_or_task_id(
parser.options.results)
container_name = (lxc.TEST_CONTAINER_NAME_FMT %
(job_or_task_id, time.time(), os.getpid()))
job_folder = job_directories.get_job_folder_name(parser.options.results)
# Implement SIGTERM handler
def handle_sigterm(signum, frame):
logging.debug('Received SIGTERM')
if pid_file_manager:
pid_file_manager.close_file(1, signal.SIGTERM)
logging.debug('Finished writing to pid_file. Killing process.')
# Update results folder's file permission. This needs to be done ASAP
# before the parsing process tries to access the log.
if use_ssp and results:
correct_results_folder_permission(results)
# TODO (sbasi) - remove the time.sleep when crbug.com/302815 is solved.
# This sleep allows the pending output to be logged before the kill
# signal is sent.
time.sleep(.1)
if use_ssp:
logging.debug(
'Destroy container %s before aborting the autoserv '
'process.', container_name)
metadata = {
'drone': socket.gethostname(),
'job_id': job_or_task_id,
'container_name': container_name,
'action': 'abort',
'success': True
}
try:
bucket = lxc.ContainerBucket()
container = bucket.get(container_name)
if container:
container.destroy()
else:
metadata['success'] = False
metadata['error'] = 'container not found'
logging.debug('Container %s is not found.', container_name)
except:
metadata['success'] = False
metadata['error'] = 'Exception: %s' % str(sys.exc_info())
# Handle any exception so the autoserv process can be aborted.
logging.exception('Failed to destroy container %s.',
container_name)
autotest_es.post(use_http=True,
type_str=lxc.CONTAINER_RUN_TEST_METADB_TYPE,
metadata=metadata)
# Try to correct the result file permission again after the
# container is destroyed, as the container might have created some
# new files in the result folder.
if results:
correct_results_folder_permission(results)
os.killpg(os.getpgrp(), signal.SIGKILL)
# Set signal handler
signal.signal(signal.SIGTERM, handle_sigterm)
# faulthandler is only needed to debug in the Lab and is not avaliable to
# be imported in the chroot as part of VMTest, so Try-Except it.
try:
import faulthandler
faulthandler.register(signal.SIGTERM, all_threads=True, chain=True)
logging.debug('faulthandler registered on SIGTERM.')
except ImportError:
sys.exc_clear()
# Ignore SIGTTOU's generated by output from forked children.
signal.signal(signal.SIGTTOU, signal.SIG_IGN)
# If we received a SIGALARM, let's be loud about it.
signal.signal(signal.SIGALRM, log_alarm)
# Server side tests that call shell scripts often depend on $USER being set
# but depending on how you launch your autotest scheduler it may not be set.
os.environ['USER'] = getpass.getuser()
label = parser.options.label
group_name = parser.options.group_name
user = parser.options.user
client = parser.options.client
server = parser.options.server
install_before = parser.options.install_before
install_after = parser.options.install_after
verify = parser.options.verify
repair = parser.options.repair
cleanup = parser.options.cleanup
provision = parser.options.provision
reset = parser.options.reset
job_labels = parser.options.job_labels
no_tee = parser.options.no_tee
parse_job = parser.options.parse_job
execution_tag = parser.options.execution_tag
if not execution_tag:
execution_tag = parse_job
ssh_user = parser.options.ssh_user
ssh_port = parser.options.ssh_port
ssh_pass = parser.options.ssh_pass
collect_crashinfo = parser.options.collect_crashinfo
control_filename = parser.options.control_filename
test_retry = parser.options.test_retry
verify_job_repo_url = parser.options.verify_job_repo_url
skip_crash_collection = parser.options.skip_crash_collection
ssh_verbosity = int(parser.options.ssh_verbosity)
ssh_options = parser.options.ssh_options
no_use_packaging = parser.options.no_use_packaging
host_attributes = parser.options.host_attributes
in_lab = bool(parser.options.lab)
# can't be both a client and a server side test
if client and server:
parser.parser.error(
"Can not specify a test as both server and client!")
if provision and client:
parser.parser.error("Cannot specify provisioning and client!")
is_special_task = (verify or repair or cleanup or collect_crashinfo
or provision or reset)
if len(parser.args) < 1 and not is_special_task:
parser.parser.error("Missing argument: control file")
if ssh_verbosity > 0:
# ssh_verbosity is an integer between 0 and 3, inclusive
ssh_verbosity_flag = '-' + 'v' * ssh_verbosity
else:
ssh_verbosity_flag = ''
# We have a control file unless it's just a verify/repair/cleanup job
if len(parser.args) > 0:
control = parser.args[0]
else:
control = None
machines = _get_machines(parser)
if group_name and len(machines) < 2:
parser.parser.error('-G %r may only be supplied with more than one '
'machine.' % group_name)
kwargs = {
'group_name': group_name,
'tag': execution_tag,
'disable_sysinfo': parser.options.disable_sysinfo
}
if parser.options.parent_job_id:
kwargs['parent_job_id'] = int(parser.options.parent_job_id)
if control_filename:
kwargs['control_filename'] = control_filename
if host_attributes:
kwargs['host_attributes'] = host_attributes
kwargs['in_lab'] = in_lab
job = server_job.server_job(control, parser.args[1:], results, label, user,
machines, client, parse_job, ssh_user,
ssh_port, ssh_pass, ssh_verbosity_flag,
ssh_options, test_retry, **kwargs)
job.logging.start_logging()
job.init_parser()
# perform checks
job.precheck()
# run the job
exit_code = 0
auto_start_servod = _CONFIG.get_config_value('AUTOSERV',
'auto_start_servod',
type=bool,
default=False)
site_utils.SetupTsMonGlobalState('autoserv',
indirect=False,
short_lived=True)
try:
try:
if repair:
if auto_start_servod and len(machines) == 1:
_start_servod(machines[0])
job.repair(job_labels)
elif verify:
job.verify(job_labels)
elif provision:
job.provision(job_labels)
elif reset:
job.reset(job_labels)
elif cleanup:
job.cleanup(job_labels)
else:
if auto_start_servod and len(machines) == 1:
_start_servod(machines[0])
if use_ssp:
try:
_run_with_ssp(job, container_name, job_or_task_id,
results, parser, ssp_url, job_folder,
machines)
finally:
# Update the ownership of files in result folder.
correct_results_folder_permission(results)
else:
if collect_crashinfo:
# Update the ownership of files in result folder. If the
# job to collect crashinfo was running inside container
# (SSP) and crashed before correcting folder permission,
# the result folder might have wrong permission setting.
try:
correct_results_folder_permission(results)
except:
# Ignore any error as the user may not have root
# permission to run sudo command.
pass
metric_name = ('chromeos/autotest/experimental/'
'autoserv_job_run_duration')
f = {
'in_container': utils.is_in_container(),
'success': False
}
with metrics.SecondsTimer(metric_name, fields=f) as c:
job.run(install_before,
install_after,
verify_job_repo_url=verify_job_repo_url,
only_collect_crashinfo=collect_crashinfo,
skip_crash_collection=skip_crash_collection,
job_labels=job_labels,
use_packaging=(not no_use_packaging))
c['success'] = True
finally:
while job.hosts:
host = job.hosts.pop()
host.close()
except:
exit_code = 1
traceback.print_exc()
finally:
metrics.Flush()
if pid_file_manager:
pid_file_manager.num_tests_failed = job.num_tests_failed
pid_file_manager.close_file(exit_code)
job.cleanup_parser()
sys.exit(exit_code)
def _WaitToFlush(self):
"""Sleeps until the next time we can call metrics.Flush(), then flushes."""
time_delta = time.time() - self.last_flush
time.sleep(max(0, FLUSH_INTERVAL - time_delta))
metrics.Flush(reset_after=self.reset_after_flush)
def main():
"""Cleans unused labels from AFE database"""
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'--db',
dest='db_server',
help='Database server',
default=DB_SERVER,
)
parser.add_argument(
'--db-user',
dest='db_user',
help='Database user',
default=USER,
)
parser.add_argument(
'--db-password',
dest='db_password',
help='Database password',
default=PASSWD,
)
parser.add_argument(
'-p',
dest='prefix',
action='store_true',
help=('Use argument <label> as a prefix for matching. '
'For example, when the argument <label> is "cros-version" '
'and this option is enabled, then labels whose name '
'beginning with "cros-version" are matched. When this '
'option is disabled, we match labels whose name is '
'exactly same as the argument <label>.'),
)
parser.add_argument(
'-n',
dest='max_delete',
type=int,
help='Max number of records to delete in each query.',
default=100,
)
parser.add_argument(
'-s',
dest='check_status',
action='store_true',
help='Enforce to run only in a server that has primary status',
)
parser.add_argument(
'--dry-run',
dest='dry_run',
action='store_true',
help='Dry run mode. Do not actually delete any labels.',
)
parser.add_argument('label', help='Label name to delete')
options = parser.parse_args()
logging_config.LoggingConfig().configure_logging(
datefmt='%Y-%m-%d %H:%M:%S', verbose=True)
if options.dry_run:
tfd, metrics_file = tempfile.mkstemp()
os.close(tfd)
ts_mon_context = ts_mon_config.SetupTsMonGlobalState(
'afe_label_cleaner',
auto_flush=False,
debug_file=metrics_file,
)
else:
ts_mon_context = ts_mon_config.SetupTsMonGlobalState(
'afe_label_cleaner',
auto_flush=False,
)
with ts_mon_context:
try:
clean_labels(options)
except:
metrics.Counter(_METRICS_PREFIX +
'/tick').increment(fields={
'target_db': options.db_server,
'success': False
})
raise
else:
metrics.Counter(_METRICS_PREFIX +
'/tick').increment(fields={
'target_db': options.db_server,
'success': True
})
finally:
metrics.Flush()
if options.dry_run:
logging.info('Dumped ts_mon metrics to %s', metrics_file)
def main():
"""Main entrance."""
start_time = datetime.datetime.now()
# Record the processed jobs so that
# we can send the duration of parsing to metadata db.
processed_jobs = set()
options, args = parse_args()
if options.detach:
_detach_from_parent_process()
parse_options = _ParseOptions(options.reparse, options.mailit,
options.dry_run, options.suite_report,
options.datastore_creds,
options.export_to_gcloud_path)
results_dir = os.path.abspath(args[0])
assert os.path.exists(results_dir)
site_utils.SetupTsMonGlobalState('tko_parse',
indirect=False,
short_lived=True)
pid_file_manager = pidfile.PidFileManager("parser", results_dir)
if options.write_pidfile:
pid_file_manager.open_file()
try:
# build up the list of job dirs to parse
if options.singledir:
jobs_list = [results_dir]
else:
jobs_list = [
os.path.join(results_dir, subdir)
for subdir in os.listdir(results_dir)
]
# build up the database
db = tko_db.db(autocommit=False,
host=options.db_host,
user=options.db_user,
password=options.db_pass,
database=options.db_name)
# parse all the jobs
for path in jobs_list:
lockfile = open(os.path.join(path, ".parse.lock"), "w")
flags = fcntl.LOCK_EX
if options.noblock:
flags |= fcntl.LOCK_NB
try:
fcntl.flock(lockfile, flags)
except IOError, e:
# lock is not available and nonblock has been requested
if e.errno == errno.EWOULDBLOCK:
lockfile.close()
continue
else:
raise # something unexpected happened
try:
new_jobs = parse_path(db, path, options.level, parse_options)
processed_jobs.update(new_jobs)
finally:
fcntl.flock(lockfile, fcntl.LOCK_UN)
lockfile.close()
except Exception as e:
pid_file_manager.close_file(1)
raise
else:
pid_file_manager.close_file(0)
finally:
metrics.Flush()
duration_secs = (datetime.datetime.now() - start_time).total_seconds()
if options.record_duration:
record_parsing(processed_jobs, duration_secs)
