def main(options):
"""Main script.
@param options: Options to run the script.
"""
config = logging_config.LoggingConfig()
if options.logfile:
config.add_file_handler(
file_path=os.path.abspath(options.logfile),
level=logging.DEBUG if options.verbose else logging.INFO)
bucket = lxc.ContainerBucket()
logging.info('')
logging.info('Cleaning container bucket %s', bucket.container_path)
success_count = 0
failure_count = 0
for container in bucket.get_all().values():
if is_container_orphaned(container):
if cleanup(container, options):
success_count += 1
else:
failure_count += 1
if options.execute:
key = 'container_cleanup.%s' % socket.gethostname().replace('.', '_')
autotest_stats.Gauge(key).send('success', success_count)
autotest_stats.Gauge(key).send('failure', failure_count)
logging.info('Cleanup finished.')
def main():
"""Main script."""
with site_utils.SetupTsMonGlobalState('check_slave_db_delay',
indirect=True):
options = parse_options()
log_config = logging_config.LoggingConfig()
if options.logfile:
log_config.add_file_handler(file_path=os.path.abspath(
options.logfile),
level=logging.DEBUG)
db_user = CONFIG.get_config_value('AUTOTEST_WEB', 'user')
db_password = CONFIG.get_config_value('AUTOTEST_WEB', 'password')
global_db_user = CONFIG.get_config_value('AUTOTEST_WEB',
'global_db_user',
default=db_user)
global_db_password = CONFIG.get_config_value('AUTOTEST_WEB',
'global_db_password',
default=db_password)
logging.info('Start checking Seconds_Behind_Master of slave databases')
for replica in options.replicas:
check_delay(replica, global_db_user, global_db_password)
if not options.replicas:
logging.warning('No replicas checked.')
slaves = server_manager_utils.get_servers(role='database_slave',
status='primary')
for slave in slaves:
check_delay(slave.hostname, db_user, db_password)
if not slaves:
logging.warning('No slaves checked.')
logging.info('Finished checking.')
def setup_logging(log_dir):
"""Setup logging information.
@param log_dir: Path to the directory storing logs of this script.
"""
config = logging_config.LoggingConfig()
logfile = os.path.join(os.path.abspath(log_dir), 'perf_csv_uploader.log')
config.add_file_handler(file_path=logfile, level=logging.DEBUG)
def main():
"""Main script."""
options = parse_options()
log_config = logging_config.LoggingConfig()
if options.logfile:
log_config.add_file_handler(file_path=os.path.abspath(options.logfile),
level=logging.DEBUG)
with ts_mon_config.SetupTsMonGlobalState(service_name='cleanup_tko_db',
indirect=True):
server = CONFIG.get_config_value('AUTOTEST_WEB',
'global_db_host',
default=CONFIG.get_config_value(
'AUTOTEST_WEB', 'host'))
user = CONFIG.get_config_value('AUTOTEST_WEB',
'global_db_user',
default=CONFIG.get_config_value(
'AUTOTEST_WEB', 'user'))
password = CONFIG.get_config_value('AUTOTEST_WEB',
'global_db_password',
default=CONFIG.get_config_value(
'AUTOTEST_WEB', 'password'))
database = CONFIG.get_config_value('AUTOTEST_WEB',
'global_db_database',
default=CONFIG.get_config_value(
'AUTOTEST_WEB', 'database'))
logging.info(
'Starting cleaning up old records in TKO database %s on '
'server %s.', database, server)
start_time = time.time()
try:
if options.recreate_test_attributes:
with metrics.SecondsTimer(RECREATE_TEST_ATTRIBUTES_METRIC,
fields={'success': False}) as fields:
_recreate_test_attributes(server, user, password, database)
fields['success'] = True
else:
with metrics.SecondsTimer(CLEANUP_METRIC,
fields={'success': False}) as fields:
utils.run_sql_cmd(server, user, password, CLEANUP_TKO_CMD,
database)
fields['success'] = True
except:
logging.exception('Cleanup failed with exception.')
finally:
duration = time.time() - start_time
logging.info('Cleanup attempt finished in %s seconds.', duration)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--db', dest='db_server',
help='Database server', default=DB_SERVER)
parser.add_argument('-p', dest='prefix', action='store_true',
help=('Use argument <label> as a prefix for matching. '
'For example, when the argument <label> is "cros-version" '
'and this option is enabled, then labels whose name '
'beginning with "cros-version" are matched. When this '
'option is disabled, we match labels whose name is '
'exactly same as the argument <label>.'))
parser.add_argument('-n', dest='max_delete', type=int,
help=('Max number of records to delete in each query.'),
default=100)
parser.add_argument('-s', dest='check_status', action='store_true',
help=('Enforce to run only in a server that has primary status'))
parser.add_argument('label', help='Label name to delete')
options = parser.parse_args()
logging_config.LoggingConfig().configure_logging(
datefmt='%Y-%m-%d %H:%M:%S')
try:
msg = 'Label cleaner starts. Will delete '
if options.prefix:
msg += 'all labels whose prefix is "%s".'
else:
msg += 'a label "%s".'
logging.info(msg, options.label)
logging.info('Target database: %s.', options.db_server)
if options.check_status and not is_primary_server():
logging.error('Cannot run in a non-primary server.')
return 1
conn = MySQLdb.connect(host=options.db_server, user=USER,
passwd=PASSWD, db=DATABASE)
used_labels = get_used_labels(conn)
labels = fetch_labels(conn, options.label, options.prefix)
delete_labels(conn, labels - used_labels, options.max_delete)
logging.info('Done.')
except:
logging.error(traceback.format_exc())
return 1
def main():
"""Main script."""
options = parse_options()
log_config = logging_config.LoggingConfig()
if options.logfile:
log_config.add_file_handler(file_path=os.path.abspath(options.logfile),
level=logging.DEBUG)
server = CONFIG.get_config_value('AUTOTEST_WEB',
'global_db_host',
default=CONFIG.get_config_value(
'AUTOTEST_WEB', 'host'))
user = CONFIG.get_config_value('AUTOTEST_WEB',
'global_db_user',
default=CONFIG.get_config_value(
'AUTOTEST_WEB', 'user'))
password = CONFIG.get_config_value('AUTOTEST_WEB',
'global_db_password',
default=CONFIG.get_config_value(
'AUTOTEST_WEB', 'password'))
database = CONFIG.get_config_value('AUTOTEST_WEB',
'global_db_database',
default=CONFIG.get_config_value(
'AUTOTEST_WEB', 'database'))
logging.info(
'Start cleaning up old records in TKO database %s on server '
'%s.', database, server)
start_time = time.time()
try:
utils.run_sql_cmd(server, user, password, CLEANUP_TKO_CMD, database)
except:
logging.exception('Cleanup failed with exception.')
finally:
duration = time.time() - start_time
metrics.SecondsDistribution(CLEANUP_METRICS).add(
duration, fields={'server': server})
logging.info('Cleanup finished in %s seconds.', duration)
def main(options):
"""Main script.
@param options: Options to run the script.
"""
config = logging_config.LoggingConfig()
if options.logfile:
config.add_file_handler(
file_path=os.path.abspath(options.logfile),
level=logging.DEBUG if options.verbose else logging.INFO)
bucket = lxc.ContainerBucket()
logging.info('')
logging.info('Cleaning container bucket %s', bucket.container_path)
success_count = 0
failure_count = 0
for container in bucket.get_all().values():
if is_container_orphaned(container):
if cleanup(container, options):
success_count += 1
else:
failure_count += 1
logging.info('Cleanup finished.')
def config_logging():
"""Configs logging to be verbose and use console handler."""
config = logging_config.LoggingConfig()
config.configure_logging(use_console=True, verbose=True)
did_reconnect = False
while not _is_adb_connected(target, adb_option):
if not did_reconnect:
logging.info('adb not connected. attempting to reconnect')
did_reconnect = True
_run_adb_cmd('connect %s' % pipes.quote(target),
adb_option=adb_option,
timeout=_ADB_COMMAND_TIMEOUT_SECONDS,
ignore_status=True)
time.sleep(_ADB_CONNECT_INTERVAL_SECONDS)
if did_reconnect:
logging.info('Reconnection succeeded')
if __name__ == '__main__':
logging_config.LoggingConfig().configure_logging()
parser = argparse.ArgumentParser(description='ensure adb is connected')
parser.add_argument('target', help='Device to connect to')
parser.add_argument('--socket',
help='ADB server socket.',
default='tcp:localhost:5037')
args = parser.parse_args()
adb_option = _get_adb_options(args.target, args.socket)
# Setup signal handler for logging on exit
for attr in dir(signal):
if not attr.startswith('SIG') or attr.startswith('SIG_'):
continue
if attr in ('SIGCHLD', 'SIGCLD', 'SIGKILL', 'SIGSTOP'):
continue
signum = getattr(signal, attr)
def _ensure_adb_connected(target, adb_option=""):
"""Ensures adb is connected to the container, reconnects otherwise.
@param target: Device to connect to.
@param adb_option: adb global options configuration.
"""
while not _is_adb_connected(target, adb_option):
logging.info('adb not connected. attempting to reconnect')
_run_adb_cmd('connect %s' % pipes.quote(target),
adb_option=adb_option,
ignore_status=True)
time.sleep(_ADB_CONNECT_INTERVAL_SECONDS)
if __name__ == '__main__':
logging_config.LoggingConfig().configure_logging(verbose=True)
parser = argparse.ArgumentParser(description='ensure adb is connected')
parser.add_argument('target', help='Device to connect to')
parser.add_argument('--socket',
help='ADB server socket.',
default='tcp:localhost:5037')
args = parser.parse_args()
adb_option = _get_adb_options(args.target, args.socket)
logging.info('Starting adb_keepalive for target %s on socket %s',
args.target, args.socket)
while True:
try:
time.sleep(_ADB_POLLING_INTERVAL_SECONDS)
_ensure_adb_connected(args.target, adb_option=adb_option)
except KeyboardInterrupt:
def main():
"""Cleans unused labels from AFE database"""
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'--db',
dest='db_server',
help='Database server',
default=DB_SERVER,
)
parser.add_argument(
'--db-user',
dest='db_user',
help='Database user',
default=USER,
)
parser.add_argument(
'--db-password',
dest='db_password',
help='Database password',
default=PASSWD,
)
parser.add_argument(
'-p',
dest='prefix',
action='store_true',
help=('Use argument <label> as a prefix for matching. '
'For example, when the argument <label> is "cros-version" '
'and this option is enabled, then labels whose name '
'beginning with "cros-version" are matched. When this '
'option is disabled, we match labels whose name is '
'exactly same as the argument <label>.'),
)
parser.add_argument(
'-n',
dest='max_delete',
type=int,
help='Max number of records to delete in each query.',
default=100,
)
parser.add_argument(
'-s',
dest='check_status',
action='store_true',
help='Enforce to run only in a server that has primary status',
)
parser.add_argument(
'--dry-run',
dest='dry_run',
action='store_true',
help='Dry run mode. Do not actually delete any labels.',
)
parser.add_argument('label', help='Label name to delete')
options = parser.parse_args()
logging_config.LoggingConfig().configure_logging(
datefmt='%Y-%m-%d %H:%M:%S', verbose=True)
if options.dry_run:
tfd, metrics_file = tempfile.mkstemp()
os.close(tfd)
ts_mon_context = ts_mon_config.SetupTsMonGlobalState(
'afe_label_cleaner',
auto_flush=False,
debug_file=metrics_file,
)
else:
ts_mon_context = ts_mon_config.SetupTsMonGlobalState(
'afe_label_cleaner',
auto_flush=False,
)
with ts_mon_context:
try:
clean_labels(options)
except:
metrics.Counter(_METRICS_PREFIX +
'/tick').increment(fields={
'target_db': options.db_server,
'success': False
})
raise
else:
metrics.Counter(_METRICS_PREFIX +
'/tick').increment(fields={
'target_db': options.db_server,
'success': True
})
finally:
metrics.Flush()
if options.dry_run:
logging.info('Dumped ts_mon metrics to %s', metrics_file)
