def start_service():
""" Starts a service using the Monit HTTP API. """
parser = argparse.ArgumentParser()
parser.add_argument('service', help='The service to start')
args = parser.parse_args()
monit_operator = MonitOperator()
monit_retry = retry(max_retries=5, retry_on_exception=DEFAULT_RETRIES)
send_w_retries = monit_retry(monit_operator.send_command_sync)
send_w_retries(args.service, 'start')
def start_service():
""" Starts a service using the Monit HTTP API. """
parser = argparse.ArgumentParser()
parser.add_argument('service', help='The service to start')
args = parser.parse_args()
monit_operator = MonitOperator()
monit_retry = retry(max_retries=5, retry_on_exception=DEFAULT_RETRIES)
send_w_retries = monit_retry(monit_operator.send_command_sync)
send_w_retries(args.service, 'start')
def main():
""" Tries to stop all Monit services until they are stopped. """
logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
monit_operator = MonitOperator()
hostname = socket.gethostname()
logging.info('Waiting for monit to stop services')
logged_service_warning = False
stopped_count = 0
while True:
entries = monit_operator.get_entries_sync()
services = {
service: state
for service, state in entries.items()
if 'cron' not in service and service != hostname
}
running = {
service: state
for service, state in services.items()
if state not in (MonitStates.STOPPED, MonitStates.UNMONITORED)
}
if not running:
logging.info('Finished stopping services')
break
if len(services) - len(running) != stopped_count:
stopped_count = len(services) - len(running)
logging.info('Stopped {}/{} services'.format(
stopped_count, len(services)))
try:
ordered_services, unrecognized_services = order_services(
running.keys())
if unrecognized_services and not logged_service_warning:
logging.warning('Unrecognized running services: {}'.format(
unrecognized_services))
logged_service_warning = True
ordered_services = ordered_services + unrecognized_services
service = next((service for service in ordered_services
if services[service] != MonitStates.PENDING))
monit_retry = retry(max_retries=5,
retry_on_exception=DEFAULT_RETRIES)
send_w_retries = monit_retry(monit_operator.send_command_sync)
send_w_retries(service, 'stop')
except StopIteration:
# If all running services are pending, just wait until they are not.
pass
time.sleep(.3)
def stop_service():
""" Stops a service using the Monit HTTP API. """
parser = argparse.ArgumentParser()
parser.add_argument('service', help='The service to stop')
args = parser.parse_args()
logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
try:
monit_operator = MonitOperator()
monit_retry = retry(max_retries=5, retry_on_exception=DEFAULT_RETRIES)
send_w_retries = monit_retry(monit_operator.send_command_sync)
send_w_retries(args.service, 'stop')
except ProcessNotFound as e:
logging.info(str(e))
sys.exit(1)
def stop_service():
""" Stops a service using the Monit HTTP API. """
parser = argparse.ArgumentParser()
parser.add_argument('service', help='The service to stop')
args = parser.parse_args()
logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
try:
monit_operator = MonitOperator()
monit_retry = retry(max_retries=5, retry_on_exception=DEFAULT_RETRIES)
send_w_retries = monit_retry(monit_operator.send_command_sync)
send_w_retries(args.service, 'stop')
except ProcessNotFound as e:
logger.info(str(e))
sys.exit(1)
def _unmonitor_and_terminate(self, watch):
""" Unmonitors an instance and terminates it.
Args:
watch: A string specifying the Monit entry.
"""
try:
monit_retry = retry(max_retries=5, retry_on_exception=DEFAULT_RETRIES)
send_w_retries = monit_retry(self._monit_operator.send_command_sync)
send_w_retries(watch, 'unmonitor')
except ProcessNotFound:
# If Monit does not know about a process, assume it is already stopped.
return
# Now that the AppServer is stopped, remove its monit config file so that
# monit doesn't pick it up and restart it.
self._monit_operator.remove_configuration(watch)
stop_instance(watch, MAX_INSTANCE_RESPONSE_TIME)
def _unmonitor_and_terminate(self, watch):
""" Unmonitors an instance and terminates it.
Args:
watch: A string specifying the Monit entry.
"""
try:
monit_retry = retry(max_retries=5, retry_on_exception=DEFAULT_RETRIES)
send_w_retries = monit_retry(self._monit_operator.send_command_sync)
send_w_retries(watch, 'unmonitor')
except ProcessNotFound:
# If Monit does not know about a process, assume it is already stopped.
return
# Now that the AppServer is stopped, remove its monit config file so that
# monit doesn't pick it up and restart it.
self._monit_operator.remove_configuration(watch)
stop_instance(watch, MAX_INSTANCE_RESPONSE_TIME)
def main():
""" Tries to stop all Monit services until they are stopped. """
logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
monit_operator = MonitOperator()
hostname = socket.gethostname()
logger.info('Waiting for monit to stop services')
logged_service_warning = False
stopped_count = 0
while True:
entries = monit_operator.get_entries_sync()
services = {service: state for service, state in entries.items()
if 'cron' not in service and service != hostname}
running = {service: state for service, state in services.items()
if state not in (MonitStates.STOPPED, MonitStates.UNMONITORED)}
if not running:
logger.info('Finished stopping services')
break
if len(services) - len(running) != stopped_count:
stopped_count = len(services) - len(running)
logger.info(
'Stopped {}/{} services'.format(stopped_count, len(services)))
try:
ordered_services, unrecognized_services = order_services(running.keys())
if unrecognized_services and not logged_service_warning:
logger.warning(
'Unrecognized running services: {}'.format(unrecognized_services))
logged_service_warning = True
ordered_services = ordered_services + unrecognized_services
service = next((service for service in ordered_services
if services[service] != MonitStates.PENDING))
monit_retry = retry(max_retries=5, retry_on_exception=DEFAULT_RETRIES)
send_w_retries = monit_retry(monit_operator.send_command_sync)
send_w_retries(service, 'stop')
except StopIteration:
# If all running services are pending, just wait until they are not.
pass
time.sleep(.3)
return '<PushQueue {}: {}>'.format(self.name, attr_str)
def is_connection_error(err):
""" This function is used as retry criteria.
Args:
err: an instance of Exception.
Returns:
True if error is related to connection, False otherwise.
"""
return isinstance(err, psycopg2.InterfaceError)
retry_pg_connection = retrying.retry(retrying_timeout=10,
retry_on_exception=is_connection_error)
class PostgresPullQueue(Queue):
"""
Before using Postgres implementation, make sure that
connection using appscale user can be created:
/etc/postgresql/9.5/main/pg_hba.conf
"""
TTL_INTERVAL_AFTER_DELETED = '7 days'
# The maximum number of tasks that can be leased at a time.
MAX_LEASE_AMOUNT = 1000
# Tasks can be leased for up to a week.
