def update(queue, queue_data):
general = Lock()
general.lock("Updating...")
am.broadcast("Server restarts in 5 minutes!", True)
time.sleep(60 * 4)
am.broadcast("Server restart in 1 minute! Ready or not, here I come!")
time.sleep(25)
am.broadcast("Seriously, restarting now.")
time.sleep(5)
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
am.stop_server()
for item_id, item_data in queue_data['items'].items():
for action in item_data['actions']:
try:
result = getattr(am, action)(**item_data['check_response'])
except Exception as e:
am.broadcast("Unable to run {}: {}".format(action, str(e)))
log.error("Unable to run {}: {}".format(action, str(e)),
exc_info=True)
am.start_server()
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
time.sleep(60 * 3)
am.broadcast(
"Server should be back up and running by now or in a few more minutes.",
True)
general.unlock()
queue.unlock()
def health_check_diskspace():
lock = Lock("DiskSpace")
total, used, free = shutil.disk_usage("/")
free_mb = free / 1024 / 1024
log.debug(f"[Healthcheck] Free disk space: {free_mb}MB")
if free_mb < 300:
if not lock.locked:
discord_message(f"[{server.name}] Free disk space is running low. {free_mb}MB is available.")
lock.lock()
else:
if lock.locked:
discord_message(f"[{server.name}] Disk space issue was fixed. {free_mb}MB is now available.")
lock.unlock()
def run_with_lock(func, lock_name="general", message=""):
log.debug(f"Running {func.__name__} with lock...")
lock = Lock(lock_name)
if lock.locked:
log.debug(f"Another script already running, abort running {func.__name__}...")
sys.exit()
lock.lock(message)
func()
log.debug(f"{func.__name__} finished. Removing lock.")
lock.unlock()
def mod_updater():
lock = Lock()
if lock.locked:
log.debug("Another script already running, exit...")
sys.exit()
modids, mod_names, memory = check_mod_versions()
if modids != None:
log.info("New mod version found, performing update.")
lock.lock("Mod updater.")
if args.nodelay == False:
if len(modids) > 1:
delay_with_notifications(
message=f"Updating mods: {mod_names}. ")
else:
delay_with_notifications(
message=f"Updating mod: {mod_names}. ")
stop_server()
log.info(f"Updating mods: {modids}")
try_counter = 0
success = False
while not success and try_counter < 10:
success = update_mods(modids)
if not success:
log.warning("Mod updates failed. Retrying in a minute.")
try_counter += 1
time.sleep(60)
if success:
write_config("mod_updater_data", memory)
fix_mods_permissions()
start_server()
time.sleep(10 * 60)
broadcast(
"Update finished. Server should be up and running by now.",
True)
lock.unlock()
else:
broadcast(
"Mod update failed after 10 retries. Manual intervention is required. @Nix#8175 notified.",
True)
else:
log.debug("No new mod version found.")
def update():
lock = Lock()
if lock.locked:
log.debug("Another script already running, exit...")
sys.exit()
if check_version():
print("Update found!")
log.info("New version found, performing update.")
lock.lock("Game updater.")
run_with_delay(update_server, message="Update detected. ")
update_server()
time.sleep(10 * 60)
broadcast("Update finished. Server should be up and running by now.",
True)
time.sleep(20 * 60)
lock.unlock()
else:
log.debug("No new versions found.")
print("No new versions.")
def rcon_command(command):
lock = Lock("warning")
try:
log.debug(f"Running RCON command: {command}")
with MCRcon(server.ip, server.password, port=server.rcon_port) as mcr:
resp = mcr.command(command)
if lock.is_locked:
lock.unlock()
if "Server received, But no response!!" in resp or "No Players Connected" in resp:
return None
return resp
except ConnectionRefusedError:
log.warning("Unable to connect to RCON, is server down?")
except Exception as e:
lock.lock(str(e))
log.error(str(e), exc_info=True)
return None
# sys.exit()
#
# if check_version():
# print ("Update found!")
# log.info("New version found, performing update.")
# lock.lock("Game updater.")
# run_with_delay(update_server, message="Update detected. ")
# update_server()
# time.sleep(10 * 60)
# broadcast("Update finished. Server should be up and running by now.", True)
# time.sleep(20 * 60)
# lock.unlock()
# else:
# log.debug("No new versions found.")
# print ("No new versions.")
if __name__ == "__main__":
import sys
general = Lock()
if general.is_locked:
sys.exit()
background = Lock("background_tasks")
if not background.is_locked:
background.lock()
background_tasks()
background.unlock()
perform_checks()
sys.exit()
lock = Lock()
if lock.is_locked:
log.debug("Another script already running, exit...")
sys.exit()
print("Starting...")
log.info("Admin initialized restart.")
lock.lock("Reboot host")
broadcast(f"Cluster will reboot in 60 minutes. {args.message}{choice(random_funny_bits)}", True, True)
time.sleep(30 * 60)
broadcast(f"Cluster will reboot in 30 minutes. {args.message}{choice(random_funny_bits)}", True, True)
time.sleep(15 * 60)
broadcast(f"Cluster will reboot in 15 minutes. {args.message}{choice(random_funny_bits)}", True, True)
time.sleep(10 * 60)
broadcast(f"Cluster will reboot in 5 minutes. {args.message}{choice(random_funny_bits)}", True, True)
time.sleep(5 * 60)
broadcast(f"Cluster will reboot in 10 seconds. {choice(random_funny_bits)}", True, True)
for i in range(1, 10):
broadcast(f"Restart in {10 - i}...")
print("Remove lock...")
lock.unlock()
print("Restarting...")
reboot_server()
def unlock(name):
lock = Lock(name)
lock.unlock()
return Response(f"Lock {name} removed.")