def perform_checks(checks=None, auto_update=True):
if not checks:
checks = read_config('updater_config')['updater_checks']
queue = Lock("queue")
if queue.is_locked:
queue_data = yaml.load(queue.message)
first_detection = False
else:
queue_data = {"items": {}}
first_detection = True
notification = ""
if first_detection:
notification += "Restart scheduled ~{} minutes from now.\n".format(
RESTART_DELAY)
# Go through all checks and run functions from arm_manager, storing returned dict values
# Return values must be dicts that will later be passed down to notification strings and action functions
detected = False
for check_id, check in enumerate(checks):
if check_id not in queue_data['items']:
check_response = getattr(am, check['check_function'])()
if check_response:
queue_data['items'][check_id] = check
checks[check_id]['check_response'] = check_response
log.debug("Adding to queue: {}; {};".format(
check['update_name'], checks[check_id]['check_response']))
notification += "{} detected, adding to queue.\n".format(
check['update_name'].format(
**checks[check_id]['check_response']))
detected = True
# Send notification if new update was detected regardless of whether it's first update or not.
# If no updates detected it's not first detection, there's no queue and no notification will be sent.
if detected and notification.strip():
queue_data['update_time'] = datetime.now() + timedelta(
minutes=RESTART_DELAY)
queue.lock(yaml.dump(queue_data, default_flow_style=False))
am.broadcast(notification, True)
# Now check if datetime for scheduled restart is near. If not warn about upcoming update. If it is then start
# performing restart and required actions.
if queue.is_locked:
if datetime.now() >= queue_data['update_time']:
if auto_update:
update(queue, queue_data)
def ApplyAttributes(self, attributes):
Container.ApplyAttributes(self, attributes)
sm.RegisterNotify(self)
self.loadingLock = Lock()
self.loadingFailed = False
self.context = None
self.startLoadingCallback = attributes.get('OnStartLoading', None)
self.stopLoadingCallback = attributes.get('OnStopLoading', None)
self.loadingWheel = LoadingWheel(parent=self,
align=uiconst.CENTER,
state=uiconst.UI_DISABLED)
self.loadingWheel.opacity = 0.0
self.sceneContainer = PreviewSceneContainer(parent=self,
align=uiconst.TOALL)
self.sceneContainer.Startup()
self.navigation = PreviewNavigation(parent=self)
self.navigation.Startup(self.sceneContainer)
def update(queue, queue_data):
general = Lock()
general.lock("Updating...")
am.broadcast("Server restarts in 5 minutes!", True)
time.sleep(60 * 4)
am.broadcast("Server restart in 1 minute! Ready or not, here I come!")
time.sleep(25)
am.broadcast("Seriously, restarting now.")
time.sleep(5)
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
am.stop_server()
for item_id, item_data in queue_data['items'].items():
for action in item_data['actions']:
try:
result = getattr(am, action)(**item_data['check_response'])
except Exception as e:
am.broadcast("Unable to run {}: {}".format(action, str(e)))
log.error("Unable to run {}: {}".format(action, str(e)),
exc_info=True)
am.start_server()
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
time.sleep(60 * 3)
am.broadcast(
"Server should be back up and running by now or in a few more minutes.",
True)
general.unlock()
queue.unlock()
def health_check_diskspace():
lock = Lock("DiskSpace")
total, used, free = shutil.disk_usage("/")
free_mb = free / 1024 / 1024
log.debug(f"[Healthcheck] Free disk space: {free_mb}MB")
if free_mb < 300:
if not lock.locked:
discord_message(f"[{server.name}] Free disk space is running low. {free_mb}MB is available.")
lock.lock()
else:
if lock.locked:
discord_message(f"[{server.name}] Disk space issue was fixed. {free_mb}MB is now available.")
lock.unlock()
def run_with_lock(func, lock_name="general", message=""):
log.debug(f"Running {func.__name__} with lock...")
lock = Lock(lock_name)
if lock.locked:
log.debug(f"Another script already running, abort running {func.__name__}...")
sys.exit()
lock.lock(message)
func()
log.debug(f"{func.__name__} finished. Removing lock.")
lock.unlock()
def mod_updater():
lock = Lock()
if lock.locked:
log.debug("Another script already running, exit...")
sys.exit()
modids, mod_names, memory = check_mod_versions()
if modids != None:
log.info("New mod version found, performing update.")
lock.lock("Mod updater.")
if args.nodelay == False:
if len(modids) > 1:
delay_with_notifications(
message=f"Updating mods: {mod_names}. ")
else:
delay_with_notifications(
message=f"Updating mod: {mod_names}. ")
stop_server()
log.info(f"Updating mods: {modids}")
try_counter = 0
success = False
while not success and try_counter < 10:
success = update_mods(modids)
if not success:
log.warning("Mod updates failed. Retrying in a minute.")
try_counter += 1
time.sleep(60)
if success:
write_config("mod_updater_data", memory)
fix_mods_permissions()
start_server()
time.sleep(10 * 60)
broadcast(
"Update finished. Server should be up and running by now.",
True)
lock.unlock()
else:
broadcast(
"Mod update failed after 10 retries. Manual intervention is required. @Nix#8175 notified.",
True)
else:
log.debug("No new mod version found.")
def update():
lock = Lock()
if lock.locked:
log.debug("Another script already running, exit...")
sys.exit()
if check_version():
print("Update found!")
log.info("New version found, performing update.")
lock.lock("Game updater.")
run_with_delay(update_server, message="Update detected. ")
update_server()
time.sleep(10 * 60)
broadcast("Update finished. Server should be up and running by now.",
True)
time.sleep(20 * 60)
lock.unlock()
else:
log.debug("No new versions found.")
print("No new versions.")
def rcon_command(command):
lock = Lock("warning")
try:
log.debug(f"Running RCON command: {command}")
with MCRcon(server.ip, server.password, port=server.rcon_port) as mcr:
resp = mcr.command(command)
if lock.is_locked:
lock.unlock()
if "Server received, But no response!!" in resp or "No Players Connected" in resp:
return None
return resp
except ConnectionRefusedError:
log.warning("Unable to connect to RCON, is server down?")
except Exception as e:
lock.lock(str(e))
log.error(str(e), exc_info=True)
return None
# sys.exit()
#
# if check_version():
# print ("Update found!")
# log.info("New version found, performing update.")
# lock.lock("Game updater.")
# run_with_delay(update_server, message="Update detected. ")
# update_server()
# time.sleep(10 * 60)
# broadcast("Update finished. Server should be up and running by now.", True)
# time.sleep(20 * 60)
# lock.unlock()
# else:
# log.debug("No new versions found.")
# print ("No new versions.")
if __name__ == "__main__":
import sys
general = Lock()
if general.is_locked:
sys.exit()
background = Lock("background_tasks")
if not background.is_locked:
background.lock()
background_tasks()
background.unlock()
perform_checks()
class PreviewContainer(Container):
__notifyevents__ = ['OnGraphicSettingsChanged', 'OnSetDevice']
def ApplyAttributes(self, attributes):
Container.ApplyAttributes(self, attributes)
sm.RegisterNotify(self)
self.loadingLock = Lock()
self.loadingFailed = False
self.context = None
self.startLoadingCallback = attributes.get('OnStartLoading', None)
self.stopLoadingCallback = attributes.get('OnStopLoading', None)
self.loadingWheel = LoadingWheel(parent=self,
align=uiconst.CENTER,
state=uiconst.UI_DISABLED)
self.loadingWheel.opacity = 0.0
self.sceneContainer = PreviewSceneContainer(parent=self,
align=uiconst.TOALL)
self.sceneContainer.Startup()
self.navigation = PreviewNavigation(parent=self)
self.navigation.Startup(self.sceneContainer)
def Close(self):
super(PreviewContainer, self).Close()
if not self.loadingLock.try_acquire():
tasklets = itertools.chain(self.loadingLock.HoldingTasklets(),
self.loadingLock.WaitingTasklets())
for tasklet in tasklets:
tasklet.raise_exception(PreviewContainerClosing)
self._Cleanup()
self.sceneContainer.scene = None
self.sceneContainer.renderJob = None
self.sceneContainer = None
def _OnStartLoading(self):
if self.startLoadingCallback:
self.startLoadingCallback(self)
else:
uicore.animations.FadeIn(self.loadingWheel, duration=0.4)
def _OnStopLoading(self):
if self.stopLoadingCallback:
self.stopLoadingCallback(self)
else:
uicore.animations.FadeOut(self.loadingWheel, duration=0.4)
def PreviewType(self, typeID, **kwargs):
typeInfo = cfg.invtypes.Get(typeID)
if not IsPreviewable(typeID):
raise InvalidPreviewType(
'{0.name} ({0.typeID}) is not previewable'.format(typeInfo))
if IsModularShip(typeID):
return self.PreviewTech3Ship(typeID,
subsystems=kwargs.get('subsystems'),
scenePath=kwargs.get('scenePath'))
elif typeInfo.categoryID == invconst.categoryApparel:
return self.PreviewApparel(typeID,
gender=kwargs.get('gender'),
background=kwargs.get('background'))
elif typeInfo.groupID in invconst.turretModuleGroups:
return self.PreviewTurret(typeID,
scenePath=kwargs.get('scenePath'))
elif typeInfo.groupID in invconst.turretAmmoGroups:
return self.PreviewAmmo(typeID, kwargs.get('scenePath'))
else:
return self.PreviewSpaceEntity(typeID,
itemID=kwargs.get('itemID'),
scenePath=kwargs.get('scenePath'))
def PreviewAmmo(self, typeID, scenePath=None):
context = AmmoSceneContext(typeID, scenePath=scenePath)
return self.LoadScene(context)
def PreviewApparel(self, typeID, gender=None, background=None):
context = ApparelSceneContext(typeID,
gender=gender,
background=background)
return self.LoadScene(context)
def PreviewCharacter(self,
charID,
dna=None,
apparel=None,
background=None):
context = CharacterSceneContext(charID,
dna=dna,
apparel=apparel,
background=background)
return self.LoadScene(context)
def PreviewSpaceEntity(self, typeID, itemID=None, scenePath=None):
context = SpaceEntitySceneContext(typeID,
itemID=itemID,
scenePath=scenePath)
return self.LoadScene(context)
def PreviewTech3Ship(self, typeID, subsystems=None, scenePath=None):
context = T3ShipSceneContext(typeID,
subsystems=subsystems,
scenePath=scenePath)
return self.LoadScene(context)
def PreviewTurret(self, typeID, scenePath=None):
context = TurretSceneContext(typeID, scenePath=scenePath)
return self.LoadScene(context)
def LoadScene(self, context, force=False):
if context == self.context and not force:
return False
with self.loadingLock:
try:
self._OnStartLoading()
self._Cleanup()
self.context = context
self.context.LoadScene(self.sceneContainer)
self.UpdateViewPort()
except PreviewContainerClosing:
self.loadingFailed = True
except Exception:
self.loadingFailed = True
log.LogException(
'Exception raised while loading preview for {context}'.
format(context=str(context)))
sys.exc_clear()
else:
self.loadingFailed = False
finally:
self._OnStopLoading()
return True
def Reload(self):
if not self.context:
return
self.LoadScene(self.context, force=True)
def _Cleanup(self):
try:
if self.context:
self.context.Cleanup()
except Exception:
log.LogException('Suppressing exception raised during cleanup')
sys.exc_clear()
finally:
self.context = None
def AnimEntry(self,
yaw0=0.0,
pitch0=0.0,
yaw1=-0.5,
pitch1=-0.5,
duration=2.0):
self.sceneContainer.AnimEntry(yaw0, pitch0, yaw1, pitch1, duration)
def UpdateViewPort(self):
if self.sceneContainer:
self.sceneContainer.UpdateViewPort()
def OnSetDevice(self):
uthread.new(self.Reload)
def OnGraphicSettingsChanged(self, changes):
if self.context and any(
(setting in self.context.relevantSettings for setting in changes)):
uthread.new(self.Reload)
def prompt_sudo():
ret = 0
if os.geteuid() != 0:
msg = "[sudo] password for %u:"
ret = subprocess.check_call("sudo -v -p '%s'" % msg, shell=True)
return ret
if prompt_sudo() != 0:
log.warning("Can't run reboot script, need root\sudo access.")
sys.exit()
lock = Lock()
if lock.is_locked:
log.debug("Another script already running, exit...")
sys.exit()
print("Starting...")
log.info("Admin initialized restart.")
lock.lock("Reboot host")
broadcast(f"Cluster will reboot in 60 minutes. {args.message}{choice(random_funny_bits)}", True, True)
time.sleep(30 * 60)
broadcast(f"Cluster will reboot in 30 minutes. {args.message}{choice(random_funny_bits)}", True, True)
time.sleep(15 * 60)
broadcast(f"Cluster will reboot in 15 minutes. {args.message}{choice(random_funny_bits)}", True, True)
time.sleep(10 * 60)
if __name__ == "__main__":
from locks import Lock
import argparse
parser = argparse.ArgumentParser(description='Restart ark server.')
parser.add_argument(
"--message",
dest='message',
default=None,
help="Message to add to broadcast, usually reason for restart.")
args = parser.parse_args()
if args.message:
args.message = args.message.strip()
if not args.message.endswith("."):
args.message = args.message + ". "
else:
args.message = args.message + " "
lock = Lock()
if lock.locked:
log.debug("Another script already running, exit...")
sys.exit()
lock.lock()
restore()
lock.unlock()
else:
print("Name not main just import")
# def update():
# lock = Lock()
#
# if lock.locked:
# log.debug("Another script already running, exit...")
# sys.exit()
#
# if check_version():
# print ("Update found!")
# log.info("New version found, performing update.")
# lock.lock("Game updater.")
# run_with_delay(update_server, message="Update detected. ")
# update_server()
# time.sleep(10 * 60)
# broadcast("Update finished. Server should be up and running by now.", True)
# time.sleep(20 * 60)
# lock.unlock()
# else:
# log.debug("No new versions found.")
# print ("No new versions.")
if __name__ == "__main__":
import sys
general = Lock()
if general.is_locked:
sys.exit()
perform_checks()
default=None,
help="Message to add to broadcast, usually reason for restart.")
parser.add_argument("--nodelay",
dest='nodelay',
default=False,
help="Run without delay.")
args = parser.parse_args()
if args.message:
args.message = args.message.strip()
if not args.message.endswith("."):
args.message = args.message + ". "
else:
args.message = args.message + " "
lock = Lock()
if lock.locked:
log.debug("Another script already running, exit...")
sys.exit()
print("Starting...")
log.info("Admin initialized restart.")
lock.lock("Locked for restart...")
if args.nodelay == False:
broadcast(
f"Server will restart in 60 minutes. {args.message}{choice(random_funny_bits)}",
True)
time.sleep(30 * 60)
broadcast(
f"Server will restart in 30 minutes. {args.message}{choice(random_funny_bits)}",
True)
def unlock(name):
lock = Lock(name)
lock.unlock()
return Response(f"Lock {name} removed.")