def send_agent_update_task(pks, version): assert isinstance(pks, list) ver = version.split("winagent-v")[1] q = Agent.objects.only("pk").filter(pk__in=pks) agents = [ i for i in q if pyver.parse(i.version) < pyver.parse(ver) and i.status == "online" ] if agents: for agent in agents: agent.update_pending = True agent.save(update_fields=["update_pending"]) minions = [i.salt_id for i in agents] r = Agent.get_github_versions() git_versions = r["versions"] data = r["data"] # full response from github versions = {} for i, release in enumerate(data): versions[i] = release["name"] key = [k for k, v in versions.items() if v == version][0] download_url = data[key]["assets"][0]["browser_download_url"] # split into chunks to not overload salt chunks = (minions[i : i + 30] for i in range(0, len(minions), 30)) for chunk in chunks: r = Agent.salt_batch_async( minions=chunk, func="win_agent.do_agent_update", kwargs={"version": ver, "url": download_url}, ) sleep(5)
def update_agent_task(pk, version): agent = Agent.objects.get(pk=pk) errors = [] file = f"/srv/salt/scripts/{version}.exe" ver = version.split("winagent-v")[1] # download the release from github if the file doesn't already exist in /srv if not os.path.exists(file): r = Agent.get_github_versions() git_versions = r["versions"] data = r["data"] # full response from github versions = {} for i, release in enumerate(data): versions[i] = release["name"] key = [k for k, v in versions.items() if v == version][0] download_url = data[key]["assets"][0]["browser_download_url"] p = subprocess.run(["wget", download_url, "-O", file], capture_output=True) app_dir = "C:\\Program Files\\TacticalAgent" temp_dir = "C:\\Windows\\Temp" logger.info( f"{agent.hostname} is attempting update from version {agent.version} to {ver}" ) # send the release to the agent r = agent.salt_api_cmd( hostname=agent.salt_id, timeout=300, func="cp.get_file", arg=[f"salt://scripts/{version}.exe", temp_dir], ) # success return example: {'return': [{'HOSTNAME': 'C:\\Windows\\Temp\\winagent-v0.1.12.exe'}]} # error return example: {'return': [{'HOSTNAME': ''}]} if not r.json()["return"][0][agent.salt_id]: agent.is_updating = False agent.save(update_fields=["is_updating"]) logger.error( f"{agent.hostname} update failed to version {ver} (unable to copy installer)" ) return f"{agent.hostname} update failed to version {ver} (unable to copy installer)" services = ( "tacticalagent", "checkrunner", ) for svc in services: r = service_action(agent.salt_id, "stop", svc) # returns non 0 if error if r.json()["return"][0][agent.salt_id]["retcode"]: errors.append(f"failed to stop {svc}") logger.error( f"{agent.hostname} was unable to stop service {svc}. Update cancelled" ) # start the services if some of them failed to stop, then don't continue if errors: agent.is_updating = False agent.save(update_fields=["is_updating"]) for svc in services: service_action(agent.salt_id, "start", svc) return "stopping services failed. started again" # install the update # success respose example: {'return': [{'HOSTNAME': {'retcode': 0, 'stderr': '', 'stdout': '', 'pid': 3452}}]} # error response example: {'return': [{'HOSTNAME': 'The minion function caused an exception: Traceback...'}]} try: r = agent.salt_api_cmd( hostname=agent.salt_id, timeout=200, func="cmd.script", arg=f"{temp_dir}\\{version}.exe", kwargs={"args": "/VERYSILENT /SUPPRESSMSGBOXES"}, ) except Exception: agent.is_updating = False agent.save(update_fields=["is_updating"]) return ( f"TIMEOUT: failed to run inno setup on {agent.hostname} for version {ver}" ) if "minion function caused an exception" in r.json()["return"][0][ agent.salt_id]: agent.is_updating = False agent.save(update_fields=["is_updating"]) return ( f"EXCEPTION: failed to run inno setup on {agent.hostname} for version {ver}" ) if r.json()["return"][0][agent.salt_id]["retcode"]: agent.is_updating = False agent.save(update_fields=["is_updating"]) logger.error( f"failed to run inno setup on {agent.hostname} for version {ver}") return f"failed to run inno setup on {agent.hostname} for version {ver}" # update the version in the agent's local database r = agent.salt_api_cmd( hostname=agent.salt_id, timeout=45, func="sqlite3.modify", arg=[ "C:\\Program Files\\TacticalAgent\\agentdb.db", f'UPDATE agentstorage SET version = "{ver}"', ], ) # success return example: {'return': [{'FSV': True}]} # error return example: {'return': [{'HOSTNAME': 'The minion function caused an exception: Traceback...'}]} sql_ret = r.json()["return"][0][agent.salt_id] if not isinstance(sql_ret, bool) and isinstance(sql_ret, str): if "minion function caused an exception" in sql_ret: logger.error(f"failed to update {agent.hostname} local database") if not sql_ret: logger.error( f"failed to update {agent.hostname} local database to version {ver}" ) # start the services for svc in services: service_action(agent.salt_id, "start", svc) agent.is_updating = False agent.save(update_fields=["is_updating"]) logger.info(f"{agent.hostname} was successfully updated to version {ver}") return f"{agent.hostname} was successfully updated to version {ver}"