def poll_console_job(session, env_id, svc_id, task_id): while True: resp = console_job_status(session, env_id, svc_id, task_id) if resp["jobId"] is not None: return resp["jobId"] time.sleep(2) output.write(".", sameline = True)
def worker(target): """Starts a Procfile target as a worker. Worker containers are intended to be short-lived, one-off tasks.""" settings = project.read_settings() session = client.acquire_session(settings) output.write("Initiating a background worker for Service: %s (procfile target = \"%s\")" % (settings['serviceId'], target)) services.initiate_worker(session, settings["environmentId"], settings["serviceId"], target) output.write("Worker started.")
def support_ids(): """Prints out various helpful IDs that can be pasted into support tickets.""" settings = project.read_settings() session = client.acquire_session(settings) del settings["token"] for pair in settings.items(): output.write("%s: %s" % pair)
def rake(task_name): """Execute a rake task. This is only applicable to ruby-based applications.""" settings = project.read_settings() session = client.acquire_session(settings) output.write("Executing Rake Task: {}".format(task_name)) resp = services.initiate_rake(session, settings["environmentId"], settings["serviceId"], task_name) output.write("Rake task output viewable in the logging server.")
def transform_single(self, data, prefix = ""): for job in data: for metric in job["metrics"]: output.write("%s%s | %8s (%s) | CPU: %6.2fs (%5.2f%%) | Net: RX: %.2f KB TX: %.2f KB | Mem: %.2f KB | Disk: %.2f KB read / %.2f KB write " % tuple([ \ prefix, time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(metric["ts"])), job["type"], job["id"]] + metric_to_list(metric)))
def redeploy(): """Redeploy an environment's service manually.""" settings = project.read_settings() session = client.acquire_session(settings) service_id = settings["serviceId"] output.write("Redeploying " + service_id) services.redeploy(session, settings["environmentId"], service_id) output.write("Redeploy successful, check status and logs for updates")
def list(): """List all set variables.""" settings = project.read_settings() session = client.acquire_session(settings) service = session.get("%s/v1/environments/%s/services/%s" % (config.paas_host, settings["environmentId"], settings["serviceId"]), verify = True) variables = service["environmentVariables"] for value, key in sorted(variables.items()): output.write("%s = %s" % (value, key))
def list_environments(): """Lists all environments to which you have access.""" settings = project.read_settings(required = False) session = client.acquire_session(settings) envs = environments.list(session) for env in envs: output.write("%s: %s (state: %s)" % (env["data"]["name"], env["environmentId"], env["state"])) if len(envs) == 0: output.write("no environments found")
def users(): """Lists users in the associated environment.""" settings = project.read_settings() session = client.acquire_session(settings) for user in environments.list_users(session, settings["environmentId"])["users"]: if user == settings["user_id"]: output.write("%s (you)" % (user,)) else: output.write(user)
def transform_single(self, data, service_id = None, service_label = None): self.write_headers_maybe() for job in data: for metric in job["metrics"]: row = [metric["ts"], job["type"], job["id"]] + metric_to_list(metric) row = row if service_id is None else [service_label, service_id] + row self.writer.writerow(row) if service_id is None: output.write(self.sio.getvalue())
def create(service_label, skip_poll): settings = project.read_settings() session = client.acquire_session(settings) service_id = services.get_by_label(session, settings["environmentId"], service_label) task_id = services.create_backup(session, settings["environmentId"], service_id) print("Backup started (task ID = %s)" % (task_id,)) if not skip_poll: output.write("Polling until backup finishes.") task = tasks.poll_status(session, settings["environmentId"], task_id, exit_on_error=False) output.write("\nEnded in status '%s'" % (task["status"],)) logs.dump(session, settings, service_label, service_id, task_id, "backup", None) if task["status"] != "finished": sys.exit(-1)
def poll_status(session, env_id, task_id, exit_on_error=True): route = "%s/v1/environments/%s/tasks/%s" % (config.paas_host, env_id, task_id) while True: time.sleep(2) task = session.get(route, verify = True) if task["status"] not in ["scheduled", "queued", "started", "running"]: if task["status"] == "finished": return task else: output.write("") output.error("Error - ended in status '%s'." % (task["status"],), exit=exit_on_error) else: output.write(".", sameline = True)
def status(): """Check the status of the environment and every service in it.""" settings = project.read_settings() session = client.acquire_session(settings) env = environments.retrieve(session, settings["environmentId"]) output.write("environment state: " + env["state"]) codes = [] noncodes = [] for service in services.list(session, settings["environmentId"]): if service["type"] != "utility": if service["type"] == "code": codes.append("\t%s (size = %s, build status = %s, deploy status = %s)" % (service["label"], service["size"], service["build_status"], service["deploy_status"])) else: noncodes.append("\t%s (size = %s, image = %s, status = %s)" % (service["label"], service["size"], service["name"], service["deploy_status"])) for item in (codes + noncodes): output.write(item)
def transform_single(self, data, prefix=""): for job in data: for metric in job["metrics"]: output.write( "%s%s | %s (%s) | CPU: %d | Net: RX: %d KB TX: %d KB | Mem: %d KB | Disk: %d KB read / %d KB write " % ( prefix, time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(metric["ts"])), job["type"], job["id"], math.ceil(metric["cpu"]["usage"] / 1000.0), math.ceil(metric["network"]["rx_bytes"]["ave"] / 1024.0), math.ceil(metric["network"]["tx_bytes"]["ave"] / 1024.0), math.ceil(metric["memory"]["ave"] / 1024.0), math.ceil(metric["diskio"]["read"] / 1024.0), math.ceil(metric["diskio"]["write"] / 1024.0), ) )
def dump(session, settings, service_label, service_id, task_id, task_type, file): """ Downloads and decrypts the logs for a given job. This job is typically a backup, restore, import, or export job. These logs are written to the path :param file: or output to the console if file is None. This should be called after every backup, restore, import, or export job. :param session: the project settings :param settings: the current session :param service_label: the human readable name of the service :param service_id: the unique identifier of the service :param task_id: the ID of the task for which the logs are being retrieved, this **should not** be a job ID :param task_type: the type of task for which the logs are being retrieved (`backup` or `restore`) :param file: the name of the file to dump the logs to or None for console output :return: """ output.write("Retrieving %s logs for task %s ..." % (service_label, task_id)) # translate the task_id into a job job = jobs.retrieve_from_task_id(session, settings["environmentId"], task_id) url = services.get_temporary_logs_url(session, settings["environmentId"], service_id, task_type, job["id"]) r = requests.get(url, stream=True) basename = os.path.basename(str(uuid.uuid4())) dir = tempfile.mkdtemp() tmp_filepath = os.path.join(dir, basename) with open(tmp_filepath, 'wb+') as f: for chunk in r.iter_content(chunk_size=1024): if chunk: f.write(chunk) f.flush() decryption = AESCrypto.Decryption(tmp_filepath, job[task_type]["key"], job[task_type]["iv"]) decrypted_tmp_filepath = os.path.join(dir, str(uuid.uuid4())) decryption.decrypt(decrypted_tmp_filepath) if file is not None: shutil.copy(decrypted_tmp_filepath, file) output.write("Logs written to %s", (file,)) else: output.write("-------------------------- Begin %s logs --------------------------" % (service_label,)) with open(decrypted_tmp_filepath, 'r') as f: for line in f: output.write(line) output.write("-------------------------- End %s logs --------------------------" % (service_label,)) os.remove(tmp_filepath) os.remove(decrypted_tmp_filepath) os.removedirs(dir)
def transform_single(self, data, service_id=None, service_label=None): self.write_headers_maybe() for job in data: for metric in job["metrics"]: row = [ metric["ts"], job["type"], job["id"], math.ceil(metric["cpu"]["usage"] / 1000), math.ceil(metric["network"]["rx_bytes"]["ave"] / 1024), math.ceil(metric["network"]["tx_bytes"]["ave"] / 1024), math.ceil(metric["memory"]["ave"] / 1024.0), math.ceil(metric["diskio"]["read"] / 1024.0), math.ceil(metric["diskio"]["write"] / 1024.0), ] row = row if service_id is None else [service_label, service_id] + row self.writer.writerow(row) if service_id is None: output.write(self.sio.getvalue())
def acquire_session(settings = None): if settings is not None and "token" in settings and "user_id" in settings: session = Session(token = settings["token"], user_id = settings["user_id"]) resp = session.get(config.baas_host + "/v2/auth/verify") if resp.status_code == 200: return session elif resp.status_code == 401: output.write("Session has timed out. Please re-enter credentials.") username = os.getenv("CATALYZE_USERNAME") or config.username if username is None: username = raw_input("Username: "******"username" not in config.behavior else config.behavior["username"] password = os.getenv("CATALYZE_PASSWORD") or config.password if password is None: password = getpass.getpass("Password: "******"token"] = session.token settings["user_id"] = session.user_id project.save_settings(settings) return session
def download(service_label, backup_id, filepath): settings = project.read_settings() session = client.acquire_session(settings) service_id = services.get_by_label(session, settings["environmentId"], service_label) job = jobs.retrieve(session, settings["environmentId"], service_id, backup_id) if job["type"] != "backup" or job["status"] != "finished": output.error("Only 'finished' 'backup' jobs may be downloaded with this command") output.write("Downloading backup %s" % (backup_id,)) url = services.get_temporary_url(session, settings["environmentId"], service_id, backup_id) r = requests.get(url, stream=True) basename = os.path.basename(filepath) dir = tempfile.mkdtemp() tmp_filepath = os.path.join(dir, basename) with open(tmp_filepath, 'wb+') as f: for chunk in r.iter_content(chunk_size=1024): if chunk: f.write(chunk) f.flush() output.write("Decrypting...") decryption = AESCrypto.Decryption(tmp_filepath, job["backup"]["key"], job["backup"]["iv"]) decryption.decrypt(filepath) os.remove(tmp_filepath) output.write("%s downloaded successfully to %s" % (service_label, filepath))
def associate(env_label, service_label, remote): """Associates the git repository in the current directory. This means that the service and environment IDs are stored locally, and a git remote is created (default name = "catalyze") so that code can be pushed, built, and deployed.""" session = client.acquire_session() for env in environments.list(session): if env["data"]["name"] == env_label: settings = { "token": session.token, "user_id": session.user_id, "environmentId": env["environmentId"] } code_services = [svc for svc in services.list(session, env["environmentId"]) if svc["type"] == "code"] selected_service = None if len(code_services) == 0: output.error("No code service found for \"%s\" environment (%s)" % (env_label, env["environmentId"])) elif service_label: for svc in code_services: if svc["label"] == service_label: selected_service = svc break if selected_service is None: output.error("No code service found with label '%s'. Labels found: %s" % \ (service_label, ", ".join([svc["label"] for svc in code_services]))) elif len(code_services) > 1: output.error("Found multiple code services. Must pass one specifically to associate with. Labels found: " + \ ", ".join([svc["label"] for svc in code_services])) else: selected_service = code_services[0] if remote in git.remote_list(): git.remote_remove(remote) git.remote_add(remote, selected_service["source"]) settings["serviceId"] = selected_service["id"] project.save_settings(settings) output.write("\"%s\" remote added." % (remote,)) return output.error("No environment with label \"%s\" found." % (env_label,))
def open_console(service_label, command): """ Opens a secure console to a code or database service. For code services, a command is required. This command is executed as root in the context of the application root directory. For database services, no command is needed - instead, the appropriate command for the database type is run. For example, for a postgres database, psql is run. """ global console_closed settings = project.read_settings() session = client.acquire_session(settings) service_id = services.get_by_label(session, settings["environmentId"], service_label) output.write("Opening console to service '%s'" % (service_id)) task_id = services.request_console(session, settings["environmentId"], service_id, command)["taskId"] output.write("Waiting for the console to be ready... This might take a bit.") job_id = services.poll_console_job(session, settings["environmentId"], service_id, task_id) creds = services.get_console_tokens(session, settings["environmentId"], service_id, job_id) try: url = creds["url"].replace("http", "ws") token = creds["token"] output.write("Connecting...") sslopt = { "ssl_version": ssl.PROTOCOL_TLSv1 } if "skip_cert_validation" in config.behavior: sslopt["check_hostname"] = False ws = ConsoleClient(url, ssl_options = sslopt, headers = [("X-Console-Token", token)]) ws.daemon = False ws.connect() with ContextedConsole() as c: while not console_closed: data = c.get_data() if data: ws.send(data) finally: output.write("Cleaning up") services.destroy_console(session, settings["environmentId"], service_id, job_id)
def restore(service_label, backup_id, skip_poll): settings = project.read_settings() session = client.acquire_session(settings) service_id = services.get_by_label(session, settings["environmentId"], service_label) task_id = services.restore_backup(session, settings["environmentId"], service_id, backup_id) output.write("Restoring (task = %s)" % (task_id,)) if not skip_poll: output.write("Polling until restore is complete.") task = tasks.poll_status(session, settings["environmentId"], task_id, exit_on_error=False) output.write("\nEnded in status '%s'" % (task["status"],)) logs.dump(session, settings, service_label, service_id, task_id, "restore", None) if task["status"] != "finished": sys.exit(-1)
def exec_list(service_label, page, page_size): """List all created backups for the service, sorted from oldest to newest.""" settings = project.read_settings() session = client.acquire_session(settings) service_id = services.get_by_label(session, settings["environmentId"], service_label) raw_backups = services.list_backups(session, settings["environmentId"], service_id, page, page_size) backup_list = [] for id, body in raw_backups.items(): body["id"] = id backup_list.append(body) backup_list.sort(lambda a, b: int((parse_date(a["created_at"]) - parse_date(b["created_at"])).total_seconds())) if len(backup_list) > 0: for item in backup_list: output.write("%s %s (status = %s)" % (item["id"], item["created_at"], item["status"])) if len(backup_list) == page_size and page == 1: output.write("(for older backups, try with --page=2 or adjust --page-size)") elif page == 1: output.write("No backups created yet for this service.")
def transform_group(self, data): for service in data: output.write(service["serviceName"] + ":") self.transform_single(service["jobs"], prefix=" ")
def emulate_console(ws): output.write("I am a test")
def transform_group(self, data): self.write_headers_maybe() for service in data: self.transform_single(service["jobs"], service["serviceId"], service["serviceName"]) output.write(self.sio.getvalue())
def transform_group(self, data): output.write(json.dumps(data))
def transform_single(self, data): output.write(json.dumps(data))
def opened(self): output.write("Connection opened")
def closed(self, code, reason): global console_closed output.write("Connection closed: %s (%s))" % (reason, str(code))) console_closed = True
def received_message(self, message): output.write(message, sameline = True)