def worker(target):
"""Starts a Procfile target as a worker. Worker containers are intended to be short-lived, one-off tasks."""
settings = project.read_settings()
session = client.acquire_session(settings)
output.write("Initiating a background worker for Service: %s (procfile target = \"%s\")" % (settings['serviceId'], target))
services.initiate_worker(session, settings["environmentId"], settings["serviceId"], target)
output.write("Worker started.")
def rake(task_name):
"""Execute a rake task. This is only applicable to ruby-based applications."""
settings = project.read_settings()
session = client.acquire_session(settings)
output.write("Executing Rake Task: {}".format(task_name))
resp = services.initiate_rake(session, settings["environmentId"], settings["serviceId"], task_name)
output.write("Rake task output viewable in the logging server.")
def metrics(service_label, format, stream, mins):
"""Print out metrics about a single service or all services in an environment."""
if stream and (format or mins != 1):
output.error("--stream cannot be used with a custom format or multiple records.")
if format is None:
transformer = TextTransformer()
elif format == "csv":
transformer = CSVTransformer()
elif format == "json":
transformer = JSONTransformer()
else:
output.error("unrecognized format '%s'" % (format,))
settings = project.read_settings()
session = client.acquire_session(settings)
if service_label is None:
transformer.set_group_mode()
transformer.set_retriever(lambda: environments.retrieve_metrics(session, settings["environmentId"], mins))
else:
service_id = services.get_by_label(session, settings["environmentId"], service_label)
transformer.set_retriever(
lambda: services.retrieve_metrics(session, settings["environmentId"], service_id, mins)
)
transformer.process(stream)
def download(service_label, backup_id, filepath):
settings = project.read_settings()
session = client.acquire_session(settings)
service_id = services.get_by_label(session, settings["environmentId"], service_label)
job = jobs.retrieve(session, settings["environmentId"], service_id, backup_id)
if job["type"] != "backup" or job["status"] != "finished":
output.error("Only 'finished' 'backup' jobs may be downloaded with this command")
output.write("Downloading backup %s" % (backup_id,))
url = services.get_temporary_url(session, settings["environmentId"], service_id, backup_id)
r = requests.get(url, stream=True)
basename = os.path.basename(filepath)
dir = tempfile.mkdtemp()
tmp_filepath = os.path.join(dir, basename)
with open(tmp_filepath, 'wb+') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
f.flush()
output.write("Decrypting...")
decryption = AESCrypto.Decryption(tmp_filepath, job["backup"]["key"], job["backup"]["iv"])
decryption.decrypt(filepath)
os.remove(tmp_filepath)
output.write("%s downloaded successfully to %s" % (service_label, filepath))
def support_ids():
"""Prints out various helpful IDs that can be pasted into support tickets."""
settings = project.read_settings()
session = client.acquire_session(settings)
del settings["token"]
for pair in settings.items():
output.write("%s: %s" % pair)
def unset(variable):
"""Unset (delete) a variable.
Variable changes will not take effect in the application until it is redeployed (via either a push or 'catalyze redeploy')."""
settings = project.read_settings()
session = client.acquire_session(settings)
environment_variables.unset(session, settings["environmentId"], settings["serviceId"], variable)
def redeploy():
"""Redeploy an environment's service manually."""
settings = project.read_settings()
session = client.acquire_session(settings)
service_id = settings["serviceId"]
output.write("Redeploying " + service_id)
services.redeploy(session, settings["environmentId"], service_id)
output.write("Redeploy successful, check status and logs for updates")
def list():
"""List all set variables."""
settings = project.read_settings()
session = client.acquire_session(settings)
service = session.get("%s/v1/environments/%s/services/%s" % (config.paas_host, settings["environmentId"], settings["serviceId"]), verify = True)
variables = service["environmentVariables"]
for value, key in sorted(variables.items()):
output.write("%s = %s" % (value, key))
def users():
"""Lists users in the associated environment."""
settings = project.read_settings()
session = client.acquire_session(settings)
for user in environments.list_users(session, settings["environmentId"])["users"]:
if user == settings["user_id"]:
output.write("%s (you)" % (user,))
else:
output.write(user)
def list_environments():
"""Lists all environments to which you have access."""
settings = project.read_settings(required = False)
session = client.acquire_session(settings)
envs = environments.list(session)
for env in envs:
output.write("%s: %s (state: %s)" % (env["data"]["name"], env["environmentId"], env["state"]))
if len(envs) == 0:
output.write("no environments found")
def restore(service_label, backup_id, skip_poll):
settings = project.read_settings()
session = client.acquire_session(settings)
service_id = services.get_by_label(session, settings["environmentId"], service_label)
task_id = services.restore_backup(session, settings["environmentId"], service_id, backup_id)
output.write("Restoring (task = %s)" % (task_id,))
if not skip_poll:
output.write("Polling until restore is complete.")
task = tasks.poll_status(session, settings["environmentId"], task_id, exit_on_error=False)
output.write("\nEnded in status '%s'" % (task["status"],))
logs.dump(session, settings, service_label, service_id, task_id, "restore", None)
if task["status"] != "finished":
sys.exit(-1)
def create(service_label, skip_poll):
settings = project.read_settings()
session = client.acquire_session(settings)
service_id = services.get_by_label(session, settings["environmentId"], service_label)
task_id = services.create_backup(session, settings["environmentId"], service_id)
print("Backup started (task ID = %s)" % (task_id,))
if not skip_poll:
output.write("Polling until backup finishes.")
task = tasks.poll_status(session, settings["environmentId"], task_id, exit_on_error=False)
output.write("\nEnded in status '%s'" % (task["status"],))
logs.dump(session, settings, service_label, service_id, task_id, "backup", None)
if task["status"] != "finished":
sys.exit(-1)
def set(variables):
"""Set or update one or more variables. Expects variables in the form <key>=<value>. Multiple variables can be set at once.
Variable changes will not take effect in the application until it is redeployed (via either a push or 'catalyze redeploy')."""
settings = project.read_settings()
session = client.acquire_session(settings)
body = {}
for var in variables:
pieces = var.split("=", 1)
if len(pieces) != 2:
output.error("Expected argument form: <key>=<value>")
else:
body[pieces[0]] = pieces[1]
environment_variables.set(session, settings["environmentId"], settings["serviceId"], body)
def status():
"""Check the status of the environment and every service in it."""
settings = project.read_settings()
session = client.acquire_session(settings)
env = environments.retrieve(session, settings["environmentId"])
output.write("environment state: " + env["state"])
codes = []
noncodes = []
for service in services.list(session, settings["environmentId"]):
if service["type"] != "utility":
if service["type"] == "code":
codes.append("\t%s (size = %s, build status = %s, deploy status = %s)" % (service["label"], service["size"], service["build_status"], service["deploy_status"]))
else:
noncodes.append("\t%s (size = %s, image = %s, status = %s)" % (service["label"], service["size"], service["name"], service["deploy_status"]))
for item in (codes + noncodes):
output.write(item)
def open_console(service_label, command):
"""
Opens a secure console to a code or database service.
For code services, a command is required. This command is executed as root in the context of the application root directory.
For database services, no command is needed - instead, the appropriate command for the database type is run. For example, for a postgres database, psql is run.
"""
global console_closed
settings = project.read_settings()
session = client.acquire_session(settings)
service_id = services.get_by_label(session, settings["environmentId"], service_label)
output.write("Opening console to service '%s'" % (service_id))
task_id = services.request_console(session, settings["environmentId"], service_id, command)["taskId"]
output.write("Waiting for the console to be ready... This might take a bit.")
job_id = services.poll_console_job(session, settings["environmentId"], service_id, task_id)
creds = services.get_console_tokens(session, settings["environmentId"], service_id, job_id)
try:
url = creds["url"].replace("http", "ws")
token = creds["token"]
output.write("Connecting...")
sslopt = {
"ssl_version": ssl.PROTOCOL_TLSv1
}
if "skip_cert_validation" in config.behavior:
sslopt["check_hostname"] = False
ws = ConsoleClient(url, ssl_options = sslopt, headers = [("X-Console-Token", token)])
ws.daemon = False
ws.connect()
with ContextedConsole() as c:
while not console_closed:
data = c.get_data()
if data:
ws.send(data)
finally:
output.write("Cleaning up")
services.destroy_console(session, settings["environmentId"], service_id, job_id)
def exec_list(service_label, page, page_size):
"""List all created backups for the service, sorted from oldest to newest."""
settings = project.read_settings()
session = client.acquire_session(settings)
service_id = services.get_by_label(session, settings["environmentId"], service_label)
raw_backups = services.list_backups(session, settings["environmentId"], service_id, page, page_size)
backup_list = []
for id, body in raw_backups.items():
body["id"] = id
backup_list.append(body)
backup_list.sort(lambda a, b: int((parse_date(a["created_at"]) - parse_date(b["created_at"])).total_seconds()))
if len(backup_list) > 0:
for item in backup_list:
output.write("%s %s (status = %s)" % (item["id"], item["created_at"], item["status"]))
if len(backup_list) == page_size and page == 1:
output.write("(for older backups, try with --page=2 or adjust --page-size)")
elif page == 1:
output.write("No backups created yet for this service.")
def cmd_export(database_label, filepath):
"""Exports all data from a chosen database service.
The export command is accomplished by first creating a backup of the database. Then requesting a temporary access URL to the encrypted backup file. The file is downloaded, decrypted, and stored at the provided location.
If there is an unexpected error, please contact Catalyze support ([email protected]).
"""
settings = project.read_settings()
session = client.acquire_session(settings)
service_id = services.get_by_label(session, settings["environmentId"], database_label)
task_id = services.create_backup(session, settings["environmentId"], service_id)
print("Export started (task ID = %s)" % (task_id,))
output.write("Polling until export finishes.")
job = tasks.poll_status(session, settings["environmentId"], task_id, exit_on_error=False)
if job["status"] != "finished":
output.write("\nExport finished with illegal status \"%s\", aborting." % (job["status"],))
logs.dump(session, settings, database_label, service_id, task_id, "backup", None)
sys.exit(-1)
output.write("\nEnded in status '%s'" % (job["status"],))
backup_id = job["id"]
output.write("Downloading...")
url = services.get_temporary_url(session, settings["environmentId"], service_id, backup_id)
r = requests.get(url, stream=True)
basename = os.path.basename(filepath)
dir = tempfile.mkdtemp()
tmp_filepath = os.path.join(dir, basename)
with open(tmp_filepath, 'wb+') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
f.flush()
output.write("Decrypting...")
decryption = AESCrypto.Decryption(tmp_filepath, job["backup"]["key"], job["backup"]["iv"])
decryption.decrypt(filepath)
os.remove(tmp_filepath)
output.write("%s exported successfully to %s" % (database_label, filepath))
logs.dump(session, settings, database_label, service_id, task_id, "backup", None)
def associate(env_label, service_label, remote):
"""Associates the git repository in the current directory. This means that the service and environment IDs are stored locally, and a git remote is created (default name = "catalyze") so that code can be pushed, built, and deployed."""
session = client.acquire_session()
for env in environments.list(session):
if env["data"]["name"] == env_label:
settings = {
"token": session.token,
"user_id": session.user_id,
"environmentId": env["environmentId"]
}
code_services = [svc for svc in services.list(session, env["environmentId"]) if svc["type"] == "code"]
selected_service = None
if len(code_services) == 0:
output.error("No code service found for \"%s\" environment (%s)" % (env_label, env["environmentId"]))
elif service_label:
for svc in code_services:
if svc["label"] == service_label:
selected_service = svc
break
if selected_service is None:
output.error("No code service found with label '%s'. Labels found: %s" % \
(service_label, ", ".join([svc["label"] for svc in code_services])))
elif len(code_services) > 1:
output.error("Found multiple code services. Must pass one specifically to associate with. Labels found: " + \
", ".join([svc["label"] for svc in code_services]))
else:
selected_service = code_services[0]
if remote in git.remote_list():
git.remote_remove(remote)
git.remote_add(remote, selected_service["source"])
settings["serviceId"] = selected_service["id"]
project.save_settings(settings)
output.write("\"%s\" remote added." % (remote,))
return
output.error("No environment with label \"%s\" found." % (env_label,))
def cmd_import(database_label, filepath, mongo_collection, mongo_database, wipe_first, postgres_database = None, mysql_database = None):
"""Imports a file into a chosen database service.
The import is accomplished by encrypting the file and uploading it to Catalyze. An automated service processes the file according to the passed parameters. The command offers the option to either wait until the processing is finished (and be notified of the end result), or to just kick it off.
The type of file depends on the database. For postgres and mysql, this should be a single SQL script with the extension "sql". For mongo, this should be a tar'd, gzipped archive of the dump that you wish to import, with the extension "tar.gz".
If there is an unexpected error, please contact Catalyze support ([email protected]).
"""
settings = project.read_settings()
session = client.acquire_session(settings)
output.write("Looking up service...")
service_id = services.get_by_label(session, settings["environmentId"], database_label)
environment = environments.retrieve(session, settings["environmentId"])
pod = pods.metadata(session, environment["podId"])
padding_required = pod["importRequiresLength"]
output.write("Importing '%s' to %s (%s)" % (filepath, database_label, service_id))
basename = os.path.basename(filepath)
dir = tempfile.mkdtemp()
key = Random.new().read(32)
iv = Random.new().read(AES.block_size)
output.write("Encrypting...")
try:
enc_filepath = os.path.join(dir, basename)
with open(filepath, 'rb') as file:
with open(enc_filepath, 'wb') as tf:
if padding_required:
filesize = os.path.getsize(filepath)
output.write("File size = %d" % (filesize,))
tf.write(struct.pack("<Q", filesize))
contents = file.read()
contents += b'\0' * (AES.block_size - len(contents) % AES.block_size)
cipher = AES.new(key, mode = AES.MODE_CBC, IV = iv)
tf.write(cipher.encrypt(contents))
with open(enc_filepath, 'rb') as file:
options = {}
if mongo_collection is not None:
options["mongoCollection"] = mongo_collection
if mongo_database is not None:
options["mongoDatabase"] = mongo_database
if postgres_database is not None:
options["pgDatabase"] = postgres_database
if mysql_database is not None:
options["mysqlDatabase"] = mysql_database
output.write("Uploading...")
upload_url = services.get_temporary_upload_url(session, settings["environmentId"], service_id)
resp = services.initiate_import(session, settings["environmentId"],
service_id, upload_url, file,
base64.b64encode(binascii.hexlify(key)),
base64.b64encode(binascii.hexlify(iv)),
wipe_first, options)
task_id = resp["id"]
output.write("Processing import... (id = %s)" % (task_id,))
job = tasks.poll_status(session, settings["environmentId"], task_id, exit_on_error=False)
output.write("\nImport complete (end status = '%s')" % (job["status"],))
logs.dump(session, settings, database_label, service_id, task_id, "restore", None)
if job["status"] != "finished":
sys.exit(-1)
finally:
shutil.rmtree(dir)
def adduser(user_id):
"""Adds another user to the associated environment. The ID required is found via 'catalyze whoami'."""
settings = project.read_settings()
session = client.acquire_session(settings)
environments.add_user(session, settings["environmentId"], user_id)
output.write("Added.")
def rmuser(user_id):
"""Removes another user from the associated environment. The ID required is found via 'catalyze whoami'."""
settings = project.read_settings()
session = client.acquire_session(settings)
environments.remove_user(session, settings["environmentId"], user_id)
output.write("Removed.")
def whoami():
"""Prompts for login, and prints out your ID so that you can be added to an environment by someone else."""
session = client.acquire_session()
output.write("user ID = " + session.user_id)
