def _verify_password(self, password):
instance = USER_THREEBOT_FACTORY.get(f"threebot_{self.threebot_info['solution_uuid']}")
if not instance.verify_secret(password):
return False
zos = get_threebot_zos(instance)
user = zos._explorer.users.get(instance.identity_tid)
words = j.data.encryption.key_to_mnemonic(password.encode().zfill(32))
seed = j.data.encryption.mnemonic_to_key(words)
pubkey = NACL(seed).get_verify_key_hex()
return pubkey == user.pubkey
def test03_change_deployed_threebot_size(self):
"""Test case for changing a threebot size.
**Test Scenario**
- Deploy a threebot.
- Stop the deployed threebot.
- Change the stopped threebot size.
- Check that threebot is reachable.
"""
self.info("Deploy a threebot")
name = self.random_name()
self.secret = self.random_name()
expiration = j.data.time.utcnow().timestamp + 60 * 15
threebot = deployer.deploy_threebot(solution_name=name,
secret=self.secret,
expiration=expiration,
ssh="")
self.solution_uuid = threebot.solution_id
self.info("Stop the deployed threebot")
stop_threebot_solution(self.tname, self.solution_uuid, self.secret)
self.assertTrue(
self.wait_for_threebot_state(name, ThreebotState.STOPPED),
"Threebot didn't stop")
self.info("Start the stopped threebot")
threebot = deployer.change_threebot_size(
name,
self.secret,
flavor="Gold (CPU 2 - Memory 4 GB - Disk 4 GB [SSD])")
self.solution_uuid = threebot.solution_id
threebot_config = USER_THREEBOT_FACTORY.get(
f"threebot_{self.solution_uuid}")
threebot_container_workload = j.sals.zos.get().workloads.get(
threebot_config.threebot_container_wid)
self.assertEqual(threebot_container_workload.capacity.cpu, 2)
self.assertEqual(threebot_container_workload.capacity.memory, 4096)
self.assertEqual(threebot_container_workload.capacity.disk_size, 4096)
self.info("Check that threebot is reachable.")
request = j.tools.http.get(f"http://{threebot.domain}",
verify=False,
timeout=self.timeout)
self.assertEqual(request.status_code, 200)
def _deploy(self):
# 1- add node to network
metadata = {
"form_info": {
"Solution name": self.solution_name,
"chatflow": "threebot"
}
}
self.solution_metadata.update(metadata)
self.workload_ids = []
deploying_message = f"""\
# Deploying your 3Bot...\n\n
<br>It will usually take a few minutes to succeed. Please wait patiently.\n
You will be automatically redirected to the next step once succeeded.
"""
self.md_show_update(dedent(deploying_message), md=True)
# 2- reserve subdomain
if not self.custom_domain:
self.workload_ids.append(
deployer.create_subdomain(
pool_id=self.gateway_pool.pool_id,
gateway_id=self.gateway.node_id,
subdomain=self.domain,
addresses=self.addresses,
solution_uuid=self.solution_id,
identity_name=self.identity_name,
**self.solution_metadata,
))
success = deployer.wait_workload(self.workload_ids[-1],
identity_name=self.identity_name)
if not success:
raise DeploymentFailed(
f"Failed to create subdomain {self.domain} on gateway {self.gateway.node_id} {self.workload_ids[-1]}. The resources you paid for will be re-used in your upcoming deployments.",
wid=self.workload_ids[-1],
identity_name=self.identity_name,
)
# Deploy proxy
proxy_id = deployer.create_proxy(
self.gateway_pool.pool_id,
self.gateway.node_id,
self.domain,
self.secret,
self.identity_name,
**self.solution_metadata,
)
self.workload_ids.append(proxy_id)
success = deployer.wait_workload(self.workload_ids[-1],
identity_name=self.identity_name)
if not success:
raise DeploymentFailed(
f"Failed to create proxy with wid: {self.workload_ids[-1]}. The resources you paid for will be re-used in your upcoming deployments.",
solution_uuid=self.solution_id,
wid=self.workload_ids[-1],
identity_name=self.identity_name,
)
test_cert = j.config.get("TEST_CERT")
# Generate a one-time token to create a user for backup
backup_token = str(j.data.idgenerator.idgenerator.uuid.uuid4())
self.backup_model.token = backup_token
self.backup_model.tname = self.solution_metadata["owner"]
self.backup_model.save()
# 3- deploy threebot container
if "test" in j.core.identity.me.explorer_url:
default_identity = "test"
elif "dev" in j.core.identity.me.explorer_url:
default_identity = "dev"
else:
default_identity = "main"
environment_vars = {
"SDK_VERSION":
self.branch,
"INSTANCE_NAME":
self.solution_name,
"THREEBOT_NAME":
self.threebot_name,
"DOMAIN":
self.domain,
"SSHKEY":
self.public_key,
"TEST_CERT":
"true" if test_cert else "false",
"MARKETPLACE_URL":
f"https://{j.sals.nginx.main.websites.threebot_deployer_threebot_deployer_root_proxy_443.domain}/",
"DEFAULT_IDENTITY":
default_identity,
# email settings
"EMAIL_HOST":
"",
"EMAIL_HOST_USER":
"",
"ESCALATION_MAIL":
"",
# TRC
"REMOTE_IP":
f"{self.gateway.dns_nameserver[0]}",
"REMOTE_PORT":
f"{self.gateway.tcp_router_port}",
"ACME_SERVER_URL":
j.core.config.get("VDC_ACME_SERVER_URL", "https://ca1.grid.tf"),
}
self.network_view = self.network_view.copy()
## Container logs
log_config = j.core.config.get("LOGGING_SINK", {})
if log_config:
log_config[
"channel_name"] = f"{self.threebot_name}-{self.SOLUTION_TYPE}-{self.solution_name}".lower(
)
# Create wallet for the 3bot
threebot_wallet = j.clients.stellar.get(
f"{self.SOLUTION_TYPE}_{self.threebot_name}_{self.solution_name}")
threebot_wallet.save()
threebot_wallet_secret = threebot_wallet.secret
try:
threebot_wallet.activate_through_threefold_service()
except Exception as e:
j.logger.warning(
f"Failed to activate wallet for {self.threebot_name} {self.solution_name} threebot due to {str(e)}"
"3Bot will start without a wallet")
threebot_wallet_secret = ""
self.workload_ids.append(
deployer.deploy_container(
pool_id=self.pool_id,
node_id=self.selected_node.node_id,
network_name=self.network_view.name,
ip_address=self.ip_address,
flist=self.FLIST_URL[self.branch],
env=environment_vars,
cpu=self.query["cru"],
memory=self.query["mru"] * 1024,
disk_size=self.query["sru"] * 1024,
secret_env={
"BACKUP_PASSWORD": self.backup_password,
"BACKUP_TOKEN": backup_token,
"EMAIL_HOST_PASSWORD": "",
"TRC_SECRET": self.secret,
"THREEBOT_WALLET_SECRET": threebot_wallet_secret,
},
interactive=False,
log_config=log_config,
solution_uuid=self.solution_id,
identity_name=self.identity_name,
**self.solution_metadata,
))
success = deployer.wait_workload(self.workload_ids[-1],
identity_name=self.identity_name)
if not success:
raise DeploymentFailed(
f"Failed to create container on node {self.selected_node.node_id} {self.workload_ids[-1]}. The resources you paid for will be re-used in your upcoming deployments.",
solution_uuid=self.solution_id,
wid=self.workload_ids[-1],
identity_name=self.identity_name,
)
self.threebot_url = f"https://{self.domain}/admin"
instance_name = f"threebot_{self.solution_id}"
user_threebot = USER_THREEBOT_FACTORY.get(instance_name)
user_threebot.solution_uuid = self.solution_id
user_threebot.identity_tid = j.core.identity.get(
self.identity_name).tid
user_threebot.name = self.solution_name
user_threebot.owner_tname = self.threebot_name
user_threebot.farm_name = self.farm_name
user_threebot.state = ThreebotState.RUNNING
if hasattr(self, "continent"):
user_threebot.continent = self.continent
if not self.custom_domain:
user_threebot.subdomain_wid = self.workload_ids[-3]
user_threebot.proxy_wid = self.workload_ids[-2]
user_threebot.threebot_container_wid = self.workload_ids[-1]
user_threebot.explorer_url = j.core.identity.get(
self.identity_name).explorer_url
user_threebot.hash_secret(self.backup_password)
user_threebot.save()
def redeploy_threebot_solution(
owner,
solution_uuid,
backup_password,
compute_pool_id=None,
gateway_pool_id=None,
solution_info=None,
node_id=None,
bot=None,
retry=False,
prompt_retry_only=True,
):
"""
Args:
owner (str): threebot_name of the logged in user
solution_uuid (str): of the not-running threebot that needs to be started
compute_pool_id (str): to override the pool id used for container deployment. if not specified, it will use the old pool id
gateway_pool_id (str): to override the pool id used for subdomain and proxy deployment. if not specified, it will use the old pool id. (should override the subdomain specified in solution_info)
solution_info (dict): to override the information used in deployment. if any key is not specified, it will use the old value
"""
if prompt_retry_only:
msg_bot = None
else:
msg_bot = bot
retries = 3 if retry else 1
solution_info = solution_info or {}
owner = text.removesuffix(owner, ".3bot")
j.logger.debug(f"Fetching solution info for uuid {solution_uuid} and owner {owner}")
while retries:
try:
if bot:
bot.md_show_update("Starting your 3Bot...")
threebot = get_threebot_config_instance(owner, solution_uuid)
if not threebot.verify_secret(backup_password):
raise j.exceptions.Validation(f"incorrect secret provided")
zos = get_threebot_zos(threebot)
identity = generate_user_identity(threebot, backup_password, zos)
with threebot_identity_context(identity.instance_name):
j.logger.debug(f"Using identity {identity.instance_name}")
with deployment_context():
zos = j.sals.zos.get(identity.instance_name)
solution_workloads = get_threebot_workloads_by_uuid(solution_uuid, identity.instance_name)
new_solution_info = build_solution_info(solution_workloads, threebot)
j.logger.debug(f"old solution info: {new_solution_info}")
new_solution_info.update(solution_info)
j.logger.debug(f"updated solution info: {new_solution_info}")
gateway_pool_id = gateway_pool_id or new_solution_info["gateway_pool"]
compute_pool_id = compute_pool_id or new_solution_info["compute_pool"]
# deploy using the new information with a new uuid. a new uuid to not conflict with the old one when listing
solution_name = new_solution_info["name"]
backup_model = BACKUP_MODEL_FACTORY.get(f"{solution_name}_{owner}")
new_solution_uuid = uuid.uuid4().hex
metadata = {
"form_info": {"Solution name": solution_name, "chatflow": "threebot"},
"owner": f"{owner}.3bot",
}
# select node and update network
j.logger.debug(f"fetching network {new_solution_info['network']}")
network_view = deployer.get_network_view(
new_solution_info["network"], identity_name=identity.instance_name
)
j.logger.debug(f"searching for available node within pool {compute_pool_id}")
if node_id:
selected_node = zos._explorer.nodes.get(node_id)
else:
selected_node = deployer.schedule_container(
pool_id=compute_pool_id,
cru=new_solution_info["cpu"] + 1,
mru=(new_solution_info["memory"] / 1024) + 1,
sru=(new_solution_info["disk_size"] / 1024) + 0.25,
ip_version="IPv6",
)
j.logger.debug(f"found node with enough capacity {selected_node.node_id}")
j.logger.debug(f"adding node {selected_node.node_id} to network {network_view.name}")
result = deployer.add_network_node(
network_view.name,
selected_node,
compute_pool_id,
network_view,
bot=msg_bot,
identity_name=identity.instance_name,
)
if result:
for wid in result["ids"]:
success = deployer.wait_workload(
wid,
msg_bot,
breaking_node_id=selected_node.node_id,
identity_name=identity.instance_name,
)
if not success:
raise DeploymentFailed(
f"Failed to add node {selected_node.node_id} to network {wid}", wid=wid
)
j.logger.debug(f"node {selected_node.node_id} added to network {network_view.name} successfuly")
j.logger.debug("searching for a free ip address")
network_view = network_view.copy()
ip_address = network_view.get_free_ip(selected_node)
j.logger.debug(f"found a free ip address {ip_address}")
workload_ids = []
j.logger.debug(f"fetching gateway {new_solution_info['gateway']}")
gateway = zos._explorer.gateway.get(new_solution_info["gateway"])
addresses = []
j.logger.debug(f"resolving gateway {gateway.node_id} name servers")
for ns in gateway.dns_nameserver:
try:
addresses.append(j.sals.nettools.get_host_by_name(ns))
except:
j.logger.error(f"failed to resolve name server {ns} of gateway {gateway.node_id}")
if not addresses:
raise StopChatFlow(
f"the gateway specfied {gateway.node_id} doesn't have any valid name servers"
)
domain = new_solution_info["domain"]
j.logger.debug(f"deploying domain {domain} pointing to addresses {addresses}")
workload_ids.append(
deployer.create_subdomain(
pool_id=gateway_pool_id,
gateway_id=gateway.node_id,
subdomain=domain,
addresses=addresses,
solution_uuid=new_solution_uuid,
identity_name=identity.instance_name,
**metadata,
)
)
j.logger.debug(f"waiting for domain workload {workload_ids[-1]} to deploy")
success = deployer.wait_workload(
workload_ids[-1], bot=msg_bot, identity_name=identity.instance_name
)
if not success:
raise DeploymentFailed(
f"Failed to create subdomain {domain} on gateway {gateway.node_id} {workload_ids[-1]}. The resources you paid for will be re-used in your upcoming deployments.",
wid=workload_ids[-1],
identity_name=identity.instance_name,
)
test_cert = j.config.get("TEST_CERT")
j.logger.debug("creating backup token")
backup_token = str(j.data.idgenerator.idgenerator.uuid.uuid4())
backup_model.token = backup_token
backup_model.tname = metadata["owner"]
backup_model.save()
j.logger.debug(f"backup token {backup_token} created for tname {backup_model.tname}")
environment_vars = {
"SDK_VERSION": new_solution_info["branch"],
"INSTANCE_NAME": new_solution_info["name"],
"THREEBOT_NAME": owner,
"DOMAIN": domain,
"SSHKEY": new_solution_info["public_key"],
"TEST_CERT": "true" if test_cert else "false",
"MARKETPLACE_URL": f"https://{j.sals.nginx.main.websites.threebot_deployer_threebot_deployer_root_proxy_443.domain}/",
"DEFAULT_IDENTITY": "test" if "test" in j.core.identity.me.explorer_url else "main",
}
j.logger.debug(f"deploying threebot container with environment {environment_vars}")
log_config = j.core.config.get("LOGGING_SINK", {})
if log_config:
log_config["channel_name"] = f'{owner}-{new_solution_info["name"]}'.lower()
workload_ids.append(
deployer.deploy_container(
pool_id=compute_pool_id,
node_id=selected_node.node_id,
network_name=network_view.name,
ip_address=ip_address,
flist=new_solution_info["flist"],
env=environment_vars,
cpu=new_solution_info["cpu"],
memory=new_solution_info["memory"],
disk_size=new_solution_info["disk_size"],
secret_env={"BACKUP_PASSWORD": backup_password, "BACKUP_TOKEN": backup_token},
interactive=False,
log_config=log_config,
solution_uuid=new_solution_uuid,
identity_name=identity.instance_name,
**metadata,
)
)
j.logger.debug(f"wating for threebot container workload {workload_ids[-1]} to be deployed")
success = deployer.wait_workload(
workload_ids[-1], bot=msg_bot, identity_name=identity.instance_name
)
if not success:
raise DeploymentFailed(
f"Failed to create container on node {selected_node.node_id} {workload_ids[-1]}. The resources you paid for will be re-used in your upcoming deployments.",
solution_uuid=new_solution_uuid,
wid=workload_ids[-1],
identity_name=identity.instance_name,
)
j.logger.debug(f"threebot container workload {workload_ids[-1]} deployed successfuly")
trc_log_config = j.core.config.get("LOGGING_SINK", {})
if trc_log_config:
trc_log_config["channel_name"] = f'{owner}-{new_solution_info["name"]}-trc'.lower()
identity_tid = identity.tid
secret = f"{identity_tid}:{uuid.uuid4().hex}"
j.logger.debug(f"deploying trc container")
workload_ids.extend(
deployer.expose_address(
pool_id=compute_pool_id,
gateway_id=gateway.node_id,
network_name=network_view.name,
local_ip=ip_address,
port=80,
tls_port=443,
trc_secret=secret,
node_id=selected_node.node_id,
reserve_proxy=True,
domain_name=domain,
proxy_pool_id=gateway_pool_id,
solution_uuid=new_solution_uuid,
log_config=trc_log_config,
identity_name=identity.instance_name,
**metadata,
)
)
j.logger.debug(f"wating for trc container workload {workload_ids[-1]} to be deployed")
success = deployer.wait_workload(
workload_ids[-1], bot=msg_bot, identity_name=identity.instance_name
)
if not success:
raise DeploymentFailed(
f"Failed to create TRC container on node {selected_node.node_id} {workload_ids[-1]}. The resources you paid for will be re-used in your upcoming deployments.",
solution_uuid=new_solution_uuid,
wid=workload_ids[-1],
identity_name=identity.instance_name,
)
j.logger.debug(f"trc container workload {workload_ids[-1]} deployed successfuly")
j.logger.debug(f"fetching farm information of pool {compute_pool_id}")
farm_id = deployer.get_pool_farm_id(compute_pool_id)
farm = zos._explorer.farms.get(farm_id)
j.logger.debug(f"saving new threebot local config with uuid {new_solution_uuid}")
instance_name = f"threebot_{new_solution_uuid}"
user_threebot = USER_THREEBOT_FACTORY.get(instance_name)
user_threebot.solution_uuid = new_solution_uuid
user_threebot.identity_tid = identity.tid
user_threebot.name = solution_name
user_threebot.owner_tname = threebot.owner_tname
user_threebot.farm_name = farm.name
user_threebot.state = ThreebotState.RUNNING
user_threebot.continent = farm.location.continent
user_threebot.explorer_url = identity.explorer_url
user_threebot.subdomain_wid = workload_ids[-4]
user_threebot.threebot_container_wid = workload_ids[-3]
user_threebot.trc_container_wid = workload_ids[-2]
user_threebot.reverse_proxy_wid = workload_ids[-1]
user_threebot.save()
j.logger.debug(f"threebot local config of uuid {new_solution_uuid} saved")
j.logger.debug(f"deleting old threebot local config with uuid {solution_uuid}")
USER_THREEBOT_FACTORY.delete(f"threebot_{solution_uuid}")
j.logger.debug("deployment successful")
return user_threebot
except DeploymentFailed as e:
retries -= 1
if retries > 0:
j.logger.error(f"3Bot {solution_uuid} redeployment failed. retrying {retries}")
if bot and e.wid:
bot.md_show_update(f"Deployment Failed for wid {e.wid}. retrying {retries} ....")
def _deploy(self):
# 1- add node to network
metadata = {
"form_info": {
"Solution name": self.solution_name,
"chatflow": "threebot"
}
}
self.solution_metadata.update(metadata)
self.workload_ids = []
deploying_message = f"""\
# Deploying your 3Bot...\n\n
<br>It will usually take a few minutes to succeed. Please wait patiently.\n
You will be automatically redirected to the next step once succeeded.
"""
self.md_show_update(dedent(deploying_message), md=True)
# 2- reserve subdomain
if not self.custom_domain:
self.workload_ids.append(
deployer.create_subdomain(
pool_id=self.gateway_pool.pool_id,
gateway_id=self.gateway.node_id,
subdomain=self.domain,
addresses=self.addresses,
solution_uuid=self.solution_id,
identity_name=self.identity_name,
**self.solution_metadata,
))
success = deployer.wait_workload(self.workload_ids[-1],
identity_name=self.identity_name)
if not success:
raise DeploymentFailed(
f"Failed to create subdomain {self.domain} on gateway {self.gateway.node_id} {self.workload_ids[-1]}. The resources you paid for will be re-used in your upcoming deployments.",
wid=self.workload_ids[-1],
identity_name=self.identity_name,
)
test_cert = j.config.get("TEST_CERT")
# Generate a one-time token to create a user for backup
backup_token = str(j.data.idgenerator.idgenerator.uuid.uuid4())
self.backup_model.token = backup_token
self.backup_model.tname = self.solution_metadata["owner"]
self.backup_model.save()
# 3- deploy threebot container
environment_vars = {
"SDK_VERSION":
self.branch,
"INSTANCE_NAME":
self.solution_name,
"THREEBOT_NAME":
self.threebot_name,
"DOMAIN":
self.domain,
"SSHKEY":
self.public_key,
"TEST_CERT":
"true" if test_cert else "false",
"MARKETPLACE_URL":
f"https://{j.sals.nginx.main.websites.threebot_deployer_threebot_deployer_root_proxy_443.domain}/",
"DEFAULT_IDENTITY":
"test" if "test" in j.core.identity.me.explorer_url else "main",
# email settings
"EMAIL_HOST":
self.email_host,
"EMAIL_HOST_USER":
self.email_host_user,
"ESCALATION_MAIL":
self.escalation_mail_address,
}
self.network_view = self.network_view.copy()
## Container logs
log_config = j.core.config.get("LOGGING_SINK", {})
if log_config:
log_config[
"channel_name"] = f"{self.threebot_name}-{self.SOLUTION_TYPE}-{self.solution_name}".lower(
)
self.workload_ids.append(
deployer.deploy_container(
pool_id=self.pool_id,
node_id=self.selected_node.node_id,
network_name=self.network_view.name,
ip_address=self.ip_address,
flist=self.FLIST_URL[self.branch],
env=environment_vars,
cpu=self.container_resources["cru"],
memory=self.container_resources["mru"] * 1024,
disk_size=self.container_resources["sru"] * 1024,
secret_env={
"BACKUP_PASSWORD": self.backup_password,
"BACKUP_TOKEN": backup_token,
"EMAIL_HOST_PASSWORD": self.email_host_password,
},
interactive=False,
log_config=log_config,
solution_uuid=self.solution_id,
identity_name=self.identity_name,
**self.solution_metadata,
))
success = deployer.wait_workload(self.workload_ids[-1],
identity_name=self.identity_name)
if not success:
raise DeploymentFailed(
f"Failed to create container on node {self.selected_node.node_id} {self.workload_ids[-1]}. The resources you paid for will be re-used in your upcoming deployments.",
solution_uuid=self.solution_id,
wid=self.workload_ids[-1],
identity_name=self.identity_name,
)
# 4- expose threebot container
wid, proxy_id = deployer.expose_address(
pool_id=self.pool_id,
gateway_id=self.gateway.node_id,
network_name=self.network_view.name,
local_ip=self.ip_address,
port=80,
tls_port=443,
trc_secret=self.secret,
node_id=self.selected_node.node_id,
reserve_proxy=True,
domain_name=self.domain,
proxy_pool_id=self.gateway_pool.pool_id,
solution_uuid=self.solution_id,
log_config=self.trc_log_config,
identity_name=self.identity_name,
**self.solution_metadata,
)
self.workload_ids.append(wid)
self.workload_ids.append(proxy_id)
success = deployer.wait_workload(self.workload_ids[-1],
identity_name=self.identity_name)
if not success:
raise DeploymentFailed(
f"Failed to create TRC container on node {self.selected_node.node_id} {self.workload_ids[-1]}. The resources you paid for will be re-used in your upcoming deployments.",
solution_uuid=self.solution_id,
wid=self.workload_ids[-1],
identity_name=self.identity_name,
)
self.threebot_url = f"https://{self.domain}/admin"
instance_name = f"threebot_{self.solution_id}"
user_threebot = USER_THREEBOT_FACTORY.get(instance_name)
user_threebot.solution_uuid = self.solution_id
user_threebot.identity_tid = j.core.identity.get(
self.identity_name).tid
user_threebot.name = self.solution_name
user_threebot.owner_tname = self.threebot_name
user_threebot.farm_name = self.farm_name
user_threebot.state = ThreebotState.RUNNING
if hasattr(self, "continent"):
user_threebot.continent = self.continent
if not self.custom_domain:
user_threebot.subdomain_wid = self.workload_ids[-4]
user_threebot.threebot_container_wid = self.workload_ids[-3]
user_threebot.trc_container_wid = self.workload_ids[-2]
user_threebot.reverse_proxy_wid = self.workload_ids[-1]
user_threebot.explorer_url = j.core.identity.get(
self.identity_name).explorer_url
user_threebot.hash_secret(self.backup_password)
user_threebot.save()