class Certbot(Base):
DEFAULT_NAME = "certbot"
DEFAULT_LOGS_DIR = j.sals.fs.join_paths(j.core.dirs.LOGDIR, DEFAULT_NAME)
DEFAULT_CONFIG_DIR = j.sals.fs.join_paths(j.core.dirs.CFGDIR, DEFAULT_NAME)
DEFAULT_WORK_DIR = j.sals.fs.join_paths(j.core.dirs.VARDIR, DEFAULT_NAME)
# the following options match the certbot command arguments
domain = fields.String(required=True)
non_interactive = fields.Boolean(default=True)
agree_tos = fields.Boolean(default=True)
logs_dir = fields.String(default=DEFAULT_LOGS_DIR)
config_dir = fields.String(default=DEFAULT_CONFIG_DIR)
work_dir = fields.String(default=DEFAULT_WORK_DIR)
email = fields.Email()
server = fields.URL()
eab_kid = fields.String()
eab_hmac_key = fields.String()
# for existing certificates
key_path = fields.String()
cert_path = fields.String()
fullchain_path = fields.String()
@property
def run_cmd(self):
args = [self.DEFAULT_NAME]
for name, value in self.to_dict().items():
if name.endswith("_"):
continue
if value:
# append only if the field has a value
name = name.replace("_", "-")
args.append(f"--{name}")
# append the value itself only if it's a boolean value
# boolean options are set by adding name only
if not isinstance(value, bool):
args.append(value)
return args
@property
def install_cmd(self):
# replace "certbot" with "certbot install"
cmd = self.run_cmd
cmd.insert(1, "install")
return cmd
@property
def renew_cmd(self):
# replace "certbot" with "certbot install"
renew_certbot = Certbot(work_dir=self.work_dir, config_dir=self.config_dir, logs_dir=self.logs_dir, domain="")
cmd = renew_certbot.run_cmd
cmd.insert(1, "renew")
return cmd
class PoolPayment(Base):
id = fields.Integer()
farmer_id = fields.Integer()
address = fields.String()
expiration = fields.DateTime()
asset = fields.String()
amount = fields.Integer()
paid = fields.Boolean()
released = fields.Boolean()
canceled = fields.Boolean()
cause = fields.String()
class SSHClient(Client):
name = fields.String()
sshkey = fields.String()
host = fields.String(default="127.0.0.1")
user = fields.String(default="root")
port = fields.Integer()
forward_agent = fields.Boolean(default=True)
connect_timeout = fields.Integer(default=10)
# gateway = ? FIXME: should help with proxyjumps. http://docs.fabfile.org/en/2.4/concepts/networking.html#ssh-gateways
# connect_kwargs = ? FIXME: how to pass dict?
inline_ssh_env = fields.Boolean(default=True) # whether to send environment variables “inline” as prefixes in front of command strings (export VARNAME=value && mycommand here), instead of trying to submit them through the SSH protocol itself (which is the default behavior). This is necessary if the remote server has a restricted AcceptEnv setting (which is the common default).
def __init__(self):
super().__init__()
self.__client = None
def _sshkey(self):
return j.clients.sshkey.get(self.sshkey)
@property
def sshclient(self):
if not self.__client:
connection_kwargs = dict(
host=self.host, user=self.user, port=self.port,
forward_agent=self.forward_agent,
connect_timeout=self.connect_timeout,
connect_kwargs={
"key_filename": self._sshkey().private_key_path,
})
# FIXME: coredump here.
# if self._sshkey.passphrase:
# connection_kwargs["connect_kwargs"]["passphrase"] = self._sshkey().passphrase
self.__client = j.core.executors.RemoteExecutor(**connection_kwargs)
return self.__client
def reset_connection(self):
self.__client = None
def __getattr__(self, name):
return getattr(self.sshclient, name)
def __dir__(self):
return list(self.__dict__.keys()) + dir(self.sshclient)
class Website(Base):
port = fields.Integer(default=80)
ssl = fields.Boolean()
domain = fields.String()
path = fields.String()
locations = fields.Factory(Location)
@property
def path_cfg_dir(self):
return f"{self.parent.path_cfg_dir}/servers"
@property
def path_cfg(self):
return f"{self.path_cfg_dir}/{self.instance_name}.http.conf"
@property
def path_web(self):
return self.parent.path_web
def configure(self):
"""Writes configuration of the website and its locations
"""
j.sals.fs.mkdir(self.path_cfg_dir)
config = render_config_template("website",
base_dir=j.core.dirs.BASEDIR,
website=self)
j.sals.fs.write_file(self.path_cfg, config)
for location_name in self.locations.list_all():
location = self.locations.get(location_name)
location.configure()
class K8s(Base):
id = fields.Integer()
size = fields.Integer()
network_id = fields.String(default="")
ipaddress = fields.IPAddress()
cluster_secret = fields.String(default="")
master_ips = fields.List(fields.IPAddress())
ssh_keys = fields.List(fields.String())
public_ip = fields.Integer()
stats_aggregator = fields.List(fields.Object(Statsaggregator))
info = fields.Object(ReservationInfo)
datastore_endpoint = fields.String(default="")
disable_default_ingress = fields.Boolean(default=True)
SIZES = VMSIZES
def resource_units(self):
resource_units = ResourceUnitAmount()
size = VMSIZES.get(self.size)
if not size:
raise j.exceptions.Input(f"kubernetes size {self.size} not supported")
resource_units.cru += size["cru"]
resource_units.mru += size["mru"]
resource_units.sru += size["sru"]
return resource_units
class Container(Base):
id = fields.Integer()
flist = fields.String(default="")
hub_url = fields.String(default="")
storage_url = fields.String(default="")
environment = fields.Typed(dict)
secret_environment = fields.Typed(dict)
entrypoint = fields.String(default="")
interactive = fields.Boolean(default=True)
volumes = fields.List(fields.Object(ContainerMount))
network_connection = fields.List(fields.Object(ContainerNetworkConnection))
stats = fields.List(fields.Object(ContainerStats))
farmer_tid = fields.Integer()
logs = fields.List(fields.Object(ContainerLogs))
capacity = fields.Object(ContainerCapacity)
info = fields.Object(ReservationInfo)
def resource_units(self):
cap = self.capacity
resource_units = ResourceUnitAmount()
resource_units.cru = cap.cpu
resource_units.mru = round(cap.memory / 1024 * 10000) / 10000
storage_size = round(cap.disk_size / 1024 * 10000) / 10000
storage_size = max(0, storage_size - 50) # we offer the 50 first GB of storage for container root filesystem
if cap.disk_type == DiskType.HDD:
resource_units.hru += storage_size
elif cap.disk_type == DiskType.SSD:
resource_units.sru += storage_size
return resource_units
class Location(Base):
name = fields.String()
path_url = fields.String()
is_auth = fields.Boolean(default=False)
force_https = fields.Boolean(default=False)
path_location = fields.String()
index = fields.String()
use_weblibs = fields.Boolean(default=False)
ipaddr_dest = fields.String()
port_dest = fields.Integer()
path_dest = fields.String()
connection_type = fields.String()
location_type = fields.String()
scheme = fields.String()
config = fields.String()
@property
def path_cfg_dir(self):
return f"{self.parent.path_cfg_dir}/{self.parent.instance_name}_locations"
@property
def path_cfg(self):
return f"{self.path_cfg_dir}/{self.instance_name}.conf"
@property
def path_web(self):
return self.parent.path_web
def write_config(self, content=""):
if not content:
content = render_config_template(f"location_{self.location_type}",
obj=self)
j.sals.fs.write_file(self.path_cfg, content)
def configure(self):
"""Config is a server config file of nginx (in text format)
"""
j.sals.fs.mkdir(self.path_cfg_dir)
if self.location_type in ["static", "spa"]:
if not self.path_location.endswith("/"):
self.path_location += "/"
# if self.location_type == "proxy": Uncomment when dependencies are handled
# j.sals.process.execute("moonc .", cwd=self.path_location)
self.write_config(self.config)
class Farm(Base):
id = fields.Integer()
threebot_id = fields.Integer()
iyo_organization = fields.String(default="")
name = fields.String(default="")
wallet_addresses = fields.List(fields.Object(WalletAddress))
location = fields.Object(Location)
email = fields.Email()
resource_prices = fields.List(fields.Object(ResourceUnitPrice))
prefix_zero = fields.IPRange()
ipaddresses = fields.List(fields.Object(FarmerIP))
enable_custom_pricing = fields.Boolean(default=False)
farm_cloudunits_price = fields.Object(CloudUnitMonthPrice)
is_grid3_compliant = fields.Boolean(default=False)
def __str__(self):
return " - ".join([x for x in [self.name, str(self.location)] if x])
def test_bool(self):
field = fields.Boolean()
self.assertEqual(field.from_raw("yes"), True)
self.assertEqual(field.from_raw("off"), False)
self.assertEqual(field.from_raw(complex(1, 2)), True)
self.assertEqual(field.from_raw(0), False)
with self.assertRaises(ValidationError):
field.validate("habd kteer")
class ZdbNamespace(Base):
id = fields.Integer()
node_id = fields.String(default="")
size = fields.Integer()
mode = fields.Enum(ZDBMode)
password = fields.String(default="")
disk_type = fields.Enum(DiskType)
public = fields.Boolean(default=False)
stats_aggregator = fields.List(fields.Object(Statsaggregator))
info = fields.Object(ReservationInfo)
class Location(Base):
name = fields.String()
path_url = fields.String()
is_auth = fields.Boolean(default=False)
force_https = fields.Boolean(default=False)
path_location = fields.String()
index = fields.String()
ipaddr_dest = fields.String()
port_dest = fields.Integer()
path_dest = fields.String()
location_type = fields.Enum(LocationType)
scheme = fields.String()
@property
def path_cfg_dir(self):
return f"{self.parent.path_cfg_dir}/{self.parent.instance_name}_locations"
@property
def path_cfg(self):
return f"{self.path_cfg_dir}/{self.instance_name}.conf"
@property
def path_web(self):
return self.parent.path_web
def write_config(self, content=""):
if not content:
content = render_config_template(f"location_{self.location_type}",
obj=self)
j.sals.fs.write_file(self.path_cfg, content)
def configure(self):
"""Config is a server config file of nginx (in text format)
"""
j.sals.fs.mkdir(self.path_cfg_dir)
if self.location_type.value in [
LocationType.STATIC.value, LocationType.SPA.value
]:
if not self.path_location.endswith("/"):
self.path_location += "/"
self.write_config(self.config)
class User(Base):
user_code = fields.String(default="")
poll_name = fields.String(default="")
wallets_addresses = fields.List(fields.String())
transaction_hashes = fields.List(fields.String())
tokens = fields.Float(default=0.0)
vote_data = fields.Typed(dict, default={})
extra_data = fields.Typed(dict, default={})
vote_data_weighted = fields.Typed(dict, default={})
has_voted = fields.Boolean(default=False)
manifesto_version = fields.String(default="2.0.0")
class SSHKeyClient(Client):
name = fields.String()
public_key = fields.String()
private_key = fields.String() #should use secret.
private_key_path = fields.String() # should use secret.
passphrase = fields.String(default="") # should use secret.
duration = fields.Integer()
allow_agent = fields.Boolean()
def __init__(self):
super().__init__()
if self.private_key_path and j.sals.fs.exists(self.private_key_path):
self.load_from_file_system()
def load_from_file_system(self):
self.public_key = j.sals.fs.read_file(self.public_key_path)
self.private_key = j.sals.fs.read_file(self.private_key_path)
def generate_keys(self):
if not self.private_key_path:
# TODO: make sure the new sshkey name doesn't exist.
sshkeys_dir = j.sals.fs.join_paths(j.core.config.config_root,
"sshkeys")
j.sals.fs.mkdirs(sshkeys_dir)
self.private_key_path = j.sals.fs.join_paths(
sshkeys_dir, j.data.idgenerator.chars(8))
if self.passphrase and len(self.passphrase) < 5:
raise ValueError(
"invalid passphrase length: should be at least 5 chars.")
cmd = 'ssh-keygen -f {} -N "{}"'.format(self.private_key_path,
self.passphrase)
rc, out, err = j.core.executors.run_local(cmd)
if rc == 0:
self.public_key = j.sals.fs.read_file(self.public_key_path)
self.private_key = j.sals.fs.read_file(self.private_key_path)
else:
raise RuntimeError("couldn't create sshkey")
@property
def public_key_path(self):
return "{}.pub".format(self.private_key_path)
def write_to_filesystem(self):
if not self.private_key:
raise RuntimeError("no private key to write")
if not self.public_key:
raise RuntimeError("no public key to write")
j.sals.fs.write_file(self.private_key_path, self.private_key)
j.sals.fs.write_file(self.public_key_path, self.public_key)
def delete_from_filesystem(self):
pass
class Location(Base):
path_url = fields.String(default="/")
force_https = fields.Boolean(default=False)
path_location = fields.String(default="/")
index = fields.String(default="index.html")
scheme = fields.String(default="http")
host = fields.String(default="127.0.0.1")
port = fields.Integer()
path_dest = fields.String(default="/")
spa = fields.Boolean(default=False)
websocket = fields.Boolean(default=False)
location_type = fields.Enum(LocationType)
is_auth = fields.Boolean(default=False)
is_admin = fields.Boolean(default=False)
custom_config = fields.String(default=None)
proxy_buffering = fields.Enum(ProxyBuffering)
proxy_buffers = fields.String()
proxy_buffer_size = fields.String()
@property
def cfg_dir(self):
return j.sals.fs.join_paths(self.parent.cfg_dir, "locations")
@property
def cfg_file(self):
return j.sals.fs.join_paths(self.cfg_dir, f"{self.instance_name}.conf")
def get_config(self):
return render_config_template(
"location",
base_dir=j.core.dirs.BASEDIR,
location=self,
threebot_connect=j.core.config.get_config().get(
"threebot_connect", True),
https_port=PORTS.HTTPS)
def configure(self):
j.sals.fs.mkdir(self.cfg_dir)
j.sals.fs.write_file(self.cfg_file, self.get_config())
class Node(Base):
node_id = fields.String(default="")
node_id_v1 = fields.String(default="")
farm_id = fields.Integer()
os_version = fields.String(default="")
created = fields.DateTime()
updated = fields.DateTime()
uptime = fields.Integer()
address = fields.String(default="")
location = fields.Object(Location)
total_resources = fields.Object(ResourceUnitAmount)
used_resources = fields.Object(ResourceUnitAmount)
reserved_resources = fields.Object(ResourceUnitAmount)
workloads = fields.Object(WorkloadsAmount)
proofs = fields.List(fields.Object(HardwareProof))
ifaces = fields.List(fields.Object(NodeIface))
public_config = fields.Object(NodePublicIface)
exit_node = fields.Boolean()
approved = fields.Boolean(default=False)
public_key_hex = fields.String(default="")
wg_ports = fields.List(fields.Integer())
free_to_use = fields.Boolean()
class Gateway(Base):
node_id = fields.String(default="")
os_version = fields.String(default="")
farm_id = fields.Integer()
created = fields.DateTime()
updated = fields.DateTime()
uptime = fields.Integer()
address = fields.String(default="")
location = fields.Object(Location)
public_key_hex = fields.String(default="")
workloads = fields.Object(WorkloadsAmount)
managed_domains = fields.List(fields.String())
tcp_router_port = fields.Integer()
dns_nameserver = fields.List(fields.String())
free_to_use = fields.Boolean()
class PaymentResult(Base):
success = fields.Boolean(default=False)
extra_paid = fields.Boolean(default=False)
transactions = fields.List(fields.Object(PaymentTransaction))
def refund_extra(self):
if self.extra_paid and self.parent.refund_extra:
for transaction in self.transactions:
if transaction.success:
trans_amount = transaction.get_amount(self.parent.wallet)
diff = float(trans_amount) - self.parent.amount
if diff <= TRANSACTION_FEES:
self.extra_paid = False
break
sender_address = self.parent.wallet.get_sender_wallet_address(
transaction.transaction_hash)
amount = round(diff - TRANSACTION_FEES, 6)
try:
j.logger.info(
f"refunding extra amount: {amount} of transaction {transaction.transaction_hash} to address: {sender_address}"
)
a = self.parent.wallet._get_asset()
refund_hash = self.parent.wallet.transfer(
sender_address,
amount=amount,
asset=f"{a.code}:{a.issuer}")
self.extra_paid = False
j.logger.info(
f"extra amount: {amount} of transaction {transaction.transaction_hash} refunded successfully in transaction: {refund_hash} to address: {sender_address}"
)
except Exception as e:
j.logger.critical(
f"failed to refund extra amount {amount} for payment: {self.parent.payment_id} due to error: {str(e)}"
)
self.parent.save()
return self.extra_paid
class PaymentTransaction(Base):
transaction_hash = fields.String(required=True)
transaction_refund = fields.Object(PaymentTransactionRefund)
success = fields.Boolean(default=False)
def refund(self, wallet):
if self.transaction_refund.success:
return True
try:
amount = round(
self.get_amount(wallet) - Decimal(TRANSACTION_FEES), 6)
if amount < 0:
self.transaction_refund.success = True
else:
a = wallet._get_asset()
sender_address = wallet.get_sender_wallet_address(
self.transaction_hash)
j.logger.info(
f"refunding transaction: {self.transaction_hash} with amount: {amount} to address: {sender_address}"
)
self.transaction_refund.transaction_hash = wallet.transfer(
sender_address,
amount=amount,
asset=f"{a.code}:{a.issuer}")
self.transaction_refund.success = True
j.logger.info(
f"transaction: {self.transaction_hash} refunded successfully with amount: {amount} to address: {sender_address} in transaction: {self.transaction_refund.transaction_hash}"
)
except Exception as e:
j.logger.critical(
f"failed to refund transaction: {self.transaction_hash} due to error: {str(e)}"
)
return self.transaction_refund.success
def get_amount(self, wallet):
try:
effects = wallet.get_transaction_effects(self.transaction_hash)
except Exception as e:
j.logger.warning(
f"failed to get transaction effects of hash {self.transaction_hash} due to error {str(e)}"
)
raise e
trans_amount = 0
for effect in effects:
if effect.asset_code != "TFT":
continue
trans_amount += effect.amount
return trans_amount
class Website(Base):
port = fields.Integer(default=80)
ssl = fields.Boolean()
domain = fields.String()
path = fields.String()
locations = fields.Factory(Location)
letsencryptemail = fields.String()
@property
def path_cfg_dir(self):
return f"{self.parent.path_cfg_dir}/servers"
@property
def path_cfg(self):
return f"{self.path_cfg_dir}/{self.instance_name}.http.conf"
@property
def path_web(self):
return self.parent.path_web
def generate_certificates(self):
"""Generate ssl certificate if ssl is enabled
"""
if self.ssl:
j.sals.process.execute(
f"certbot --nginx -d {self.domain} --non-interactive --agree-tos -m {self.letsencryptemail} --nginx-server-root {self.parent.path_cfg_dir}"
)
def configure(self, generate_certificates=True):
"""Writes configuration of the website and its locations
Args:
generate_certificates (bool, optional): Will generate certificates if true. Defaults to True.
"""
j.sals.fs.mkdir(self.path_cfg_dir)
config = render_config_template("website",
base_dir=j.core.dirs.BASEDIR,
website=self)
j.sals.fs.write_file(self.path_cfg, config)
for location_name in self.locations.list_all():
location = self.locations.get(location_name)
location.configure()
if generate_certificates:
self.generate_certificates()
class Container(Base):
id = fields.Integer()
flist = fields.String(default="")
hub_url = fields.String(default="")
storage_url = fields.String(default="")
environment = fields.Typed(dict)
secret_environment = fields.Typed(dict)
entrypoint = fields.String(default="")
interactive = fields.Boolean(default=True)
volumes = fields.List(fields.Object(ContainerMount))
network_connection = fields.List(fields.Object(ContainerNetworkConnection))
stats_aggregator = fields.List(fields.Object(Statsaggregator))
farmer_tid = fields.Integer()
logs = fields.List(fields.Object(ContainerLogs))
capacity = fields.Object(ContainerCapacity)
info = fields.Object(ReservationInfo)
class RefundRequest(Base):
payment_id = fields.String(required=True)
success = fields.Boolean(default=False)
refund_transaction_hash = fields.String()
last_tried = fields.DateTime()
amount = fields.Float(default=-1)
def apply(self):
payment = PAYMENT_FACTORY.find_by_id(self.payment_id)
if not payment.is_finished():
j.logger.warning(f"can't refund active payment {self.payment_id}")
return False
self.last_tried = datetime.datetime.utcnow()
amount = payment.amount
# check if refund extra is False. then amount should be same as successful transaction in case of extra was paid but not refunded automatically
sender_address = None
for transaction in payment.result.transactions:
if transaction.success:
sender_address = payment.wallet.get_sender_wallet_address(
transaction.transaction_hash)
if not payment.refund_extra:
amount = float(transaction.get_amount(payment.wallet))
# if a specific amount was specified by the refund request
if self.amount > 0:
amount = self.amount
if amount <= TRANSACTION_FEES or not sender_address:
self.success = True
else:
try:
a = payment.wallet._get_asset()
self.refund_transaction_hash = payment.wallet.transfer(
sender_address,
amount=round(amount - TRANSACTION_FEES, 6),
asset=f"{a.code}:{a.issuer}")
self.success = True
j.logger.info(
f"refund request successful for payment: {self.payment_id} amount: {amount} to address: {sender_address} in transaction: {self.refund_transaction_hash}"
)
except Exception as e:
j.logger.critical(
f"failed to apply refund request for payment {self.payment_id} due to error {str(e)}"
)
self.save()
return self.success
class ZdbNamespace(Base):
id = fields.Integer()
node_id = fields.String(default="")
size = fields.Integer()
mode = fields.Enum(ZDBMode)
password = fields.String(default="")
disk_type = fields.Enum(DiskType)
public = fields.Boolean(default=False)
stats_aggregator = fields.List(fields.Object(Statsaggregator))
info = fields.Object(ReservationInfo)
def resource_units(self):
resource_units = ResourceUnitAmount()
if self.disk_type == DiskType.HDD:
resource_units.hru += self.size
elif self.disk_type == DiskType.SSD:
resource_units.sru += self.size
return resource_units
class GedisHTTPServer(Base):
host = fields.String(default="127.0.0.1")
port = fields.Integer(default=8000)
allow_cors = fields.Boolean(default=True)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._app = Bottle()
self._client = None
http_methods = ["GET", "POST"]
if self.allow_cors:
http_methods.extend(["OPTIONS", "PUT", "DELETE"])
self._app.route("/<package>/<actor>/<method>", http_methods,
self.enable_cors(self.handler, self.allow_cors))
@property
def client(self):
if self._client is None:
self._client = j.clients.gedis.get(self.instance_name)
self._client.disable_deserialization = True
return self._client
def make_response(self, code, content):
response.status = code
response.content_type = "application/json"
return json.dumps(content)
def enable_cors(self, fn, allow_cors=True):
def _enable_cors(*args, **kwargs):
# set CORS headers
response.headers["Access-Control-Allow-Origin"] = "*"
response.headers[
"Access-Control-Allow-Methods"] = "GET, POST, PUT, OPTIONS, DELETE"
response.headers[
"Access-Control-Allow-Headers"] = "Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token"
if request.method != "OPTIONS":
# actual request; reply with the actual response
return fn(*args, **kwargs)
if allow_cors:
return _enable_cors
else:
return fn
def handler(self, package, actor, method):
actors = self.client.actors
actor = getattr(actors, f"{package}_{actor}", None)
if not actor:
return self.make_response(400, {"error": "actor not found"})
method = getattr(actor, method, None)
if not method:
return self.make_response(400, {"error": "method not found"})
kwargs = request.json or dict()
response = method(**kwargs)
if not response.success:
if response.error_type == GedisErrorTypes.NOT_FOUND:
return self.make_response(404, {"error": response.error})
elif response.error_type == GedisErrorTypes.BAD_REQUEST:
return self.make_response(400, {"error": response.error})
elif response.error_type == GedisErrorTypes.PERMISSION_ERROR:
return self.make_response(403, {"error": response.error})
else:
return self.make_response(500, {"error": response.error})
return self.make_response(200, response.result)
@property
def gevent_server(self):
return WSGIServer((self.host, self.port), self._app, spawn=Pool())
class SSHKeyClient(Client):
public_key = fields.String()
private_key = fields.Secret()
private_key_path = fields.Secret()
passphrase = fields.Secret(default="")
duration = fields.Integer()
allow_agent = fields.Boolean()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if self.private_key_path and j.sals.fs.exists(self.private_key_path):
self.load_from_file_system()
def load_from_file_system(self):
""" Load public key and private key from files using private key path and public key path
e.g
ssh_cl = j.clients.sshkey.get("ssh_test")
ssh_cl.load_from_file_system()
"""
self.public_key = j.sals.fs.read_file(self.public_key_path)
self.private_key = j.sals.fs.read_file(self.private_key_path)
def generate_keys(self):
"""Generate a new ssh key
e.g
ssh_cl = j.clients.sshkey.get("ssh_test")
ssh_cl.generate_keys()
"""
if not self.private_key_path:
# TODO: make sure the new sshkey name doesn't exist.
sshkeys_dir = j.sals.fs.join_paths(j.core.config.config_root,
"sshkeys")
j.sals.fs.mkdirs(sshkeys_dir)
self.private_key_path = j.sals.fs.join_paths(
sshkeys_dir, j.data.idgenerator.chars(8))
if self.passphrase and len(self.passphrase) < 5:
raise ValueError(
"invalid passphrase length: should be at least 5 chars.")
cmd = 'ssh-keygen -f {} -N "{}"'.format(self.private_key_path,
self.passphrase)
rc, out, err = j.core.executors.run_local(cmd)
if rc == 0:
self.public_key = j.sals.fs.read_file(self.public_key_path)
self.private_key = j.sals.fs.read_file(self.private_key_path)
else:
raise RuntimeError("couldn't create sshkey")
@property
def public_key_path(self):
""" Get the public key path
e.g
ssh_cl = j.clients.sshkey.get("ssh_test")
ssh_cl.public_key_path -> "/root/.config/jumpscale/sshkeys/tU59lc6P.pub"
Returns
str: the path for public key
"""
return "{}.pub".format(self.private_key_path)
def write_to_filesystem(self):
""" Write public key and private key to files using private key path and public key path.
e.g
ssh_cl = j.clients.sshkey.get("ssh_test")
ssh_cl.write_to_filesystem()
"""
if not self.private_key:
raise RuntimeError("no private key to write")
if not self.public_key:
raise RuntimeError("no public key to write")
j.sals.fs.write_file(self.private_key_path, self.private_key)
j.sals.fs.write_file(self.public_key_path, self.public_key)
def delete_from_filesystem(self):
pass
class UserEntry(Base):
tname = fields.String()
has_agreed = fields.Boolean(default=False)
class SSHClient(Client):
"""
SSHClient has the following properties:
sshkey (str): sshkey to use within that client
host (str): host ip
user (str): user to connect as default: True
port (int): the port to use
forward_agent (bool): forward agent or not (default True)
connect_timeout (int): timeout (default 10 seconds)
"""
sshkey = fields.String(required=True)
host = fields.String(default="127.0.0.1", required=True)
user = fields.String(default="root", required=True)
port = fields.Integer(default=22, required=True)
forward_agent = fields.Boolean(default=True)
connect_timeout = fields.Integer(default=10)
connection_kwargs = fields.Typed(dict, default={})
# gateway = ? FIXME: should help with proxyjumps. http://docs.fabfile.org/en/2.4/concepts/networking.html#ssh-gateways
inline_ssh_env = fields.Boolean(
default=True
) # whether to send environment variables “inline” as prefixes in front of command strings (export VARNAME=value && mycommand here), instead of trying to submit them through the SSH protocol itself (which is the default behavior). This is necessary if the remote server has a restricted AcceptEnv setting (which is the common default).
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.__client = None
@property
def _sshkey(self):
""" Get sshkey client that you have loaded
e.g
JS-NG> localconnection = j.clients.sshclient.new("localconnection")
JS-NG> localconnection.sshkey = "xmonader"
JS-NG> localconnection._sshkey() -> SHKeyClient(_Base__instance_name='xmonader', _Base__parent=None, ...
Returns:
Obj: It returns object of SSHkeyClient
"""
return j.clients.sshkey.get(self.sshkey)
@property
def sshclient(self):
self.validate()
if not self.__client:
self.connection_kwargs[
"key_filename"] = self._sshkey.private_key_path
connection_kwargs = dict(
host=self.host,
user=self.user,
port=self.port,
forward_agent=self.forward_agent,
connect_timeout=self.connect_timeout,
connect_kwargs=self.connection_kwargs,
)
if self._sshkey.passphrase:
connection_kwargs["connect_kwargs"][
"passphrase"] = self._sshkey.passphrase
self.__client = j.core.executors.RemoteExecutor(
**connection_kwargs)
return self.__client
def reset_connection(self):
""" Reset the connection
e.g
localconnection = j.clients.sshclient.new("localconnection")
localconnection.reset_connection()
"""
self.__client = None
class ContainerNetworkConnection(Base):
network_id = fields.String(default="")
ipaddress = fields.IPAddress()
public_ip6 = fields.Boolean()
class UserEntry(Base):
explorer_url = fields.String()
tname = fields.String()
has_agreed = fields.Boolean(default=False)
class Permission(Base):
is_admin = fields.Boolean()
class ZDBClient(Client):
addr = fields.String(default="localhost")
port = fields.Integer(default=9900)
secret_ = fields.String(default="1234567")
nsname = fields.String(default="test")
admin = fields.Boolean(default=False)
mode = fields.Enum(Mode)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# if not self.secret_:
# self.secret_ = j.core.myenv.adminsecret
assert len(self.secret_) > 5
if self.admin:
self.nsname = "default"
self.type = "ZDB"
self._redis = None
self.nsname = self.nsname.lower().strip()
# if j.data.bcdb._master:
# self._model.trigger_add(self._update_trigger)
def _update_trigger(self, obj, action, **kwargs):
if action in ["save", "change"]:
self._redis = None
@property
def redis(self):
if not self._redis:
pool = redis.ConnectionPool(
host=self.addr,
port=self.port,
password=self.secret_,
connection_class=ZDBConnection,
namespace=self.nsname,
namespace_password=self.secret_,
admin=self.admin,
)
self._redis = _patch_redis_client(redis.Redis(connection_pool=pool))
return self._redis
def _key_encode(self, key):
if self.mode.value == Mode.SEQ.value:
if key is None:
key = ""
else:
key = struct.pack("<I", key)
return key
def _key_decode(self, key):
if self.mode.value == Mode.SEQ.value:
key = struct.unpack("<I", key)[0]
return key
def set(self, data, key=None):
key = key or ""
key = self._key_encode(key)
res = self.redis.execute_command("SET", key, data)
if not res:
return res
return self._key_decode(res)
def get(self, key):
key = self._key_encode(key)
return self.redis.execute_command("GET", key)
def exists(self, key):
key = self._key_encode(key)
return self.redis.execute_command("EXISTS", key) == 1
def delete(self, key):
if not key:
raise j.exceptions.Value("key must be provided")
key = self._key_encode(key)
self.redis.execute_command("DEL", key)
def flush(self):
"""
will remove all data from the database DANGEROUS !!!!
This is only allowed on private and password protected namespace
You need to select the namespace before running the command.
:return:
"""
if not self.nsname in ["default", "system"]:
self.redis.execute_command("FLUSH")
def stop(self):
pass
@property
def nsinfo(self):
cmd = self.redis.execute_command("NSINFO", self.nsname)
return _parse_nsinfo(cmd.decode())
def list(self, key_start=None, reverse=False):
"""
list all the keys in the namespace
:param key_start: if specified start to walk from that key instead of the first one, defaults to None
:param key_start: str, optional
:param reverse: decide how to walk the namespace
if False, walk from older to newer keys
if True walk from newer to older keys
defaults to False
:param reverse: bool, optional
:return: list of keys
:rtype: [str]
"""
result = []
for key, data in self.iterate(key_start=key_start, reverse=reverse, keyonly=True):
result.append(key)
return result
def iterate(self, key_start=None, reverse=False, keyonly=False):
"""
walk over all the namespace and yield (key,data) for each entries in a namespace
:param key_start: if specified start to walk from that key instead of the first one, defaults to None
:param key_start: str, optional
:param reverse: decide how to walk the namespace
if False, walk from older to newer keys
if True walk from newer to older keys
defaults to False
:param reverse: bool, optional
:param keyonly: [description], defaults to False
:param keyonly: bool, optional
:raises e: [description]
"""
next = None
data = None
if key_start is not None:
next = self.redis.execute_command("KEYCUR", self._key_encode(key_start))
if not keyonly:
data = self.get(key_start)
yield (key_start, data)
CMD = "SCANX" if not reverse else "RSCAN"
while True:
try:
if not next:
response = self.redis.execute_command(CMD)
else:
response = self.redis.execute_command(CMD, next)
# format of the response
# see https://github.com/threefoldtech/0-db/tree/development#scan
except redis.ResponseError as e:
if e.args[0] == "No more data":
return
raise e
(next, results) = response
for item in results:
keyb, size, epoch = item
key_new = self._key_decode(keyb)
data = None
if not keyonly:
data = self.redis.execute_command("GET", keyb)
yield (key_new, data)
@property
def count(self):
"""
:return: return the number of entries in the namespace
:rtype: int
"""
return self.nsinfo["entries"]
def ping(self):
"""
go to default namespace & ping
:return:
"""
return self.redis.ping()
@property
def next_id(self):
"""
:return: return the next id
:rtype: int
"""
id_bytes = struct.pack("<I", int(self.nsinfo["next_internal_id"], 16))
return int.from_bytes(id_bytes, byteorder="big", signed=True)