def download(url, localpath, username=None, passwd=None, overwrite=True):
"""Download a url to a file or a directory, supported protocols: http, https, ftp, file
@param url: URL to download from
@type url: string
@param localpath: filename or directory to download the url to pass - to return data
@type localpath: string
@param username: username for the url if it requires authentication
@type username: string
@param passwd: password for the url if it requires authentication
@type passwd: string
"""
if not url:
raise Value("URL can not be None or empty string")
if not localpath:
raise Value(
"Local path to download the url to can not be None or empty string"
)
filename = ""
if localpath == "-":
filename = "-"
if jumpscale.sals.fs.exists(localpath) and jumpscale.sals.fs.is_dir(
localpath):
filename = jumpscale.sals.fs.join_paths(
localpath, jumpscale.sals.fs.basename(url))
else:
if jumpscale.sals.fs.is_dir(jumpscale.sals.fs.dirname(localpath)):
filename = localpath
else:
raise Value("Local path is an invalid path")
if not jumpscale.sals.fs.exists(filename):
overwrite = True
if overwrite:
if username and passwd and jumpscale.tools.http.urllib3.util.parse_url(
url).scheme == "ftp":
url = url.split("://")[0] + "://%s:%s@" % (
username, passwd) + url.split("://")[1]
response = jumpscale.tools.http.get(
url,
stream=True,
auth=jumpscale.tools.http.auth.HTTPBasicAuth(username, passwd))
response.raise_for_status()
if filename != "-":
with open(filename, "wb") as fw:
for chunk in response.iter_content(chunk_size=2048):
if chunk: # filter out keep-alive new chunks
fw.write(chunk)
return
else:
return response.content
def ping_machine(ip, pingtimeout=60, allowhostname=True):
"""Ping a machine to check if it's up/running and accessible
@param ip: Machine Ip Address
@param pingtimeout: time in sec after which ip will be declared as unreachable
@param allowhostname: allow pinging on hostname (default is false)
@rtype: True if machine is pingable, False otherwise
"""
if not allowhostname:
if not IPAddress().check(ip):
raise Value(
"Invalid ip address, set allowedhostname to use hostnames")
start = time.time()
pingsucceeded = False
while time.time() - start < pingtimeout:
if jumpscale.data.platform.is_linux():
# ping -c 1 -W 1 IP
exitcode, _, _ = jumpscale.core.executors.run_local(
f"ping -c 1 -W 1 -w 1 {ip}", warn=True, hide=True)
elif jumpscale.data.platform.is_osx():
exitcode, _, _ = jumpscale.core.executors.run_local(
f"ping -c 1 {ip}", warn=True, hide=True)
if exitcode == 0:
pingsucceeded = True
return True
time.sleep(1)
if not pingsucceeded:
return False
def stop(name):
"""Stops an existing container
Args:
name (str): name of the container
"""
if not docker_client.exists(name):
raise Value("Container with specified name doesn't exist")
docker_client.stop(name)
def me(self):
if not self._me:
config = get_config()
default = config["threebot"]["default"]
if default:
self.__class__._me = self.get(name=default)
else:
for identity in self.list_all():
self.__class__._me = self.get(identity)
break
else:
raise Value("No configured identity found")
return self._me
def install(name, image, development: bool = False, volumes=None):
"""Creates a container
Args:
name (str): name of the container
image (str): container image.
development (bool, optional): if true will mount codedir. Defaults to False.
volumes (dict, optional): paths to be mounted
Raises:
Value: Container with specified name already exists
"""
if docker_client.exists(name):
raise Value("Container with specified name already exists")
volumes = volumes or {}
if development:
volumes = {Dirs.CODEDIR: {"bind": "/sandbox/code", "mode": "rw"}}
print(f"Creating container {name}")
return docker_client.run(name, image, entrypoint="/sbin/my_init", volumes=volumes, detach=True)
def register(self, app_name, module_name=None):
"""
Register and bind given module (and sub-modules) logs with a given app name
Will also add the app to `applications` set in redis.
If `module_name` is not passed or empty, it would be the caller module name.
Args:
app_name (str): app name
module_name (str, optional): module name. Defaults to None.
"""
if not app_name:
raise Value(f"invalid app name '{app_name}'")
if not module_name:
module_name = self._get_caller_module()
self._module_apps[module_name] = app_name
self._add_app(app_name)
self.info(
f"Logging from '{module_name}' is now bound to '{app_name}' app")
def verify_configuration(self):
"""
Verifies passed arguments to constructor
Raises: NotFound incase tid is passed but does not exists on the explorer
Raises: Input: when params are missing
"""
if not self.words:
raise Input("Words are mandotory for an indentity")
if self._tid != -1:
resp = requests.get(f"{self.explorer_url}/users/{self._tid}")
resp.raise_for_status()
user = User.from_dict(resp.json())
if self.tname and self.tname != user.name:
raise Input("Name of user does not match name in explorer")
self.tname = user.name
if self.nacl.get_verify_key_hex() != user.pubkey:
raise Input(
"The verify key on your local system does not correspond with verify key on the TFGrid explorer"
)
else:
for key in ["email", "tname"]:
if not getattr(self, key):
raise Value(f"Threebot {key} not configured")
def get_nic_type(interface):
"""Get Nic Type on a certain interface
Args:
interface (str): interface name
Raises:
Runtime: if ethtool not installed on the system
Value: if interface given is invalid
Returns:
str: type of the interface
"""
output = ""
if jumpscale.data.platform.is_linux():
if jumpscale.sals.fs.exists(f"/sys/class/net/{interface}"):
output = jumpscale.sals.fs.read_file(
f"/sys/class/net/{interface}/type")
if output.strip() == "32":
return "INFINIBAND"
else:
if jumpscale.sals.fs.exists("/proc/net/vlan/%s" % (interface)):
return "VLAN"
exitcode, _, _ = jumpscale.core.executors.run_local(
"which ethtool", hide=True, warn=True)
if exitcode != 0:
raise Runtime("Ethtool is not installed on this system!")
exitcode, output, _ = jumpscale.core.executors.run_local(
f"ethtool -i {interface}", hide=True, warn=True)
if exitcode != 0:
return "VIRTUAL"
match = re.search(r"^driver:\s+(?P<driver>\w+)\s*$", output,
re.MULTILINE)
if match and match.group("driver") == "tun":
return "VIRTUAL"
if match and match.group("driver") == "bridge":
return "VLAN"
return "ETHERNET_GB"
elif jumpscale.data.platform.is_osx():
command = f"ifconfig {interface}"
exitcode, output, err = jumpscale.core.executors.run_local(command,
hide=True,
warn=True)
if exitcode != 0:
# temporary plumb the interface to lookup its mac
jumpscale.core.executors.run_local(f"{command} plumb", hide=True)
exitcode, output, err = jumpscale.core.executors.run_local(
command, hide=True)
jumpscale.core.executors.run_local(f"{command} unplumb", hide=True)
if output.find("ipib") >= 0:
return "INFINIBAND"
else:
# work with interfaces which are subnetted on vlans eq e1000g5000:1
interfacepieces = interface.split(":")
interface = interfacepieces[0]
match = re.search(r"^\w+?(?P<interfaceid>\d+)$", interface,
re.MULTILINE)
if not match:
raise Value(f"Invalid interface {interface}")
if len(match.group("interfaceid")) >= 4:
return "VLAN"
else:
if len(interfacepieces) > 1:
return "VIRTUAL"
else:
return "ETHERNET_GB"